yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by wangnan9279
Commit 0580a2892741377db9f81cb8f79326191e92ccd5 1 parent 9880bf78

tblproduct索引用表数据重建

Inline Side-by-side
Showing 12 changed files with 495 additions and 300 deletions
@@ -5,28 +5,17 @@ import com.alibaba.fastjson.JSONArray; @@ -5,28 +5,17 @@ import com.alibaba.fastjson.JSONArray;
5 import com.alibaba.fastjson.JSONObject; 5 import com.alibaba.fastjson.JSONObject;
6 import com.yoho.search.base.utils.HttpClientUtils; 6 import com.yoho.search.base.utils.HttpClientUtils;
7 import com.yoho.search.base.utils.MD5Util; 7 import com.yoho.search.base.utils.MD5Util;
8 -import com.yoho.search.consumer.service.logicService.common.ConsumerConfiger;  
9 import com.yoho.search.consumer.index.common.IIndexBuilder; 8 import com.yoho.search.consumer.index.common.IIndexBuilder;
10 -import com.yoho.search.consumer.index.fullbuild.tblProductNew.TblProductSkcMapBuilder;  
11 -import com.yoho.search.consumer.index.fullbuild.tblProductNew.TblProductSkuMapBuilder;  
12 -import com.yoho.search.consumer.service.daoService.TblBrandService;  
13 -import com.yoho.search.consumer.service.daoService.TblProductService;  
14 -import com.yoho.search.consumer.service.daoService.TblProductSkcService;  
15 -import com.yoho.search.consumer.service.daoService.TblSiteService;  
16 -import com.yoho.search.consumer.service.logicService.tbl.TblAdaptorLogicService;  
17 -import com.yoho.search.consumer.service.logicService.tbl.util.ImagesUtils;  
18 -import com.yoho.search.dal.model.*;  
19 -import org.apache.commons.collections.CollectionUtils; 9 +import com.yoho.search.consumer.service.logicService.common.ConsumerConfiger;
20 import org.apache.commons.lang.StringUtils; 10 import org.apache.commons.lang.StringUtils;
21 import org.slf4j.Logger; 11 import org.slf4j.Logger;
22 import org.slf4j.LoggerFactory; 12 import org.slf4j.LoggerFactory;
23 -import org.springframework.beans.BeanUtils;  
24 import org.springframework.beans.factory.annotation.Autowired; 13 import org.springframework.beans.factory.annotation.Autowired;
25 import org.springframework.stereotype.Component; 14 import org.springframework.stereotype.Component;
26 15
27 -import java.math.BigDecimal;  
28 -import java.util.*;  
29 -import java.util.stream.Collectors; 16 +import java.util.List;
  17 +import java.util.Map;
  18 +import java.util.UUID;
30 19
31 /** 20 /**
32 * tblproduct索引，使用调用接口的方式重建。 21 * tblproduct索引，使用调用接口的方式重建。
@@ -38,290 +27,15 @@ public class TblProductIndexBuilder extends IIndexBuilder { @@ -38,290 +27,15 @@ public class TblProductIndexBuilder extends IIndexBuilder {
38 27
39 @Autowired 28 @Autowired
40 private ConsumerConfiger configurer; 29 private ConsumerConfiger configurer;
41 - @Autowired  
42 - private TblProductService tblProductService;  
43 - @Autowired  
44 - private TblProductSkcService tblProductSkcService;  
45 - @Autowired  
46 - private TblSiteService tblSiteService;  
47 - @Autowired  
48 - private TblBrandService tblBrandService;  
49 - @Autowired  
50 - private TblProductSkcMapBuilder tblProductSkcMapBuilder;  
51 - @Autowired  
52 - private TblProductSkuMapBuilder tblProductSkuMapBuilder;  
53 - @Autowired  
54 - private TblAdaptorLogicService tblAdaptorLogicService;  
55 30
56 @Override 31 @Override
57 public int getTotalCount() throws Exception { 32 public int getTotalCount() throws Exception {
58 - //走接口构建，否则用表数据构建  
59 - if (configurer.getTblRebuildUseUrl()) {  
60 - return getCount();  
61 - }  
62 - return tblProductService.count(); 33 + return getCount();
63 } 34 }
64 35
65 @Override 36 @Override
66 public List<?> getPageLists(int offset, int limit) throws Exception { 37 public List<?> getPageLists(int offset, int limit) throws Exception {
67 - //走接口构建，否则用表数据构建  
68 - if (configurer.getTblRebuildUseUrl()) {  
69 - return getPageList(offset, limit);  
70 - }  
71 -  
72 - //查询tblProduct列表  
73 - List<TblProduct> tblProductList = tblProductService.getPageLists(offset, limit);  
74 - //构建tblSiteMap  
75 - Map<Integer, TblSite> tblSiteMap = getSiteMap();  
76 - //构建tblBrandMap  
77 - Map<Integer, TblBrand> tblBrandMap = getTblBrandMap();  
78 - //skc  
79 - Map<Integer, List<TblProductSkc>> tblProductSkcMap = tblProductSkcMapBuilder.build(tblProductList);  
80 - //sku  
81 - Map<Integer, List<TblProductSku>> tblProductSkuMap = tblProductSkuMapBuilder.build(tblProductList);  
82 - //构建ColorIds字段Map  
83 - Map<Integer, String> colorIdsMap = getColorIdsMap(tblProductList);  
84 -  
85 - //构建TblProductBO  
86 - List<TblProductBO> tblProductBOList = new ArrayList<>();  
87 - for (TblProduct tblProduct : tblProductList) {  
88 - //拼装与skc有关的字段，获取skn下面的skc列表  
89 - List<TblProductSkc> tblProductSkcList = tblProductSkcMap.get(tblProduct.getProductSkn());  
90 - //如果某个skn下不包含skc则跳过  
91 - if (CollectionUtils.isEmpty(tblProductSkcList)) {  
92 - continue;  
93 - }  
94 - String sizeList = "";  
95 - String url = "";  
96 - List<JSONObject> skc = new ArrayList<>();  
97 - //遍历skc列表  
98 - for (TblProductSkc tblProductSkc : tblProductSkcList) {  
99 - //构建cover和url  
100 - JSONArray jsonArray = JSONArray.parseArray(tblProductSkc.getPics());  
101 - String cover = "";  
102 - String imageUrl = "";  
103 - if (jsonArray != null) {  
104 - //默认的cover和url是第一个skc的  
105 - JSONObject pic1 = (JSONObject) jsonArray.get(0);  
106 - cover = pic1.get("src").toString();  
107 - //满足条件重新设置cover  
108 - for (int i = 0; i < jsonArray.size(); i++) {  
109 - JSONObject pic = (JSONObject) jsonArray.get(i);  
110 - if (pic.get("cover") != null && pic.get("cover").equals("1")) {  
111 - cover = pic.get("src").toString();  
112 - }  
113 - }  
114 - imageUrl = ImagesUtils.getImageUrl(cover, 112, 154);  
115 - //满足条件设置url  
116 - if ("1".equals(tblProductSkc.getIsDefault()) && !"1".equals(tblProductSkc.getIsDeleted())) {  
117 - imageUrl = ImagesUtils.getImageUrl(cover, 112, 154);  
118 - url = imageUrl.substring(0, imageUrl.indexOf("?"));  
119 - }  
120 - }  
121 - //拼装size_List  
122 - List<TblProductSku> tblProductSkuList = tblProductSkuMap.get(tblProduct.getProductSkn());  
123 - if (CollectionUtils.isNotEmpty(tblProductSkuList)) {  
124 - for (TblProductSku tblProductSku : tblProductSkuList) {  
125 - if (tblProductSku.getProductSkc().intValue() == tblProductSkc.getProductSkc().intValue()) {  
126 - if (!"1".equals(tblProductSku.getIsDeleted())) {  
127 - sizeList += tblProductSku.getSize() + ",";  
128 - }  
129 - }  
130 - }  
131 - }  
132 - //拼装skc列表  
133 - JSONObject skcObject = new JSONObject();  
134 - skcObject.put("product_skc", tblProductSkc.getProductSkc());  
135 - skcObject.put("color_sys_id", tblProductSkc.getColorSysId());  
136 - skcObject.put("color", tblProductSkc.getColor());  
137 - skcObject.put("cover", imageUrl.substring(0, imageUrl.indexOf("?")));  
138 - skcObject.put("format_cover", ImagesUtils.template2(imageUrl.substring(0, imageUrl.indexOf("?")), "global", "center", "d2hpdGU="));  
139 - skc.add(skcObject);  
140 - }  
141 -  
142 - //设置默认url  
143 - if (StringUtils.isBlank(url)) {  
144 - TblProductSkc tblProductSkc = tblProductSkcList.get(0);  
145 - JSONArray jsonArray = JSONArray.parseArray(tblProductSkc.getPics());  
146 - JSONObject pic1 = (JSONObject) jsonArray.get(0);  
147 - String cover = pic1.get("src").toString();  
148 - String imageUrl = ImagesUtils.getImageUrl(cover, 112, 154);  
149 - url = imageUrl.substring(0, imageUrl.indexOf("?"));  
150 - }  
151 -  
152 - //拼装TblProductBO数据  
153 - TblProductBO tblProductBO = new TblProductBO();  
154 - //简单字段:productSkn,productName,brandId,countryId,currencyId,siteId,gender,isLimited,sortOne,sortTwo,sortThree,sortFour,status,stockStatus,updateTime,shelfTime  
155 - BeanUtils.copyProperties(tblProduct, tblProductBO);  
156 - //colorIds  
157 - String colorIds = colorIdsMap.get(tblProduct.getProductSkn());  
158 - tblProductBO.setColorIds(colorIds);  
159 - //goodsPrice,tagPrice,orignPrice  
160 - if (tblProductBO.getGoodsPrice() != null) {  
161 - tblProductBO.setGoodsPrice(tblProductBO.getGoodsPrice().setScale(2, BigDecimal.ROUND_DOWN));  
162 - }  
163 - if (tblProductBO.getTagPrice() != null) {  
164 - tblProductBO.setTagPrice(tblProductBO.getTagPrice().setScale(2, BigDecimal.ROUND_DOWN));  
165 - }  
166 - if (tblProductBO.getOrignPrice() != null) {  
167 - tblProductBO.setOrignPrice(tblProductBO.getOrignPrice().setScale(2, BigDecimal.ROUND_DOWN));  
168 - }  
169 - //isPlane  
170 - if (tblProductBO.getCountryId() == 86) {  
171 - tblProductBO.setIsPlane("N");  
172 - }  
173 - //showStatus  
174 - String showStatus = this.buildShowStatus(tblProduct, tblSiteMap, tblBrandMap);  
175 - tblProductBO.setShowStatus(showStatus);  
176 - //cover  
177 - tblProductBO.setCover(url);  
178 - //formatCover  
179 - tblProductBO.setFormatCover(ImagesUtils.template2(url, null, "center", "d2hpdGU="));  
180 - //skc  
181 - tblProductBO.setSkc(skc);  
182 - //sizeLIst  
183 - tblProductBO.setSizeList(sizeList.substring(0, sizeList.length() - 1));  
184 -  
185 - //塞入列表  
186 - tblProductBOList.add(tblProductBO);  
187 - }  
188 -  
189 - //融合进有货的商品  
190 - if (offset == 0) {  
191 - List<Integer> brandIdList = this.getBrandIdList();  
192 - if (CollectionUtils.isEmpty(brandIdList)) {  
193 - return tblProductBOList;  
194 - }  
195 - Map<Integer, List<TblBrand>> brandMap = this.getBrandMap();  
196 - if (brandMap.isEmpty()) {  
197 - return tblProductBOList;  
198 - }  
199 - List<TplProductBo> tplProductBoList = tblAdaptorLogicService.convertYohoToTplProduct(brandIdList);  
200 - if (CollectionUtils.isEmpty(tplProductBoList)) {  
201 - return tblProductBOList;  
202 - }  
203 - for (TplProductBo tplProductBo : tplProductBoList) {  
204 - if (StringUtils.isBlank(tplProductBo.getOrignPrice()) || StringUtils.isBlank(tplProductBo.getGoodsPrice())) {  
205 - continue;  
206 - }  
207 - TblProductBO tblProductBO = new TblProductBO();  
208 - BeanUtils.copyProperties(tplProductBo, tblProductBO);  
209 - //brandId  
210 - List<TblBrand> tblBrandList = brandMap.get(tplProductBo.getBrandId());  
211 - if (CollectionUtils.isNotEmpty(tblBrandList)) {  
212 - TblBrand tblBrand = tblBrandList.get(0);  
213 - tblProductBO.setBrandId(tblBrand.getBrandId());  
214 - }  
215 - tblProductBO.setIsGlobal("N");  
216 - tblProductBO.setIsPlane("N");  
217 - tblProductBO.setStatus(4);  
218 - tblProductBO.setFormatCover("");  
219 - tblProductBO.setSkc(new ArrayList<>());  
220 - tblProductBO.setSizeList("");  
221 - tblProductBO.setCountryId(0);  
222 - tblProductBO.setShowStatus("Y");  
223 - tblProductBO.setGoodsPrice(new BigDecimal(tplProductBo.getOrignPrice()).setScale(2, BigDecimal.ROUND_DOWN));  
224 - tblProductBO.setOrignPrice(new BigDecimal(tplProductBo.getOrignPrice()).setScale(2, BigDecimal.ROUND_DOWN));  
225 - tblProductBO.setTagPrice(new BigDecimal(0.00).setScale(2, BigDecimal.ROUND_DOWN));  
226 - tblProductBOList.add(tblProductBO);  
227 - }  
228 - }  
229 -  
230 - return tblProductBOList;  
231 -  
232 - }  
233 -  
234 - private String buildShowStatus(TblProduct tblProduct, Map<Integer, TblSite> tblSiteMap, Map<Integer, TblBrand> tblBrandMap) {  
235 - TblBrand tblBrand = tblBrandMap.get(tblProduct.getBrandId());  
236 - TblSite tblSite = tblSiteMap.get(tblProduct.getSiteId());  
237 - String showStatus = "Y";  
238 - if (tblProduct.getSource() == null) {  
239 - return showStatus;  
240 - }  
241 - //爬虫商品检查品牌网站是否关闭  
242 - if (tblProduct.getSource().equals("1")) {  
243 - if (tblBrand == null || tblSite == null) {  
244 - showStatus = "N";  
245 - }  
246 - }  
247 - //采购商品查看 品牌是否关闭  
248 - if (tblProduct.getSource().equals("2")) {  
249 - if (tblBrand == null) {  
250 - showStatus = "N";  
251 - }  
252 - }  
253 - return showStatus;  
254 - }  
255 -  
256 - private Map<Integer, String> getColorIdsMap(List<TblProduct> tblProductList) {  
257 - Map<Integer, String> colorIdsMap = new HashMap<>();  
258 - List<Integer> productSknList = tblProductList.stream().map(TblProduct::getProductSkn).collect(Collectors.toList());  
259 - if (CollectionUtils.isNotEmpty(productSknList)) {  
260 - List<TblProductSkc> tblProductSkcList = tblProductSkcService.getBySkns(productSknList);  
261 - if (CollectionUtils.isNotEmpty(tblProductSkcList)) {  
262 - for (TblProductSkc tblProductSkc : tblProductSkcList) {  
263 - if (colorIdsMap.containsKey(tblProductSkc.getProductSkn())) {  
264 - String colorIds = colorIdsMap.get(tblProductSkc.getProductSkn());  
265 - colorIds += tblProductSkc.getColorSysId() + ",";  
266 - colorIdsMap.put(tblProductSkc.getProductSkn(), colorIds);  
267 - } else {  
268 - String colorIds;  
269 - colorIds = tblProductSkc.getColorSysId().toString();  
270 - colorIdsMap.put(tblProductSkc.getProductSkn(), colorIds);  
271 - }  
272 - }  
273 - }  
274 - }  
275 - return colorIdsMap;  
276 - }  
277 -  
278 - private Map<Integer, TblSite> getSiteMap() {  
279 - Map<Integer, TblSite> siteMap = new HashMap<>();  
280 - List<TblSite> tblSites = tblSiteService.selectPageList(0, Integer.MAX_VALUE);  
281 - if (CollectionUtils.isNotEmpty(tblSites)) {  
282 - siteMap = tblSites.stream().filter(p -> p.getStatus().equals("2")).collect(Collectors.toMap(TblSite::getSiteId, (p) -> p));  
283 - }  
284 - return siteMap;  
285 - }  
286 -  
287 - private Map<Integer, TblBrand> getTblBrandMap() {  
288 - Map<Integer, TblBrand> tblBrandMap = new HashMap<>();  
289 - List<TblBrand> tblBrands = tblBrandService.selectBrandPageList(0, Integer.MAX_VALUE);  
290 - if (CollectionUtils.isNotEmpty(tblBrands)) {  
291 - tblBrandMap = tblBrands.stream().filter(p -> p.getStatus().equals("2")).collect(Collectors.toMap(TblBrand::getBrandId, (p) -> p));  
292 - }  
293 - return tblBrandMap;  
294 - }  
295 -  
296 -  
297 - private List<Integer> getBrandIdList() {  
298 - List<Integer> brandIdList = new ArrayList<>();  
299 - List<TblBrand> tblBrands = tblBrandService.selectBrandPageList(0, Integer.MAX_VALUE);  
300 - if (CollectionUtils.isNotEmpty(tblBrands)) {  
301 - brandIdList = tblBrands.stream().filter(p -> p.getStatus().equals("2")).filter(p -> p.getYohoBrandId() != null).map(TblBrand::getYohoBrandId).collect(Collectors.toList());  
302 - }  
303 - return brandIdList;  
304 - }  
305 -  
306 - private Map<Integer, List<TblBrand>> getBrandMap() {  
307 - Map<Integer, List<TblBrand>> brandMap = new HashMap<>();  
308 - List<TblBrand> tblBrands = tblBrandService.selectBrandPageList(0, Integer.MAX_VALUE);  
309 - if (CollectionUtils.isNotEmpty(tblBrands)) {  
310 - for (TblBrand tblBrand : tblBrands) {  
311 - if (tblBrand.getYohoBrandId() == null || tblBrand.getYohoBrandId() == 0) {  
312 - continue;  
313 - }  
314 - if (brandMap.containsKey(tblBrand.getYohoBrandId())) {  
315 - List<TblBrand> tblBrandList = brandMap.get(tblBrand.getYohoBrandId());  
316 - tblBrandList.add(tblBrand);  
317 - } else {  
318 - List<TblBrand> tblBrandList = new ArrayList<>();  
319 - tblBrandList.add(tblBrand);  
320 - brandMap.put(tblBrand.getYohoBrandId(), tblBrandList);  
321 - }  
322 - }  
323 - }  
324 - return brandMap; 38 + return getPageList(offset, limit);
325 } 39 }
326 40
327 41
  1 +package com.yoho.search.consumer.index.fullbuild;
  2 +
  3 +import com.alibaba.fastjson.JSONArray;
  4 +import com.alibaba.fastjson.JSONObject;
  5 +import com.yoho.search.consumer.index.common.IIndexBuilder;
  6 +import com.yoho.search.consumer.index.fullbuild.tblProductNew.TblProductSkcMapBuilder;
  7 +import com.yoho.search.consumer.index.fullbuild.tblProductNew.TblProductSkuMapBuilder;
  8 +import com.yoho.search.consumer.service.daoService.TblBrandService;
  9 +import com.yoho.search.consumer.service.daoService.TblProductService;
  10 +import com.yoho.search.consumer.service.daoService.TblProductSkcService;
  11 +import com.yoho.search.consumer.service.daoService.TblSiteService;
  12 +import com.yoho.search.consumer.service.logicService.tbl.TblAdaptorLogicService;
  13 +import com.yoho.search.consumer.service.logicService.tbl.util.ImagesUtils;
  14 +import com.yoho.search.dal.model.*;
  15 +import org.apache.commons.collections.CollectionUtils;
  16 +import org.apache.commons.lang.StringUtils;
  17 +import org.springframework.beans.BeanUtils;
  18 +import org.springframework.beans.factory.annotation.Autowired;
  19 +import org.springframework.stereotype.Component;
  20 +
  21 +import java.math.BigDecimal;
  22 +import java.util.*;
  23 +import java.util.stream.Collectors;
  24 +
  25 +/**
  26 + * @Author: wangnan
  27 + * @Date: Created in 2018/4/20
  28 + */
  29 +@Component
  30 +public class TblProductIndexNewBuilder extends IIndexBuilder {
  31 +
  32 + @Autowired
  33 + private TblProductService tblProductService;
  34 + @Autowired
  35 + private TblProductSkcService tblProductSkcService;
  36 + @Autowired
  37 + private TblSiteService tblSiteService;
  38 + @Autowired
  39 + private TblBrandService tblBrandService;
  40 + @Autowired
  41 + private TblProductSkcMapBuilder tblProductSkcMapBuilder;
  42 + @Autowired
  43 + private TblProductSkuMapBuilder tblProductSkuMapBuilder;
  44 + @Autowired
  45 + private TblAdaptorLogicService tblAdaptorLogicService;
  46 +
  47 + @Override
  48 + public int getTotalCount() throws Exception {
  49 + return tblProductService.count();
  50 + }
  51 +
  52 + @Override
  53 + public List<?> getPageLists(int offset, int limit) throws Exception {
  54 + //查询tblProduct列表
  55 + List<TblProduct> tblProductList = tblProductService.getPageLists(offset, limit);
  56 + //构建tblSiteMap
  57 + Map<Integer, TblSite> tblSiteMap = getSiteMap();
  58 + //构建tblBrandMap
  59 + Map<Integer, TblBrand> tblBrandMap = getTblBrandMap();
  60 + //skc
  61 + Map<Integer, List<TblProductSkc>> tblProductSkcMap = tblProductSkcMapBuilder.build(tblProductList);
  62 + //sku
  63 + Map<Integer, List<TblProductSku>> tblProductSkuMap = tblProductSkuMapBuilder.build(tblProductList);
  64 + //构建ColorIds字段Map
  65 + Map<Integer, String> colorIdsMap = getColorIdsMap(tblProductList);
  66 +
  67 + //构建TblProductBO
  68 + List<TblProductBO> tblProductBOList = new ArrayList<>();
  69 + for (TblProduct tblProduct : tblProductList) {
  70 + //拼装与skc有关的字段，获取skn下面的skc列表
  71 + List<TblProductSkc> tblProductSkcList = tblProductSkcMap.get(tblProduct.getProductSkn());
  72 + //如果某个skn下不包含skc则跳过
  73 + if (CollectionUtils.isEmpty(tblProductSkcList)) {
  74 + continue;
  75 + }
  76 + String sizeList = "";
  77 + String url = "";
  78 + List<JSONObject> skc = new ArrayList<>();
  79 + //遍历skc列表
  80 + for (TblProductSkc tblProductSkc : tblProductSkcList) {
  81 + //构建cover和url
  82 + JSONArray jsonArray = JSONArray.parseArray(tblProductSkc.getPics());
  83 + String cover = "";
  84 + String imageUrl = "";
  85 + if (jsonArray != null) {
  86 + //默认的cover和url是第一个skc的
  87 + JSONObject pic1 = (JSONObject) jsonArray.get(0);
  88 + cover = pic1.get("src").toString();
  89 + //满足条件重新设置cover
  90 + for (int i = 0; i < jsonArray.size(); i++) {
  91 + JSONObject pic = (JSONObject) jsonArray.get(i);
  92 + if (pic.get("cover") != null && pic.get("cover").equals("1")) {
  93 + cover = pic.get("src").toString();
  94 + }
  95 + }
  96 + imageUrl = ImagesUtils.getImageUrl(cover, 112, 154);
  97 + //满足条件设置url
  98 + if ("1".equals(tblProductSkc.getIsDefault()) && !"1".equals(tblProductSkc.getIsDeleted())) {
  99 + imageUrl = ImagesUtils.getImageUrl(cover, 112, 154);
  100 + url = imageUrl.substring(0, imageUrl.indexOf("?"));
  101 + }
  102 + }
  103 + //拼装size_List
  104 + List<TblProductSku> tblProductSkuList = tblProductSkuMap.get(tblProduct.getProductSkn());
  105 + if (CollectionUtils.isNotEmpty(tblProductSkuList)) {
  106 + for (TblProductSku tblProductSku : tblProductSkuList) {
  107 + if (tblProductSku.getProductSkc().intValue() == tblProductSkc.getProductSkc().intValue()) {
  108 + if (!"1".equals(tblProductSku.getIsDeleted())) {
  109 + sizeList += tblProductSku.getSize() + ",";
  110 + }
  111 + }
  112 + }
  113 + }
  114 + //拼装skc列表
  115 + JSONObject skcObject = new JSONObject();
  116 + skcObject.put("product_skc", tblProductSkc.getProductSkc());
  117 + skcObject.put("color_sys_id", tblProductSkc.getColorSysId());
  118 + skcObject.put("color", tblProductSkc.getColor());
  119 + skcObject.put("cover", imageUrl.substring(0, imageUrl.indexOf("?")));
  120 + skcObject.put("format_cover", ImagesUtils.template2(imageUrl.substring(0, imageUrl.indexOf("?")), "global", "center", "d2hpdGU="));
  121 + skc.add(skcObject);
  122 + }
  123 +
  124 + //设置默认url
  125 + if (StringUtils.isBlank(url)) {
  126 + TblProductSkc tblProductSkc = tblProductSkcList.get(0);
  127 + JSONArray jsonArray = JSONArray.parseArray(tblProductSkc.getPics());
  128 + JSONObject pic1 = (JSONObject) jsonArray.get(0);
  129 + String cover = pic1.get("src").toString();
  130 + String imageUrl = ImagesUtils.getImageUrl(cover, 112, 154);
  131 + url = imageUrl.substring(0, imageUrl.indexOf("?"));
  132 + }
  133 +
  134 + //拼装TblProductBO数据
  135 + TblProductBO tblProductBO = new TblProductBO();
  136 + //简单字段:productSkn,productName,brandId,countryId,currencyId,siteId,gender,isLimited,sortOne,sortTwo,sortThree,sortFour,status,stockStatus,updateTime,shelfTime
  137 + BeanUtils.copyProperties(tblProduct, tblProductBO);
  138 + //colorIds
  139 + String colorIds = colorIdsMap.get(tblProduct.getProductSkn());
  140 + tblProductBO.setColorIds(colorIds);
  141 + //goodsPrice,tagPrice,orignPrice
  142 + if (tblProductBO.getGoodsPrice() != null) {
  143 + tblProductBO.setGoodsPrice(tblProductBO.getGoodsPrice().setScale(2, BigDecimal.ROUND_DOWN));
  144 + }
  145 + if (tblProductBO.getTagPrice() != null) {
  146 + tblProductBO.setTagPrice(tblProductBO.getTagPrice().setScale(2, BigDecimal.ROUND_DOWN));
  147 + }
  148 + if (tblProductBO.getOrignPrice() != null) {
  149 + tblProductBO.setOrignPrice(tblProductBO.getOrignPrice().setScale(2, BigDecimal.ROUND_DOWN));
  150 + }
  151 + //isPlane
  152 + if (tblProductBO.getCountryId() == 86) {
  153 + tblProductBO.setIsPlane("N");
  154 + }
  155 + //showStatus
  156 + String showStatus = this.buildShowStatus(tblProduct, tblSiteMap, tblBrandMap);
  157 + tblProductBO.setShowStatus(showStatus);
  158 + //cover
  159 + tblProductBO.setCover(url);
  160 + //formatCover
  161 + tblProductBO.setFormatCover(ImagesUtils.template2(url, null, "center", "d2hpdGU="));
  162 + //skc
  163 + tblProductBO.setSkc(skc);
  164 + //sizeLIst
  165 + tblProductBO.setSizeList(sizeList.substring(0, sizeList.length() - 1));
  166 +
  167 + //塞入列表
  168 + tblProductBOList.add(tblProductBO);
  169 + }
  170 +
  171 + //融合进有货的商品
  172 + if (offset == 0) {
  173 + List<Integer> brandIdList = this.getBrandIdList();
  174 + if (CollectionUtils.isEmpty(brandIdList)) {
  175 + return tblProductBOList;
  176 + }
  177 + Map<Integer, List<TblBrand>> brandMap = this.getBrandMap();
  178 + if (brandMap.isEmpty()) {
  179 + return tblProductBOList;
  180 + }
  181 + List<TplProductBo> tplProductBoList = tblAdaptorLogicService.convertYohoToTplProduct(brandIdList);
  182 + if (CollectionUtils.isEmpty(tplProductBoList)) {
  183 + return tblProductBOList;
  184 + }
  185 + for (TplProductBo tplProductBo : tplProductBoList) {
  186 + if (StringUtils.isBlank(tplProductBo.getOrignPrice()) || StringUtils.isBlank(tplProductBo.getGoodsPrice())) {
  187 + continue;
  188 + }
  189 + TblProductBO tblProductBO = new TblProductBO();
  190 + BeanUtils.copyProperties(tplProductBo, tblProductBO);
  191 + //brandId
  192 + List<TblBrand> tblBrandList = brandMap.get(tplProductBo.getBrandId());
  193 + if (CollectionUtils.isNotEmpty(tblBrandList)) {
  194 + TblBrand tblBrand = tblBrandList.get(0);
  195 + tblProductBO.setBrandId(tblBrand.getBrandId());
  196 + }
  197 + tblProductBO.setIsGlobal("N");
  198 + tblProductBO.setIsPlane("N");
  199 + tblProductBO.setStatus(4);
  200 + tblProductBO.setFormatCover("");
  201 + tblProductBO.setSkc(new ArrayList<>());
  202 + tblProductBO.setSizeList("");
  203 + tblProductBO.setCountryId(0);
  204 + tblProductBO.setShowStatus("Y");
  205 + tblProductBO.setGoodsPrice(new BigDecimal(tplProductBo.getOrignPrice()).setScale(2, BigDecimal.ROUND_DOWN));
  206 + tblProductBO.setOrignPrice(new BigDecimal(tplProductBo.getOrignPrice()).setScale(2, BigDecimal.ROUND_DOWN));
  207 + tblProductBO.setTagPrice(new BigDecimal(0.00).setScale(2, BigDecimal.ROUND_DOWN));
  208 + tblProductBOList.add(tblProductBO);
  209 + }
  210 + }
  211 +
  212 + return tblProductBOList;
  213 + }
  214 +
  215 + private String buildShowStatus(TblProduct tblProduct, Map<Integer, TblSite> tblSiteMap, Map<Integer, TblBrand> tblBrandMap) {
  216 + TblBrand tblBrand = tblBrandMap.get(tblProduct.getBrandId());
  217 + TblSite tblSite = tblSiteMap.get(tblProduct.getSiteId());
  218 + String showStatus = "Y";
  219 + if (tblProduct.getSource() == null) {
  220 + return showStatus;
  221 + }
  222 + //爬虫商品检查品牌网站是否关闭
  223 + if (tblProduct.getSource().equals("1")) {
  224 + if (tblBrand == null || tblSite == null) {
  225 + showStatus = "N";
  226 + }
  227 + }
  228 + //采购商品查看 品牌是否关闭
  229 + if (tblProduct.getSource().equals("2")) {
  230 + if (tblBrand == null) {
  231 + showStatus = "N";
  232 + }
  233 + }
  234 + return showStatus;
  235 + }
  236 +
  237 + private Map<Integer, String> getColorIdsMap(List<TblProduct> tblProductList) {
  238 + Map<Integer, String> colorIdsMap = new HashMap<>();
  239 + List<Integer> productSknList = tblProductList.stream().map(TblProduct::getProductSkn).collect(Collectors.toList());
  240 + if (CollectionUtils.isNotEmpty(productSknList)) {
  241 + List<TblProductSkc> tblProductSkcList = tblProductSkcService.getBySkns(productSknList);
  242 + if (CollectionUtils.isNotEmpty(tblProductSkcList)) {
  243 + for (TblProductSkc tblProductSkc : tblProductSkcList) {
  244 + if (colorIdsMap.containsKey(tblProductSkc.getProductSkn())) {
  245 + String colorIds = colorIdsMap.get(tblProductSkc.getProductSkn());
  246 + colorIds += tblProductSkc.getColorSysId() + ",";
  247 + colorIdsMap.put(tblProductSkc.getProductSkn(), colorIds);
  248 + } else {
  249 + String colorIds;
  250 + colorIds = tblProductSkc.getColorSysId().toString();
  251 + colorIdsMap.put(tblProductSkc.getProductSkn(), colorIds);
  252 + }
  253 + }
  254 + }
  255 + }
  256 + return colorIdsMap;
  257 + }
  258 +
  259 + private Map<Integer, TblSite> getSiteMap() {
  260 + Map<Integer, TblSite> siteMap = new HashMap<>();
  261 + List<TblSite> tblSites = tblSiteService.selectPageList(0, Integer.MAX_VALUE);
  262 + if (CollectionUtils.isNotEmpty(tblSites)) {
  263 + siteMap = tblSites.stream().filter(p -> p.getStatus().equals("2")).collect(Collectors.toMap(TblSite::getSiteId, (p) -> p));
  264 + }
  265 + return siteMap;
  266 + }
  267 +
  268 + private Map<Integer, TblBrand> getTblBrandMap() {
  269 + Map<Integer, TblBrand> tblBrandMap = new HashMap<>();
  270 + List<TblBrand> tblBrands = tblBrandService.selectBrandPageList(0, Integer.MAX_VALUE);
  271 + if (CollectionUtils.isNotEmpty(tblBrands)) {
  272 + tblBrandMap = tblBrands.stream().filter(p -> p.getStatus().equals("2")).collect(Collectors.toMap(TblBrand::getBrandId, (p) -> p));
  273 + }
  274 + return tblBrandMap;
  275 + }
  276 +
  277 +
  278 + private List<Integer> getBrandIdList() {
  279 + List<Integer> brandIdList = new ArrayList<>();
  280 + List<TblBrand> tblBrands = tblBrandService.selectBrandPageList(0, Integer.MAX_VALUE);
  281 + if (CollectionUtils.isNotEmpty(tblBrands)) {
  282 + brandIdList = tblBrands.stream().filter(p -> p.getStatus().equals("2")).filter(p -> p.getYohoBrandId() != null).map(TblBrand::getYohoBrandId).collect(Collectors.toList());
  283 + }
  284 + return brandIdList;
  285 + }
  286 +
  287 + private Map<Integer, List<TblBrand>> getBrandMap() {
  288 + Map<Integer, List<TblBrand>> brandMap = new HashMap<>();
  289 + List<TblBrand> tblBrands = tblBrandService.selectBrandPageList(0, Integer.MAX_VALUE);
  290 + if (CollectionUtils.isNotEmpty(tblBrands)) {
  291 + for (TblBrand tblBrand : tblBrands) {
  292 + if (tblBrand.getYohoBrandId() == null || tblBrand.getYohoBrandId() == 0) {
  293 + continue;
  294 + }
  295 + if (brandMap.containsKey(tblBrand.getYohoBrandId())) {
  296 + List<TblBrand> tblBrandList = brandMap.get(tblBrand.getYohoBrandId());
  297 + tblBrandList.add(tblBrand);
  298 + } else {
  299 + List<TblBrand> tblBrandList = new ArrayList<>();
  300 + tblBrandList.add(tblBrand);
  301 + brandMap.put(tblBrand.getYohoBrandId(), tblBrandList);
  302 + }
  303 + }
  304 + }
  305 + return brandMap;
  306 + }
  307 +
  308 + @Override
  309 + public String getId(Object object) {
  310 + return UUID.randomUUID().toString();
  311 + }
  312 +}
@@ -87,6 +87,7 @@ public class IndexRebuildJob implements ApplicationEventPublisherAware { @@ -87,6 +87,7 @@ public class IndexRebuildJob implements ApplicationEventPublisherAware {
87 public void rebuildTblProductIndex() { 87 public void rebuildTblProductIndex() {
88 long begin = System.currentTimeMillis(); 88 long begin = System.currentTimeMillis();
89 logger.info("indexRebuildJob rebuildTblProductIndex start----[begin={}]", begin); 89 logger.info("indexRebuildJob rebuildTblProductIndex start----[begin={}]", begin);
  90 + this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_TBLPRODUCT_NEW);
90 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_TBLPRODUCT); 91 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_TBLPRODUCT);
91 logger.info("indexRebuildJob rebuildTblProductIndex end----[end={}][cost={}]", System.currentTimeMillis(), (System.currentTimeMillis() - begin)); 92 logger.info("indexRebuildJob rebuildTblProductIndex end----[end={}][cost={}]", System.currentTimeMillis(), (System.currentTimeMillis() - begin));
92 } 93 }
  1 +index:
  2 + analysis:
  3 + filter:
  4 + dynamic_synonym:
  5 + type: dynamic-synonym
  6 + synonym_expand: true
  7 + synonym_ignore_case: true
  8 + synonym_analyzer: ik_smart
  9 + dynamic_synonym_rewrite:
  10 + type: dynamic-synonym-rewrite
  11 + synonym_expand: true
  12 + synonym_ignore_case: true
  13 + synonym_analyzer: ik_smart
  14 + analyzer:
  15 + comma_spliter:
  16 + type: pattern
  17 + pattern: "[,|\\s]+"
  18 + ik_complex:
  19 + type: custom
  20 + tokenizer: ik_smart
  21 + filter:
  22 + - lowercase
  23 + - dynamic_synonym
  24 + - trim
  25 + - unique
  26 + ik_complex_search:
  27 + type: custom
  28 + tokenizer: ik_smart
  29 + filter:
  30 + - lowercase
  31 + - dynamic_synonym_rewrite
  32 + - trim
  33 + - unique
  34 +
  35 +index.analysis.analyzer.default.type: keyword
  1 +{
  2 + "tblproductnew": {
  3 + "_all": {
  4 + "enabled": false
  5 + },
  6 + "_source": {
  7 + "enabled": true
  8 + },
  9 + "properties": {
  10 + "product_skn": {
  11 + "type": "integer"
  12 + },
  13 + "product_name": {
  14 + "type": "text",
  15 + "analyzer": "ik_complex",
  16 + "search_analyzer": "ik_complex_search"
  17 + },
  18 + "goods_price": {
  19 + "type": "double"
  20 + },
  21 + "brand_id": {
  22 + "type": "long"
  23 + },
  24 + "site_id": {
  25 + "type": "long"
  26 + },
  27 + "sort_one": {
  28 + "type": "long"
  29 + },
  30 + "sort_two": {
  31 + "type": "long"
  32 + },
  33 + "sort_three": {
  34 + "type": "long"
  35 + },
  36 + "sort_four": {
  37 + "type": "long"
  38 + },
  39 + "is_limited": {
  40 + "type": "long"
  41 + },
  42 + "country_id": {
  43 + "type": "long"
  44 + },
  45 + "gender": {
  46 + "type": "keyword"
  47 + },
  48 + "currency_id": {
  49 + "type": "long"
  50 + },
  51 + "tag_price": {
  52 + "type": "double"
  53 + },
  54 + "orign_price": {
  55 + "type": "double"
  56 + },
  57 + "status": {
  58 + "type": "long"
  59 + },
  60 + "stock_status": {
  61 + "type": "long"
  62 + },
  63 + "update_time": {
  64 + "type": "long"
  65 + },
  66 + "shelf_time": {
  67 + "type": "long"
  68 + },
  69 + "color_ids": {
  70 + "type": "text",
  71 + "analyzer": "comma_spliter",
  72 + "fielddata": "true"
  73 + },
  74 + "is_global": {
  75 + "type": "keyword"
  76 + },
  77 + "skc": {
  78 + "properties": {
  79 + "cover": {
  80 + "type": "text"
  81 + },
  82 + "color": {
  83 + "type": "text"
  84 + },
  85 + "product_skc": {
  86 + "type": "text"
  87 + },
  88 + "format_cover": {
  89 + "type": "text"
  90 + },
  91 + "color_sys_id": {
  92 + "type": "text"
  93 + }
  94 + }
  95 + },
  96 + "size_list": {
  97 + "type": "keyword"
  98 + },
  99 + "format_cover": {
  100 + "type": "text"
  101 + },
  102 + "cover": {
  103 + "type": "text"
  104 + },
  105 + "is_plane": {
  106 + "type": "text"
  107 + },
  108 + "show_status": {
  109 + "type": "text"
  110 + }
  111 + }
  112 + }
  113 +}
@@ -11,10 +11,6 @@ public class ConsumerConfiger { @@ -11,10 +11,6 @@ public class ConsumerConfiger {
11 private SearchPropertyPlaceholderConfigurer configurer; 11 private SearchPropertyPlaceholderConfigurer configurer;
12 12
13 /******************************for full build**************************************/ 13 /******************************for full build**************************************/
14 - public boolean getTblRebuildUseUrl() {  
15 - return configurer.getBoolean("tbl.rebuild.ues.url", false);  
16 - }  
17 -  
18 public int getSearchIndexBatchLimit(){ 14 public int getSearchIndexBatchLimit(){
19 return configurer.getInt("search.index.batch.limit", 5000); 15 return configurer.getInt("search.index.batch.limit", 5000);
20 } 16 }
@@ -18,7 +18,6 @@ search.index.batch.max.thread.size=10 @@ -18,7 +18,6 @@ search.index.batch.max.thread.size=10
18 tbl.product.private.key=a85bb0674e08986c6b115d5e3a4884fa 18 tbl.product.private.key=a85bb0674e08986c6b115d5e3a4884fa
19 tbl.product.url=http://test.soa.global.yohobuy.com/product/api/v1/detail/ 19 tbl.product.url=http://test.soa.global.yohobuy.com/product/api/v1/detail/
20 ip.port.search.server = "192.168.102.216:8080" 20 ip.port.search.server = "192.168.102.216:8080"
21 -tbl.rebuild.ues.url = false  
22 21
23 #adaptor 22 #adaptor
24 tpl.adaptor.valid=true 23 tpl.adaptor.valid=true
1 datasources: 1 datasources:
2 - yhb_search_2016: 2 + yhb_search:
3 servers: 3 servers:
4 - 192.168.102.216:3306 4 - 192.168.102.216:3306
5 - 192.168.102.216:3306 5 - 192.168.102.216:3306
@@ -127,6 +127,19 @@ @@ -127,6 +127,19 @@
127 </index> 127 </index>
128 128
129 <index> 129 <index>
  130 + <name>tblproductnew</name>
  131 + <properties>
  132 + <property key="number_of_shards" value="1"/>
  133 + <property key="number_of_replicas" value="1"/>
  134 + <property key="refresh_interval" value="10s"/>
  135 + <property key="translog.flush_threshold_size" value="100mb"/>
  136 + </properties>
  137 + <builderClass>com.yoho.search.consumer.index.fullbuild.TblProductIndexNewBuilder</builderClass>
  138 + <mappingFile>esmapping/tblproductnew.json</mappingFile>
  139 + <analysisFile>analysis/tblproductnew.yml</analysisFile>
  140 + </index>
  141 +
  142 + <index>
130 <name>standard</name> 143 <name>standard</name>
131 <properties> 144 <properties>
132 <property key="number_of_shards" value="1"/> 145 <property key="number_of_shards" value="1"/>
@@ -18,7 +18,6 @@ search.index.batch.max.thread.size=${search.index.batch.max.thread.size} @@ -18,7 +18,6 @@ search.index.batch.max.thread.size=${search.index.batch.max.thread.size}
18 tbl.product.private.key=a85bb0674e08986c6b115d5e3a4884fa 18 tbl.product.private.key=a85bb0674e08986c6b115d5e3a4884fa
19 tbl.product.url=${tbl.product.url} 19 tbl.product.url=${tbl.product.url}
20 ip.port.search.server = ${ip.port.search.server} 20 ip.port.search.server = ${ip.port.search.server}
21 -tbl.rebuild.ues.url = ${tbl.rebuild.ues.url}  
22 21
23 #adaptor 22 #adaptor
24 tpl.adaptor.valid=${tpl.adaptor.valid} 23 tpl.adaptor.valid=${tpl.adaptor.valid}
@@ -127,6 +127,19 @@ @@ -127,6 +127,19 @@
127 </index> 127 </index>
128 128
129 <index> 129 <index>
  130 + <name>tblproductnew</name>
  131 + <properties>
  132 + <property key="number_of_shards" value="1"/>
  133 + <property key="number_of_replicas" value="${search.index.number_of_replicas}"/>
  134 + <property key="refresh_interval" value="${search.index.refresh_interval}"/>
  135 + <property key="translog.flush_threshold_size" value="${search.index.translog.flush_threshold_size}"/>
  136 + </properties>
  137 + <builderClass>com.yoho.search.consumer.index.fullbuild.TblProductIndexNewBuilder</builderClass>
  138 + <mappingFile>esmapping/tblproductnew.json</mappingFile>
  139 + <analysisFile>analysis/tblproductnew.yml</analysisFile>
  140 + </index>
  141 +
  142 + <index>
130 <name>standard</name> 143 <name>standard</name>
131 <properties> 144 <properties>
132 <property key="number_of_shards" value="1"/> 145 <property key="number_of_shards" value="1"/>
1 datasources: 1 datasources:
2 - yhb_search_2016: 2 + yhb_search:
3 servers: 3 servers:
4 - 192.168.102.216:3306 4 - 192.168.102.216:3306
5 - 192.168.102.216:3306 5 - 192.168.102.216:3306