Merge branch 'master' into wn_promotion

Conflicts:
	service/src/main/java/com/yoho/search/recall/scene/beans/cache/BatchRecallCacheBean.java
	service/src/main/java/com/yoho/search/recall/scene/beans/cache/SknBaseInfoCacheBean.java
	service/src/main/java/com/yoho/search/recall/scene/beans/persional/AbstractPageComponent.java
	service/src/main/java/com/yoho/search/recall/scene/beans/persional/PageProductIdBitSetComponent.java
	service/src/main/java/com/yoho/search/recall/scene/models/personal/PageProductIdBitSet.java
	service/src/main/java/com/yoho/search/recall/scene/models/req/SknBaseInfoRequest.java
	service/src/main/java/com/yoho/search/recall/scene/models/req/SknBaseInfoResponse.java

@@ -25,8 +25,8 @@ public class CommonRecallRequestBuilder{
         List<RecallRequest> requests = new ArrayList<>();
         //1.1) firstSkn的召回
         requests.add(this.buildFirstSknRequest(paramQueryFilter, firstProductSkns, SknCountConstants.FIRST_SKN));
-        //1.2) 直通车的召回
-        requests.add(this.buildDirectTrainRequest(paramQueryFilter, SknCountConstants.DIRECT_TRAIN));
+        //1.2) 直通车召回
+        requests.add(this.buildDirectTrainRequest(paramQueryFilter, SknCountConstants.DIRECT_TRAIN_RECALL_COUNT));
         //1.3) 人气的召回
         requests.add(this.buildCommonRequest(paramQueryFilter,SknCountConstants.COMMON_HEAT_VALUE));
         //1.4) 新开店铺的召回

@@ -86,6 +86,7 @@ public class RecallMergerResultBuilder {
             List<StrategyEnum> strategyEnumList = skn2StrategyListMap.getOrDefault(sknResult.getProductSkn(), new ArrayList<>());
             Collections.sort(strategyEnumList, (o1, o2) -> o2.getPriority().compareTo(o1.getPriority()));
             sknResult.setStrategy(strategyEnumList.get(0));//取优先级最高的召回策略
+            sknResult.setOnlyOneStrategy(strategyEnumList.size()==1?true:false);
         }
         return sknResults;
     }

@@ -168,7 +168,7 @@ public class UserRecallResponseBuilder {
         PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.getPersonalizedSearch(paramMap);
         UserFeatureFactor userFeatureFactor = new UserFeatureFactor(personalizedSearch);
         //2、计算相关性
-        int recommendSknIndex = 1000;
+        int recommendSknIndex = 10000;
         for (RecallMergerResult.SknResult sknResult : sknResultList) {
             double score = 0d;
             //1)策略判断
@@ -184,13 +184,20 @@ public class UserRecallResponseBuilder {
             }
             //3)向量计算
             score = productFeatureFactorHepler.calProductFeatureFactor(userFeatureFactor, sknResult.getFactor());
-            //4）如果满足品类价格带偏好，则加分
+
+            //4)如果是直通车商品，则拿人气和向量综合评分
+            if (strategy.equals(StrategyEnum.DIRECT_TRAIN)) {
+                score = score * sknResult.getHeatValue();
+            }
+
+            //5）如果满足品类价格带偏好，则加分
             if (sknResult.isLikePriceArea()) {
-                score = score + 100;
+                score = score + 1000;
             }
-            //5）如果兜底策略不参与评分，并且当前skn不是兜底召回的，则加分【将兜底和非兜底的拆分】
+
+            //6）如果兜底策略不参与评分，并且当前skn不是兜底召回的，则加分【将兜底和非兜底的拆分】
             if (!searchDynamicConfigService.searchPersionalNewStrategyCommonJoinScoreOpen() && !strategy.equals(StrategyEnum.COMMON)) {
-                score = score + 50;
+                score = score + 500;
             }
             sknResult.setScore(score);
         }
@@ -250,16 +257,26 @@ public class UserRecallResponseBuilder {
         }
 
         // 3、插入【直接推荐】的商品-随机插入
-        this.addByIndexIndex(sknResultList, results, 1, 2, (sknResult -> StrategyEnum.RECOMMEND_SKN.equals(sknResult.getStrategy())));
+        this.addByIndexIndex(sknResultList, results, 1, 2, (sknResult -> StrategyEnum.RECOMMEND_SKN.equals(sknResult.getStrategy())),dropTransfer);
 
-        // 4、插入【直通车】商品
+        // 4、插入【直通车】商品-随机插入
         int directTrainIndexInterval = searchDynamicConfigService.directTrainIndexInterval();
-        this.addByIndexIndex(sknResultList, results, 4, directTrainIndexInterval, (sknResult -> StrategyEnum.DIRECT_TRAIN.equals(sknResult.getStrategy())));
+        this.addByIndexIndex(sknResultList, results, 4, directTrainIndexInterval, (sknResult -> StrategyEnum.DIRECT_TRAIN.equals(sknResult.getStrategy())),dropTransfer);
 
         return results;
     }
 
-    private static <T>  void addByIndexIndex(List<T> fromList, List<T> toList,int fromIndex, int indexInterval,Transfer<T, Boolean> match) {
+    /**
+     * 单策略召回时，超出数量直接丢弃
+     */
+    private static Transfer<RecallMergerResult.SknResult,Boolean> dropTransfer = new Transfer<RecallMergerResult.SknResult, Boolean>() {
+        @Override
+        public Boolean transfer(RecallMergerResult.SknResult sknResult) {
+            return sknResult.isOnlyOneStrategy()?true:false;
+        }
+    };
+
+    private static <T>  void addByIndexIndex(List<T> fromList, List<T> toList,int fromIndex, int indexInterval,Transfer<T, Boolean> match,Transfer<T, Boolean> drop) {
         Iterator<T> iterator = fromList.iterator();
         while (iterator.hasNext()) {
             T object = iterator.next();
@@ -268,15 +285,14 @@ public class UserRecallResponseBuilder {
             }
             // 生成插入位置
             int toListNewSize = toList.size();
-            System.out.println(fromIndex);
             // 尽量不影响第1个
             if (fromIndex == 0 && !toList.isEmpty()) {
                 fromIndex = 1;
             }
-            //超出新列表的长度，则加到最后面
+            //超出新列表的长度，判断是否直接丢弃
             if (fromIndex <= toListNewSize) {
                 toList.add(fromIndex, object);
-            }else{
+            }else if(drop !=null && !drop.transfer(object)){
                 toList.add(object);
             }
             iterator.remove();
@@ -296,10 +312,10 @@ public class UserRecallResponseBuilder {
         }
 
         List<Integer> toList = new ArrayList<>();
-        for (int index =0;index <100;index ++){
+        for (int index =0;index <1;index ++){
             toList.add(0);
         }
-        addByIndexIndex(fromList, toList, 1, 2, (value -> value%2==1));
+        addByIndexIndex(fromList, toList, 1, 2, (value -> value%2==1),null);
         System.out.println(toList);
 //        addByIndexIndex(fromList, toList, 4, 4, (value -> value%2==0));
 //        System.out.println(toList);

@@ -38,7 +38,7 @@ public class BatchRecallCacheBean extends AbstractCacheBean<RecallRequest,Recall
             results.add(new RecallRequestResponse(request));
         }
         //2、执行查询
-        this.bacthFillResponseWithCache(results,10);
+        this.bacthFillResponseWithCache(results,15);
         //3、返回结果
         return results;
     }

@@ -123,8 +123,9 @@ public class SknBaseInfoCacheBean extends AbstractCacheBean<SknBaseInfoRequest,
             Integer middleSortId = MapUtils.getInteger(productInfo, ProductIndexEsField.middleSortId, 0);
             Integer priceArea = MapUtils.getInteger(productInfo, ProductIndexEsField.priceArea, 0);
             String productFeatureFactor = MapUtils.getString(productInfo, ProductIndexEsField.productFeatureFactor, "");
+            Integer heatValue = MapUtils.getInteger(productInfo, ProductIndexEsField.heatValue, 0);
 
-            SknBaseInfoResponse sknBaseInfo = new SknBaseInfoResponse(productId, productSkn, brandId, middleSortId, priceArea, productFeatureFactor);
+            SknBaseInfoResponse sknBaseInfo = new SknBaseInfoResponse(productId, productSkn, brandId, middleSortId, priceArea, productFeatureFactor,heatValue);
             productTempMap.put(productSkn, sknBaseInfo);
         }
         //5、构造最终结果-有可能是null的

@@ -54,8 +54,6 @@ public class SknRecallCacheBean {
         try {
             if (recommedSknList == null || recommedSknList.isEmpty()) {
                 return new ArrayList<>();
-                //localhost:8080/yohosearch/productindex/productList.json?uid=11145&status=1&gender=1,2,3,4,5
-                //recommedSknList = Arrays.asList(50020667,50026235,50026280,50024584,50026611,50028826,50031356,50031387,50029959,51001108);
             }
             List<Integer> filterSknList;
             if (recallWithCache) {
@@ -63,15 +61,17 @@ public class SknRecallCacheBean {
             } else {
                 filterSknList = this.filterRecommedSknListByEs(userRecallRequest, recommedSknList);
             }
+            RECALL_NEW_LOGGER.info("recommedSknList size is [{}], filterSknListSize is[{}]", recommedSknList.size(), filterSknList.size());
             return this.buildResults(userRecallRequest, filterSknList, maxReturnCount);
-        }catch (Exception e){
-            RECALL_NEW_LOGGER.error(e.getMessage(),e);
+        } catch (Exception e) {
+            RECALL_NEW_LOGGER.error(e.getMessage(), e);
             return new ArrayList<>();
         }
     }
 
     /**
      * 从ehcahc或者redis中构造返回结果
+     *
      * @param userRecallRequest
      * @param recommedSknList
      * @return
@@ -79,7 +79,7 @@ public class SknRecallCacheBean {
     private List<Integer> filterRecommendWithCache(UserRecallRequest userRecallRequest, List<Integer> recommedSknList) {
         //1、skn转成productId
         Map<Integer, Integer> productSknToIdMap = sknBaseInfoCacheBean.queryProductSknToProductIdMap(recommedSknList);
-        if(productSknToIdMap==null || productSknToIdMap.isEmpty()){
+        if (productSknToIdMap == null || productSknToIdMap.isEmpty()) {
             return new ArrayList<>();
         }
         List<Integer> recommedProductIds = new ArrayList<>();
@@ -88,29 +88,26 @@ public class SknRecallCacheBean {
         //2、先从本地缓存获取
         ParamQueryFilter paramQueryFilter = userRecallRequest.getParamQueryFilter();
         RedisKeyBuilder redisKeyBuilder = pageProductIdBitSetComponent.genRedisKeyBuilder(paramQueryFilter);
-        Map<Integer, Boolean> ehcacheQueryResult = ehCache.getFromBitSet(redisKeyBuilder,recommedProductIds);
+        Map<Integer, Boolean> ehcacheQueryResult = ehCache.getFromBitSet(redisKeyBuilder, recommedProductIds);
         if (ehcacheQueryResult != null) {
-            return this.filterSknByCacheQueryResult(recommedSknList,productSknToIdMap, ehcacheQueryResult);
+            RECALL_NEW_LOGGER.info("filterSknBy EhCache  success");
+            return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap, ehcacheQueryResult);
         }
-
         //3、再从redis的bitset缓存结构中过滤
         Map<Integer, Boolean> redisQueryResult = searchRedis.getFromBitSet(redisKeyBuilder, recommedProductIds);
         if (redisQueryResult != null) {
-            //TODO 回写本地缓存
-            return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap,redisQueryResult);
+            RECALL_NEW_LOGGER.info("filterSknBy Redis success");
+            return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap, redisQueryResult);
         }
-
         //4、执行查询，添加到缓存
         PageProductIdBitSet pageProductIdBitSet = pageProductIdBitSetComponent.queryPageProductIdBitSet(userRecallRequest.getParamQueryFilter());
-        //3.1)添加到本地缓存
-        this.ehCache.setBitSet(redisKeyBuilder,pageProductIdBitSet.getProductIdBitSet(),pageProductIdBitSetComponent.cacheTimeInMinute());
-        //3.2)添加到redis缓存
-        //TODO 异步加入redis缓存
-        this.searchRedis.setBitSet(redisKeyBuilder,pageProductIdBitSet.getProductIdBitSet(),pageProductIdBitSetComponent.cacheTimeInMinute());
-        return this.filterSknByPageProductIdBitSet(recommedSknList,productSknToIdMap, pageProductIdBitSet.getProductIdBitSet());
+        //4.1)添加到本地缓存
+        this.ehCache.setBitSet(redisKeyBuilder, pageProductIdBitSet.getProductIdBitSet(), pageProductIdBitSetComponent.cacheTimeInMinute());
+        //4.2)异步添加到redis缓存
+        this.searchRedis.setBitSet(redisKeyBuilder, pageProductIdBitSet.getProductIdBitSet(), pageProductIdBitSetComponent.cacheTimeInMinute());
+        return this.filterSknByPageProductIdBitSet(recommedSknList, productSknToIdMap, pageProductIdBitSet.getProductIdBitSet());
     }
 
-
     /**
      * 从bitset对象中构造返回对象
      *
@@ -118,11 +115,11 @@ public class SknRecallCacheBean {
      * @param bitSet
      * @return
      */
-    private List<Integer> filterSknByPageProductIdBitSet(List<Integer> recommedSknList,Map<Integer, Integer> productSknToIdMap, BitSet bitSet) {
+    private List<Integer> filterSknByPageProductIdBitSet(List<Integer> recommedSknList, Map<Integer, Integer> productSknToIdMap, BitSet bitSet) {
         List<Integer> existProductSkns = new ArrayList<>();
         for (Integer recommedSkn : recommedSknList) {
             Integer productId = productSknToIdMap.get(recommedSkn);
-            if(productId==null){
+            if (productId == null) {
                 continue;
             }
             if (!bitSet.get(productId)) {
@@ -140,11 +137,11 @@ public class SknRecallCacheBean {
      * @param cacheQueryResult
      * @return
      */
-    private List<Integer> filterSknByCacheQueryResult(List<Integer> recommedSknList,Map<Integer, Integer> productSknToIdMap, Map<Integer, Boolean> cacheQueryResult) {
+    private List<Integer> filterSknByCacheQueryResult(List<Integer> recommedSknList, Map<Integer, Integer> productSknToIdMap, Map<Integer, Boolean> cacheQueryResult) {
         List<Integer> existProductSkns = new ArrayList<>();
         for (Integer recommedSkn : recommedSknList) {
             Integer productId = productSknToIdMap.get(recommedSkn);
-            if(productId==null){
+            if (productId == null) {
                 continue;
             }
             if (!cacheQueryResult.getOrDefault(productId, false)) {

@@ -28,7 +28,7 @@ public abstract class AbstractPageComponent<T> {
     private ConcurrentHashMap<String, Integer> mapLock;
     private ExecutorService executorService;
 
-    private boolean synchronous = true;//是否同步
+    private boolean synchronous = false;//是否同步
 
     @PostConstruct
     void init() {

@@ -59,14 +59,15 @@ public class PageProductIdBitSetComponent extends AbstractPageComponent<PageProd
 
         BoolQueryBuilder filter = QueryBuilders.boolQuery();
         filter.must(paramQueryFilter.getParamFilter());
-        filter.mustNot(ExtendFilterHelper.notRecallFilter());//聚合的时候带上不召回的数据
+        //filter.mustNot(QueryBuilders.termQuery(ProductIndexEsField.isGlobal, "Y"));
+        //filter.mustNot(QueryBuilders.rangeQuery(ProductIndexEsField.breakSizePercent).gt(50));
         searchParam.setFiter(filter);
 
         searchParam.setSize(0);
 
         //2、构造聚合参数
         List<AbstractAggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
-        aggregationBuilders.add(AggregationBuilders.terms("productIdAgg").field(ProductIndexEsField.productId).size(100000).order(Terms.Order.term(false)));//品类-品牌聚合
+        aggregationBuilders.add(AggregationBuilders.terms("productIdAgg").field(ProductIndexEsField.productId).size(100000).order(Terms.Order.term(false)));
         searchParam.setAggregationBuilders(aggregationBuilders);
 
         //3、执行查询

@@ -8,7 +8,9 @@ public class SknCountConstants {
     public static final int COMMON_HEAT_VALUE = 60;
 
     public static final int FIRST_SKN = 1;
-    public static final int DIRECT_TRAIN = 20;
+    public static final int DIRECT_TRAIN_RECALL_COUNT = 100;
+    public static final int DIRECT_TRAIN_RETURN_COUNT = 20;
+
     public static final int NEW_SHOP = 10;
     public static final int ADD_FLOW = 10;
 

@@ -35,10 +35,12 @@ public class RecallMergerResult {
         private Integer brandId;
         private Integer middleSortId;
         private Integer priceArea;
+        private Integer heatValue;
         private String factor;
 
         private Double score;
         private boolean likePriceArea;
+        private boolean onlyOneStrategy;
 
         public SknResult(Integer productSkn) {
             this.productSkn = productSkn;
@@ -50,6 +52,7 @@ public class RecallMergerResult {
             this.brandId = sknBaseInfoResponse.getBrandId();
             this.middleSortId = sknBaseInfoResponse.getMisortId();
             this.priceArea = sknBaseInfoResponse.getPriceArea();
+            this.heatValue = sknBaseInfoResponse.getHeatValue();
             this.factor = sknBaseInfoResponse.getProductFeatureFactor();
         }
 
@@ -57,6 +60,10 @@ public class RecallMergerResult {
             this.strategy = strategy;
         }
 
+        public void setOnlyOneStrategy(boolean onlyOneStrategy) {
+            this.onlyOneStrategy = onlyOneStrategy;
+        }
+
         public void setScore(Double score) {
             this.score = score;
         }
@@ -98,6 +105,13 @@ public class RecallMergerResult {
             return likePriceArea;
         }
 
+        public Integer getHeatValue() {
+            return heatValue;
+        }
+
+        public boolean isOnlyOneStrategy() {
+            return onlyOneStrategy;
+        }
     }
 
 }

@@ -21,19 +21,8 @@ public class PageProductIdBitSet implements Serializable{
         this.productIdBitSet.set(bitSetIndex);
     }
 
-    public boolean exist(int bitSetIndex) {
-        return productIdBitSet.get(bitSetIndex);
-    }
-
-    public int getCardinality(){
-        return productIdBitSet.cardinality();
-    }
-
     public BitSet getProductIdBitSet() {
         return productIdBitSet;
     }
 
-    public void setProductIdBitSet(BitSet productIdBitSet) {
-        this.productIdBitSet = productIdBitSet;
-    }
 }

@@ -5,11 +5,14 @@ import com.yoho.search.base.utils.ProductIndexEsField;
 import com.yoho.search.recall.scene.constants.CacheTimeConstants;
 import com.yoho.search.recall.scene.models.common.ICacheRequest;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
 public class SknBaseInfoRequest implements ICacheRequest{
 
+    private static final List<String> includeFields = Arrays.asList(ProductIndexEsField.productId,ProductIndexEsField.productSkn,ProductIndexEsField.brandId,ProductIndexEsField.middleSortId,ProductIndexEsField.priceArea,ProductIndexEsField.productFeatureFactor,ProductIndexEsField.heatValue);
+
     private Integer productSkn;
 
     public SknBaseInfoRequest(Integer productSkn){
@@ -30,12 +33,8 @@ public class SknBaseInfoRequest implements ICacheRequest{
         return productSkn;
     }
 
-    /**
-     * 请求返回的字段
-     * @return
-     */
     public List<String> includeFields(){
-        return Arrays.asList(ProductIndexEsField.productId,ProductIndexEsField.productSkn,ProductIndexEsField.brandId,ProductIndexEsField.middleSortId,ProductIndexEsField.priceArea,ProductIndexEsField.productFeatureFactor);
+        return includeFields;
     }
 
 }

@@ -12,18 +12,20 @@ public class SknBaseInfoResponse implements Serializable{
     private Integer misortId;
     private Integer priceArea;
     private String productFeatureFactor;
+    private Integer heatValue;
 
     public SknBaseInfoResponse() {
 
     }
 
-    public SknBaseInfoResponse(Integer productId, Integer productSkn, Integer brandId, Integer misortId, Integer priceArea, String productFeatureFactor) {
+    public SknBaseInfoResponse(Integer productId, Integer productSkn, Integer brandId, Integer misortId, Integer priceArea, String productFeatureFactor,Integer heatValue) {
         this.productId = productId;
         this.productSkn = productSkn;
         this.brandId = brandId;
         this.misortId = misortId;
         this.priceArea = priceArea;
         this.productFeatureFactor = productFeatureFactor;
+        this.heatValue = heatValue;
     }
 
     public Integer getProductSkn() {
@@ -73,4 +75,12 @@ public class SknBaseInfoResponse implements Serializable{
     public void setProductFeatureFactor(String productFeatureFactor) {
         this.productFeatureFactor = productFeatureFactor;
     }
+
+    public Integer getHeatValue() {
+        return heatValue;
+    }
+
+    public void setHeatValue(Integer heatValue) {
+        this.heatValue = heatValue;
+    }
 }