列表中的页面元素聚合使用异步的方式

 package com.yoho.search.common.cache;
 
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
 import com.yoho.search.common.cache.aop.SearchCacheAble;
 import com.yoho.search.common.cache.impls.CacheInterface;
 import com.yoho.search.common.cache.impls.EhCache;
 import com.yoho.search.common.cache.impls.SearchRedis;
 import com.yoho.search.common.cache.impls.YohoRedis;
 import com.yoho.search.common.cache.model.SearchCache;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 @Service
 public class SearchCacheFactory {
@@ -56,17 +55,6 @@ public class SearchCacheFactory {
 	}
 
 	/**
-	 * 获取聚合相关的缓存
-	 *
-	 * @return
-	 */
-	public SearchCache getPagePersionalFactorCache() {
-		CacheType cacheType = CacheType.EHCACHE;
-		int cacheInMinute = 60;
-		return this.getOrCreateSearchCache("PAGE_PERSIONAL_FACTOR", cacheType, cacheInMinute);
-	}
-
-	/**
 	 * 获取默认的搜索缓存
 	 * 
 	 * @return

+package com.yoho.search.recall.scene.beans.persional;
+
+import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
+import com.yoho.search.common.cache.impls.EhCache;
+import com.yoho.search.common.cache.model.CacheObject;
+import com.yoho.search.recall.scene.models.common.ParamQueryFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.PostConstruct;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public abstract class AbstractPageComponent {
+
+    private static final Logger logger = LoggerFactory.getLogger(AbstractPageComponent.class);
+
+    @Autowired
+    private EhCache ehCache;
+
+    private ConcurrentHashMap<String, Integer> mapLock;
+    private ExecutorService executorService;
+
+    @PostConstruct
+    void init() {
+        mapLock = new ConcurrentHashMap(20);//使用一个Map来限流
+        executorService = Executors.newFixedThreadPool(5);
+    }
+
+    public Object queryWithCache(ParamQueryFilter paramQueryFilter) {
+        //1、生成RedisKeyBuilder
+        RedisKeyBuilder redisKeyBuilder = this.genRedisKeyBuilder(paramQueryFilter);
+        if (redisKeyBuilder == null) {
+            return null;
+        }
+        //2、缓存命中，则直接返回
+        CacheObject cacheObject = ehCache.get(redisKeyBuilder);
+        if (cacheObject != null) {
+            return cacheObject;
+        }
+        //3、限流判断以及重复请求预防
+        if (mapLock.size() > 20 || mapLock.putIfAbsent(redisKeyBuilder.getKey(), 1) != null) {
+            return null;
+        }
+        //4、异步执行查询并加入缓存
+        executorService.submit(() -> {
+            try {
+                Object queryResult = doRealQuery(paramQueryFilter);
+                CacheObject toCacheResult = new CacheObject(queryResult);
+                ehCache.addOrUpdate(redisKeyBuilder, toCacheResult, this.cacheTimeInSecond() / 60);
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
+            } finally {
+                mapLock.remove(redisKeyBuilder.getKey());
+            }
+        });
+        return null;
+    }
+
+    protected abstract RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter);
+
+    protected abstract int cacheTimeInSecond();
+
+    protected abstract Object doRealQuery(ParamQueryFilter paramQueryFilter);
+
+}

@@ -3,15 +3,12 @@ package com.yoho.search.recall.scene.beans.persional;
 import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
 import com.yoho.search.base.utils.ISearchConstants;
 import com.yoho.search.base.utils.ProductIndexEsField;
-import com.yoho.search.common.cache.SearchCacheFactory;
-import com.yoho.search.common.cache.model.SearchCache;
 import com.yoho.search.core.es.model.SearchParam;
 import com.yoho.search.core.es.model.SearchResult;
+import com.yoho.search.recall.scene.constants.CacheTimeConstants;
 import com.yoho.search.recall.scene.models.common.ParamQueryFilter;
 import com.yoho.search.recall.scene.models.personal.PagePersonalFactor;
 import com.yoho.search.recall.scene.models.personal.PageBrandSorts;
-import com.yoho.search.recall.scene.models.personal.PageSortPriceAreas;
-import com.yoho.search.service.base.SearchCacheService;
 import com.yoho.search.service.base.SearchCommonService;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -21,33 +18,36 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import javax.annotation.PostConstruct;
 import java.util.*;
 
 @Component
-public class PagePersionalFactorComponent {
+public class PagePersionalFactorComponent extends AbstractPageComponent {
 
     @Autowired
     private SearchCommonService searchCommonService;
-    @Autowired
-    private SearchCacheService searchCacheService;
-    @Autowired
-    private SearchCacheFactory searchCacheFactory;
-
-    private SearchCache searchCache;
-
-    @PostConstruct
-    void init(){
-        searchCache = searchCacheFactory.getPagePersionalFactorCache();
-    }
 
     /**
-     * 获取链接中的个性化因子-品牌+品类
-     *
+     * 查询个性化因子
      * @param paramQueryFilter
      * @return
      */
     public PagePersonalFactor queryPagePersionalFactor(ParamQueryFilter paramQueryFilter) {
+        Object value = super.queryWithCache(paramQueryFilter);
+        return value==null?null:(PagePersonalFactor)value;
+    }
+
+    @Override
+    protected RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
+        return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("PAGE_FACTOR:").appendVar(paramQueryFilter.getParamMd5Key());
+    }
+
+    @Override
+    protected int cacheTimeInSecond() {
+        return CacheTimeConstants.PAGE_PERSIONAL_FACTOR;
+    }
+
+    @Override
+    protected Object doRealQuery(ParamQueryFilter paramQueryFilter) {
         //1、构造参数
         SearchParam searchParam = new SearchParam();
         searchParam.setQuery(paramQueryFilter.getParamQuery());
@@ -59,28 +59,17 @@ public class PagePersionalFactorComponent {
         aggregationBuilders.add(brandSortAggBuilder());//品类-品牌聚合
         searchParam.setAggregationBuilders(aggregationBuilders);
 
-        //3、缓存中获取
-        RedisKeyBuilder redisKeyBuilder = searchCacheService.genSearchParamString(ISearchConstants.INDEX_NAME_PRODUCT_INDEX,searchParam);
-        PagePersonalFactor pagePagePersonalFactor = searchCacheService.getSerializableObjectFromCache(searchCache,redisKeyBuilder,PagePersonalFactor.class,true);
-        if(pagePagePersonalFactor !=null) {
-            return pagePagePersonalFactor;
-        }
-
-        //4、执行查询
+        //3、执行查询
         SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
 
-        //5、构造结果
+        //4、构造结果
         Map<String, Aggregation> aggregationMap = searchResult.getAggMaps();
         List<PageBrandSorts> sortBrands = this.getBrandSortsFromAggregationMap(aggregationMap);
-
-        pagePagePersonalFactor = new PagePersonalFactor(sortBrands);
-        //6、加入缓存
-        searchCacheService.addSerializableObjectToCache(searchCache,redisKeyBuilder, pagePagePersonalFactor,true);
-        return pagePagePersonalFactor;
+        return new PagePersonalFactor(sortBrands);
     }
 
     /**
-     * 品类-品牌聚合
+     * 品类+品牌聚合
      *
      * @return
      */

 package com.yoho.search.recall.scene.beans.persional;
 
-import com.google.common.hash.BloomFilter;
-import com.google.common.hash.Funnels;
 import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
-import com.yoho.search.common.cache.impls.EhCache;
-import com.yoho.search.common.cache.model.CacheObject;
+import com.yoho.search.base.utils.ISearchConstants;
+import com.yoho.search.base.utils.ProductIndexEsField;
+import com.yoho.search.core.es.model.SearchParam;
+import com.yoho.search.core.es.model.SearchResult;
+import com.yoho.search.recall.scene.constants.CacheTimeConstants;
 import com.yoho.search.recall.scene.models.common.ParamQueryFilter;
 import com.yoho.search.recall.scene.models.personal.PageSknBitSet;
-import org.apache.lucene.util.RamUsageEstimator;
+import com.yoho.search.service.base.SearchCommonService;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 @Component
-public class PageSknBitSetComponent {
+public class PageSknBitSetComponent extends AbstractPageComponent {
 
     @Autowired
-    private EhCache ehCache;
+    private SearchCommonService searchCommonService;
 
     /**
-     * 获取页面上的skn列表
+     * 获取页面上的skn的bitset
      *
      * @param paramQueryFilter
      * @return
      */
     public PageSknBitSet queryPageSknBitSet(ParamQueryFilter paramQueryFilter) {
-        RedisKeyBuilder redisKeyBuilder = RedisKeyBuilder.newInstance();
-        redisKeyBuilder.appendFixed("YOHOSEARCH:").appendFixed("PAGESKN").appendVar(paramQueryFilter.getParamMd5Key());
-        CacheObject cacheObject = ehCache.get(redisKeyBuilder);
-        if(cacheObject!=null){
-           return  (PageSknBitSet)cacheObject.toObject();
+       Object value = super.queryWithCache(paramQueryFilter);
+       return value==null?null:(PageSknBitSet)value;
     }
-        PageSknBitSet pageSknBitSet = new PageSknBitSet();
-        cacheObject = new CacheObject(pageSknBitSet);
-        ehCache.addOrUpdate(redisKeyBuilder,cacheObject,30);
+
+    @Override
+    protected RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
+        return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:PAGE_SKN_BITSET:").appendVar(paramQueryFilter.getParamMd5Key());
+    }
+
+    @Override
+    protected int cacheTimeInSecond() {
+        return CacheTimeConstants.PAGE_SKN_BITSET;
+    }
+
+    @Override
+    protected Object doRealQuery(ParamQueryFilter paramQueryFilter) {
+        //1、构造请求参数
+        SearchParam searchParam = new SearchParam();
+        searchParam.setQuery(paramQueryFilter.getParamQuery());
+        searchParam.setFiter(paramQueryFilter.getParamFilter());
+        searchParam.setSize(0);
+
+        //2、构造聚合参数
+        List<AbstractAggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
+        aggregationBuilders.add(AggregationBuilders.terms("productIdAgg").field(ProductIndexEsField.productId).size(10000).order(Terms.Order.term(false)));//品类-品牌聚合
+        searchParam.setAggregationBuilders(aggregationBuilders);
+
+        //3、执行查询
+        SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
+
+        //4、构造结果
+        Map<String, Aggregation> aggregationMap = searchResult.getAggMaps();
+        PageSknBitSet pageSknBitSet = this.getPageSknBitSetFromAggregationMap(aggregationMap,"productIdAgg");
         return pageSknBitSet;
     }
 
-    public static void main(String[] args) {
-        Charset charset = Charset.forName("utf-8");
-        BloomFilter<String> bloomFilter = BloomFilter.create(Funnels.stringFunnel(charset),2<<21);//指定bloomFilter的容量
-        for(int i =0;i<100000;i++){
-            bloomFilter.put(i+"");
+    private PageSknBitSet getPageSknBitSetFromAggregationMap(Map<String, Aggregation> aggregationMap,String firstAggName){
+        if(!aggregationMap.containsKey(firstAggName)){
+            return null;
         }
-        System.out.println(bloomFilter.mightContain("1000000"));
+        List<Integer> productIdList = new ArrayList<Integer>();
+        MultiBucketsAggregation firstAggregation = (MultiBucketsAggregation) aggregationMap.get(firstAggName);
+        Iterator<? extends MultiBucketsAggregation.Bucket> firstAggregationIterator = firstAggregation.getBuckets().iterator();
+        while (firstAggregationIterator.hasNext()) {
+            MultiBucketsAggregation.Bucket bucket = firstAggregationIterator.next();
+            Integer value = Integer.valueOf(bucket.getKeyAsString());
+            productIdList.add(value);
+        }
+        PageSknBitSet pageSknBitSet = new PageSknBitSet();
+        for (Integer productId: productIdList) {
+            pageSknBitSet.add(productId);
         }
+        return pageSknBitSet;
+    }
+
 
 }

@@ -2,21 +2,21 @@ package com.yoho.search.recall.scene.beans.strategy;
 
 public enum StrategyEnum {
 
-    FIRST_SKN(110),
-    DIRECT_TRAIN(109),
+    FIRST_SKN(110),//配置的skn
+    DIRECT_TRAIN(109),//直通车
 
-    RECOMMEND_SKN(99),
+    RECOMMEND_SKN(99),//推荐的skn，线上还没有
 
-    SORT_BRAND_HEAT_VALUE(31),
-    SORT_BRAND_REDUCE_PRICE(32),
-    SORT_BRAND_PROMOTION(33),
-    SORT_BRAND_NEW(34),
+    SORT_BRAND_HEAT_VALUE(31),//品牌+品类的人气值
+    SORT_BRAND_REDUCE_PRICE(32),//品牌+品类的最新降价
+    SORT_BRAND_PROMOTION(33),//品牌+品类的新开促销
+    SORT_BRAND_NEW(34),//品牌+品类的新品
 
-    ADD_FLOW(12),
-    NEW_SHOP(11),
+    ADD_FLOW(12),//流量补偿
+    NEW_SHOP(11),//新开店铺
 
-    COMMON(1),
-    DEFAULT(0);
+    COMMON(1),//整个页面的人气兜底
+    DEFAULT(0);//其他，无视即可
 
     private Integer priority;
 

@@ -6,20 +6,25 @@ package com.yoho.search.recall.scene.constants;
  */
 public class CacheTimeConstants {
 
+    //通用召回的缓存 - 10分钟
+    public static final int COMMON_RECALL_STRATEGY_CACHE_TIME = 10 * 60;
 
-    //通用召回的缓存
-    public static final int COMMON_RECALL_STRATEGY_CACHE_TIME = 10 * 60;//兜底的缓存
-
-    //品类+品牌的缓存
+    //品类+品牌的缓存 - 60分钟
     public static final int SORT_BRAND_RECALL_STRATEGY_CACHE_TIME = 60 * 60;
 
-    //SKN的缓存
-    public static final int SKN_INFO = 15 *  60;
-
-    //SKN向量的缓存
+    //SKN向量的缓存-一个小时
     public static final int SKN_VECTOR = 60 *  60;
 
-    //用户召回结果的缓存
-    public static final int USER_RECALL_SKN_LIST = 5 *  60;
+    //SKN信息的缓存 - 10分钟
+    public static final int SKN_INFO = 10 *  60;
+
+    //用户召回结果的缓存-三分钟
+    public static final int USER_RECALL_SKN_LIST = 3 *  60;
+
+    //页面skn的bitset缓存
+    public static final int PAGE_SKN_BITSET = 60 *  60;
+
+    //页面个性化因子的缓存
+    public static final int PAGE_PERSIONAL_FACTOR = 60 *  60;
 
 }

@@ -5,21 +5,23 @@ import java.util.BitSet;
 
 public class PageSknBitSet implements Serializable{
 
+    public static final int maxValue = 2000000;
+
     private static final long serialVersionUID = 7185024266096124078L;
     private BitSet sknBitSet;
 
-    public BitSet getSknBitSet() {
-        return sknBitSet;
+    public PageSknBitSet(){
+        this.sknBitSet = new BitSet(maxValue);
     }
 
-    public void setSknBitSet(BitSet sknBitSet) {
-        this.sknBitSet = sknBitSet;
+    public void add(int bitSetIndex) {
+       if(bitSetIndex>maxValue){
+           return;
+       }
+        this.sknBitSet.set(bitSetIndex);
     }
 
     public boolean exist(int bitSetIndex) {
-        if(this.sknBitSet==null){
-            return false;
-        }
         return sknBitSet.get(bitSetIndex);
     }
 }