yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit cf069e1320486f61caeeb45a96c8b964700db7e4 1 parent 12e56c97

fix AbstractCacheComponent

Inline Side-by-side
Showing 5 changed files with 298 additions and 12 deletions
  1 +package com.yoho.search.cache.beans;
  2 +
  3 +import com.alibaba.fastjson.JSON;
  4 +import com.alibaba.fastjson.JSONArray;
  5 +import com.alibaba.fastjson.JSONObject;
  6 +import com.alibaba.fastjson.TypeReference;
  7 +import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
  8 +import com.yoho.search.cache.impls.EhCache;
  9 +import com.yoho.search.cache.impls.SearchRedis;
  10 +import com.yoho.search.cache.model.CacheObject;
  11 +import com.yoho.search.service.recall.models.common.ParamQueryFilter;
  12 +import org.springframework.beans.factory.annotation.Autowired;
  13 +
  14 +import java.lang.reflect.ParameterizedType;
  15 +import java.lang.reflect.Type;
  16 +import java.util.List;
  17 +
  18 +public abstract class AbstractCacheComponent<T> {
  19 +
  20 + @Autowired
  21 + private EhCache ehCache;
  22 + @Autowired
  23 + private SearchRedis searchRedis;
  24 +
  25 + public T queryWithCache(ParamQueryFilter paramQueryFilter) {
  26 + //1、生成RedisKeyBuilder
  27 + RedisKeyBuilder redisKeyBuilder = this.genRedisKeyBuilder(paramQueryFilter);
  28 + if (redisKeyBuilder == null) {
  29 + return null;
  30 + }
  31 + //2、取本地缓存，缓存命中，则直接返回
  32 + T result = this.getValueFromEhcache(redisKeyBuilder);
  33 + if (result != null) {
  34 + return result;
  35 + }
  36 + //3、取redis缓存，缓存命中，则回写ehcahce
  37 + result = this.getValueFromRedis(redisKeyBuilder);
  38 + if (result != null && useEhcache()) {
  39 + this.addValueToEhcache(redisKeyBuilder, result);
  40 + return result;
  41 + }
  42 + return this.doInnerQuery(redisKeyBuilder, paramQueryFilter);
  43 + }
  44 +
  45 + private T doInnerQuery(RedisKeyBuilder redisKeyBuilder, ParamQueryFilter paramQueryFilter) {
  46 + T result = this.doRealQuery(paramQueryFilter);
  47 + if (result == null) {
  48 + return result;
  49 + }
  50 + if (useEhcache()) {
  51 + this.addValueToEhcache(redisKeyBuilder, result);
  52 + }
  53 + if (useRedis()) {
  54 + this.addValueToRedis(redisKeyBuilder, result);
  55 + }
  56 + return result;
  57 + }
  58 +
  59 + private T getValueFromEhcache(RedisKeyBuilder redisKeyBuilder) {
  60 + if(!useEhcache()){
  61 + return null;
  62 + }
  63 + CacheObject cacheObject = ehCache.get(redisKeyBuilder);
  64 + if (cacheObject != null) {
  65 + Object object = cacheObject.toObject();
  66 + return (T) object;
  67 + }
  68 + return null;
  69 + }
  70 +
  71 + private void addValueToEhcache(RedisKeyBuilder redisKeyBuilder, T result) {
  72 + ehCache.addOrUpdate(redisKeyBuilder, new CacheObject(result), this.cacheTimeInMinute());
  73 + }
  74 +
  75 + protected T getValueFromRedis(RedisKeyBuilder redisKeyBuilder) {
  76 + if(!useRedis()){
  77 + return null;
  78 + }
  79 + CacheObject cacheObject = searchRedis.get(redisKeyBuilder);
  80 + if (cacheObject == null) {
  81 + return null;
  82 + }
  83 + String redisValue = (String)cacheObject.toObject();
  84 + Type superClass = getClass().getGenericSuperclass();
  85 + Type type = ((ParameterizedType) superClass).getActualTypeArguments()[0];
  86 + return JSON.parseObject(redisValue, type);
  87 + }
  88 +
  89 + protected void addValueToRedis(RedisKeyBuilder redisKeyBuilder, T result) {
  90 + String jsonString = JSON.toJSONString(result);
  91 + CacheObject toCacheResult = new CacheObject(jsonString);
  92 + searchRedis.addOrUpdate(redisKeyBuilder, toCacheResult, this.cacheTimeInMinute());
  93 + }
  94 +
  95 + protected abstract RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter);
  96 +
  97 + protected abstract int cacheTimeInMinute();
  98 +
  99 + protected abstract T doRealQuery(ParamQueryFilter paramQueryFilter);
  100 +
  101 + protected boolean useEhcache() {
  102 + return false;
  103 + }
  104 +
  105 + protected boolean useRedis() {
  106 + return true;
  107 + }
  108 +
  109 +}
  1 +package com.yoho.search.service.recall.beans.persional;
  2 +
  3 +import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
  4 +import com.yoho.search.base.utils.ISearchConstants;
  5 +import com.yoho.search.base.utils.ProductIndexEsField;
  6 +import com.yoho.search.cache.beans.AbstractCacheComponent;
  7 +import com.yoho.search.core.es.model.SearchParam;
  8 +import com.yoho.search.core.es.model.SearchResult;
  9 +import com.yoho.search.core.personalized.models.SortBrand;
  10 +import com.yoho.search.service.recall.beans.helper.ExtendFilterHelper;
  11 +import com.yoho.search.cache.CacheTimeConstants;
  12 +import com.yoho.search.service.recall.models.common.ParamQueryFilter;
  13 +import com.yoho.search.service.recall.models.personal.PagePersonalFactor;
  14 +import com.yoho.search.common.SearchCommonService;
  15 +import org.elasticsearch.index.query.BoolQueryBuilder;
  16 +import org.elasticsearch.index.query.QueryBuilders;
  17 +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
  18 +import org.elasticsearch.search.aggregations.Aggregation;
  19 +import org.elasticsearch.search.aggregations.AggregationBuilders;
  20 +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
  21 +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
  22 +import org.springframework.beans.factory.annotation.Autowired;
  23 +import org.springframework.stereotype.Component;
  24 +
  25 +import java.util.*;
  26 +
  27 +@Component
  28 +public class CachePersionalFactorComponent extends AbstractCacheComponent<PagePersonalFactor> {
  29 +
  30 + @Autowired
  31 + private SearchCommonService searchCommonService;
  32 +
  33 + /**
  34 + * 查询个性化因子
  35 + *
  36 + * @param paramQueryFilter
  37 + * @return
  38 + */
  39 + public PagePersonalFactor queryPagePersionalFactor(ParamQueryFilter paramQueryFilter) {
  40 + Object value = super.queryWithCache(paramQueryFilter);
  41 + return value == null ? null : (PagePersonalFactor) value;
  42 + }
  43 +
  44 + @Override
  45 + protected RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
  46 + return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("PAGE_FACTORS_NEW:").appendVar(paramQueryFilter.getParamMd5Key());
  47 + }
  48 +
  49 + @Override
  50 + protected int cacheTimeInMinute() {
  51 + return CacheTimeConstants.PAGE_PERSIONAL_FACTOR;
  52 + }
  53 +
  54 + @Override
  55 + protected PagePersonalFactor doRealQuery(ParamQueryFilter paramQueryFilter) {
  56 + //1、构造参数
  57 + SearchParam searchParam = new SearchParam();
  58 + searchParam.setQuery(paramQueryFilter.getParamQuery());
  59 +
  60 + BoolQueryBuilder filter = QueryBuilders.boolQuery();
  61 + filter.must(paramQueryFilter.getParamFilter());
  62 + filter.mustNot(ExtendFilterHelper.notRecallFilter());//聚合的时候带上不召回的数据
  63 + searchParam.setFiter(filter);
  64 +
  65 + searchParam.setSize(0);
  66 +
  67 + //2、构造聚合参数
  68 + List<AbstractAggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
  69 + aggregationBuilders.add(brandSortAggBuilder());//品类-品牌聚合
  70 + searchParam.setAggregationBuilders(aggregationBuilders);
  71 +
  72 + //3、执行查询
  73 + SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
  74 +
  75 + //4、构造结果
  76 + Map<String, Aggregation> aggregationMap = searchResult.getAggMaps();
  77 + List<SortBrand> sortBrands = this.getBrandSortsFromAggregationMap(aggregationMap);
  78 + List<Integer> misortIds = this.getMisortIds(sortBrands);
  79 + List<Integer> brandIds = this.getBrandIds(sortBrands);
  80 + return new PagePersonalFactor(misortIds, brandIds, sortBrands);
  81 + }
  82 +
  83 + private List<Integer> getMisortIds(List<SortBrand> sortBrands) {
  84 + if (sortBrands == null || sortBrands.isEmpty()) {
  85 + return new ArrayList<>();
  86 + }
  87 + List<Integer> results = new ArrayList<>();
  88 + for (SortBrand sortBrand : sortBrands) {
  89 + Integer misort = sortBrand.getMisort();
  90 + if (!results.contains(misort)) {
  91 + results.add(misort);
  92 + }
  93 + }
  94 + return results;
  95 + }
  96 +
  97 + private List<Integer> getBrandIds(List<SortBrand> sortBrands) {
  98 + if (sortBrands == null || sortBrands.isEmpty()) {
  99 + return new ArrayList<>();
  100 + }
  101 + List<Integer> results = new ArrayList<>();
  102 + for (SortBrand sortBrand : sortBrands) {
  103 + Integer brandId = sortBrand.getBrandId();
  104 + if (!results.contains(brandId)) {
  105 + results.add(brandId);
  106 + }
  107 + }
  108 + return results;
  109 + }
  110 +
  111 + /**
  112 + * 品类+品牌聚合
  113 + *
  114 + * @return
  115 + */
  116 + private TermsAggregationBuilder brandSortAggBuilder() {
  117 + TermsAggregationBuilder middleSortAggBuilder = AggregationBuilders.terms("sortBrandBrandIdAgg").field(ProductIndexEsField.brandId).size(1000);
  118 + middleSortAggBuilder.subAggregation(AggregationBuilders.terms("sortBrandMiddleSortAgg").field(ProductIndexEsField.middleSortId).size(200));
  119 + return middleSortAggBuilder;
  120 + }
  121 +
  122 + private List<SortBrand> getBrandSortsFromAggregationMap(Map<String, Aggregation> aggregationMap) {
  123 + Map<Integer, List<Integer>> brand2MiSortIdsMap = this.getValueFromAggregationMap(aggregationMap, "sortBrandBrandIdAgg", "sortBrandMiddleSortAgg");
  124 + List<SortBrand> pageBrandSorts = new ArrayList<>();
  125 + for (Map.Entry<Integer, List<Integer>> entry : brand2MiSortIdsMap.entrySet()) {
  126 + Integer brandId = entry.getKey();
  127 + List<Integer> misorts = entry.getValue();
  128 + for (Integer misort : misorts) {
  129 + pageBrandSorts.add(new SortBrand(misort, brandId));
  130 + }
  131 + }
  132 + return pageBrandSorts;
  133 + }
  134 +
  135 + /**
  136 + * 从聚合结果中获取参数，仅支持二层聚合
  137 + *
  138 + * @param aggregationMap
  139 + * @param firstAggName
  140 + * @param secondAggName
  141 + * @return
  142 + */
  143 + private Map<Integer, List<Integer>> getValueFromAggregationMap(Map<String, Aggregation> aggregationMap, String firstAggName, String secondAggName) {
  144 + Map<Integer, List<Integer>> aggResultMap = new HashMap<>();
  145 + if (!aggregationMap.containsKey(firstAggName)) {
  146 + return aggResultMap;
  147 + }
  148 + MultiBucketsAggregation firstAggregation = (MultiBucketsAggregation) aggregationMap.get(firstAggName);
  149 + Iterator<? extends MultiBucketsAggregation.Bucket> firstAggregationIterator = firstAggregation.getBuckets().iterator();
  150 + while (firstAggregationIterator.hasNext()) {
  151 + MultiBucketsAggregation.Bucket firstAggregationBucket = firstAggregationIterator.next();
  152 + Integer firstAggregationBucketKey = Integer.valueOf(firstAggregationBucket.getKeyAsString());
  153 + Map<String, Aggregation> secondAggregationMap = firstAggregationBucket.getAggregations().asMap();
  154 + if (secondAggregationMap == null || !secondAggregationMap.containsKey(secondAggName)) {
  155 + continue;
  156 + }
  157 + List<Integer> secondAggregationBucketKeys = this.getAggValuesFromMultiBucketsAggregation((MultiBucketsAggregation) secondAggregationMap.get(secondAggName));
  158 + aggResultMap.put(firstAggregationBucketKey, secondAggregationBucketKeys);
  159 + }
  160 + return aggResultMap;
  161 + }
  162 +
  163 + private List<Integer> getAggValuesFromMultiBucketsAggregation(MultiBucketsAggregation aggregation) {
  164 + List<Integer> results = new ArrayList<>();
  165 + if (aggregation == null) {
  166 + return results;
  167 + }
  168 + Iterator<? extends MultiBucketsAggregation.Bucket> bucketsIterator = aggregation.getBuckets().iterator();
  169 + while (bucketsIterator.hasNext()) {
  170 + MultiBucketsAggregation.Bucket bucket = bucketsIterator.next();
  171 + Integer value = Integer.valueOf(bucket.getKeyAsString());
  172 + results.add(value);
  173 + }
  174 + return results;
  175 + }
  176 +
  177 +}
@@ -24,7 +24,7 @@ public class QueryUserPersionalFactorBean { @@ -24,7 +24,7 @@ public class QueryUserPersionalFactorBean {
24 private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL"); 24 private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
25 25
26 @Autowired 26 @Autowired
27 - private PagePersionalFactorComponent pageComponent; 27 + private CachePersionalFactorComponent pageComponent;
28 @Autowired 28 @Autowired
29 private UserPersionalFactorComponent userComponent; 29 private UserPersionalFactorComponent userComponent;
30 @Autowired 30 @Autowired
@@ -4,7 +4,7 @@ import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder; @@ -4,7 +4,7 @@ import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
4 import com.yoho.search.base.utils.ISearchConstants; 4 import com.yoho.search.base.utils.ISearchConstants;
5 import com.yoho.search.base.utils.ProductIndexEsField; 5 import com.yoho.search.base.utils.ProductIndexEsField;
6 import com.yoho.search.cache.CacheTimeConstants; 6 import com.yoho.search.cache.CacheTimeConstants;
7 -import com.yoho.search.cache.beans.AbstractPageComponent; 7 +import com.yoho.search.cache.beans.AbstractCacheComponent;
8 import com.yoho.search.core.es.model.SearchParam; 8 import com.yoho.search.core.es.model.SearchParam;
9 import com.yoho.search.core.es.model.SearchResult; 9 import com.yoho.search.core.es.model.SearchResult;
10 import com.yoho.search.service.recall.beans.builder.UserRecallRequestBuilder; 10 import com.yoho.search.service.recall.beans.builder.UserRecallRequestBuilder;
@@ -25,7 +25,7 @@ import org.springframework.stereotype.Component; @@ -25,7 +25,7 @@ import org.springframework.stereotype.Component;
25 import java.util.*; 25 import java.util.*;
26 26
27 @Component 27 @Component
28 -public class ActivityShopBrandListService extends AbstractPageComponent<ActivityShopBrandList> { 28 +public class ActivityShopBrandListService extends AbstractCacheComponent<List<ActivityShopBrand>> {
29 29
30 private static final Logger logger = LoggerFactory.getLogger(ActivityShopBrandListService.class); 30 private static final Logger logger = LoggerFactory.getLogger(ActivityShopBrandListService.class);
31 31
@@ -43,11 +43,11 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity @@ -43,11 +43,11 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity
43 public List<ActivityShopBrand> queryActivityShopBrandList(Map<String, String> paramMap) { 43 public List<ActivityShopBrand> queryActivityShopBrandList(Map<String, String> paramMap) {
44 try { 44 try {
45 ParamQueryFilter paramQueryFilter = userRecallRequestBuilder.buildParamQueryFilter(paramMap); 45 ParamQueryFilter paramQueryFilter = userRecallRequestBuilder.buildParamQueryFilter(paramMap);
46 - Object value = super.queryWithCache(paramQueryFilter, ActivityShopBrandList.class); 46 + List<ActivityShopBrand> value = super.queryWithCache(paramQueryFilter);
47 if(value==null){ 47 if(value==null){
48 return new ArrayList<>(); 48 return new ArrayList<>();
49 } 49 }
50 - return new ArrayList<>(((ActivityShopBrandList) value).getShopBrandList()); 50 + return value;
51 } catch (Exception e) { 51 } catch (Exception e) {
52 logger.error(e.getMessage(), e); 52 logger.error(e.getMessage(), e);
53 return new ArrayList<>(); 53 return new ArrayList<>();
@@ -56,7 +56,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity @@ -56,7 +56,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity
56 56
57 @Override 57 @Override
58 protected RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) { 58 protected RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
59 - return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("ACTIVITY_SHOP_BRAND_LIST:").appendVar(paramQueryFilter.getParamMd5Key()); 59 + return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("ACTIVITY_SHOP_BRANDS:").appendVar(paramQueryFilter.getParamMd5Key());
60 } 60 }
61 61
62 @Override 62 @Override
@@ -65,7 +65,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity @@ -65,7 +65,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity
65 } 65 }
66 66
67 @Override 67 @Override
68 - protected ActivityShopBrandList doRealQuery(ParamQueryFilter paramQueryFilter) { 68 + protected List<ActivityShopBrand> doRealQuery(ParamQueryFilter paramQueryFilter) {
69 //1、构造参数 69 //1、构造参数
70 SearchParam searchParam = new SearchParam(); 70 SearchParam searchParam = new SearchParam();
71 searchParam.setQuery(paramQueryFilter.getParamQuery()); 71 searchParam.setQuery(paramQueryFilter.getParamQuery());
@@ -95,7 +95,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity @@ -95,7 +95,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity
95 return middleSortAggBuilder; 95 return middleSortAggBuilder;
96 } 96 }
97 97
98 - private ActivityShopBrandList genActivityBrandShops(Map<String, Aggregation> aggregationMap) { 98 + private List<ActivityShopBrand> genActivityBrandShops(Map<String, Aggregation> aggregationMap) {
99 //1、获取二层聚合的结果 99 //1、获取二层聚合的结果
100 List<AggKeyCountTwoLevel> aggKeyCountTwoLevelList = AggCommonHelper.getAggKeyCountTwoLevelList(aggregationMap, "shopAgg", "brandAgg"); 100 List<AggKeyCountTwoLevel> aggKeyCountTwoLevelList = AggCommonHelper.getAggKeyCountTwoLevelList(aggregationMap, "shopAgg", "brandAgg");
101 //2、使用第一层聚合的总数量排序 101 //2、使用第一层聚合的总数量排序
@@ -124,7 +124,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity @@ -124,7 +124,7 @@ public class ActivityShopBrandListService extends AbstractPageComponent<Activity
124 break; 124 break;
125 } 125 }
126 } 126 }
127 - return new ActivityShopBrandList(shopBrandList); 127 + return shopBrandList;
128 } 128 }
129 129
130 } 130 }
@@ -6,7 +6,7 @@ import com.yoho.error.event.SearchEvent; @@ -6,7 +6,7 @@ import com.yoho.error.event.SearchEvent;
6 import com.yoho.search.base.utils.EventReportEnum; 6 import com.yoho.search.base.utils.EventReportEnum;
7 import com.yoho.search.base.utils.ISearchConstants; 7 import com.yoho.search.base.utils.ISearchConstants;
8 import com.yoho.search.cache.CacheTimeConstants; 8 import com.yoho.search.cache.CacheTimeConstants;
9 -import com.yoho.search.cache.beans.AbstractPageComponent; 9 +import com.yoho.search.cache.beans.AbstractCacheComponent;
10 import com.yoho.search.common.SearchCommonService; 10 import com.yoho.search.common.SearchCommonService;
11 import com.yoho.search.core.es.agg.IAggregation; 11 import com.yoho.search.core.es.agg.IAggregation;
12 import com.yoho.search.core.es.model.SearchParam; 12 import com.yoho.search.core.es.model.SearchParam;
@@ -28,7 +28,7 @@ import java.util.Arrays; @@ -28,7 +28,7 @@ import java.util.Arrays;
28 import java.util.Map; 28 import java.util.Map;
29 29
30 @Service 30 @Service
31 -public class DiscountService extends AbstractPageComponent<JSONObject> implements ApplicationEventPublisherAware { 31 +public class DiscountService extends AbstractCacheComponent<JSONObject> implements ApplicationEventPublisherAware {
32 32
33 private static final Logger logger = LoggerFactory.getLogger(DiscountService.class); 33 private static final Logger logger = LoggerFactory.getLogger(DiscountService.class);
34 34
@@ -55,7 +55,7 @@ public class DiscountService extends AbstractPageComponent<JSONObject> implement @@ -55,7 +55,7 @@ public class DiscountService extends AbstractPageComponent<JSONObject> implement
55 public JSONObject discount(Map<String, String> paramMap, BoolQueryBuilder mustFilter) { 55 public JSONObject discount(Map<String, String> paramMap, BoolQueryBuilder mustFilter) {
56 try { 56 try {
57 ParamQueryFilter paramQueryFilter = searchParamHelper.buildParamQueryFilter(paramMap, mustFilter); 57 ParamQueryFilter paramQueryFilter = searchParamHelper.buildParamQueryFilter(paramMap, mustFilter);
58 - Object value = super.queryWithCache(paramQueryFilter, JSONObject.class); 58 + Object value = super.queryWithCache(paramQueryFilter);
59 return value == null ? null : (JSONObject) value; 59 return value == null ? null : (JSONObject) value;
60 } catch (Exception e) { 60 } catch (Exception e) {
61 logger.error(e.getMessage(), e); 61 logger.error(e.getMessage(), e);