添加缓存，聚合优化

hugufei
Commit 057b7b1ff0e4b8248c34db6f9134e7b0b9e9f1cd 1 parent 803d1b5c
Showing 5 changed files with 44 additions and 189 deletions
service/src/main/java/com/yoho/search/service/helper/AggCommonHelper.java
service/src/main/java/com/yoho/search/service/helper/AggProductListHelper.java
service/src/main/java/com/yoho/search/service/scene/AggProductListService.java
service/src/main/java/com/yoho/search/service/service/impl/ShopListServiceImpl.java
service/src/main/java/com/yoho/search/service/service/impl/WebNewShelveServiceImpl.java
--- a/service/src/main/java/com/yoho/search/service/helper/AggCommonHelper.java
View file @057b7b1
+++ b/service/src/main/java/com/yoho/search/service/helper/AggCommonHelper.java
View file @057b7b1
@@ -31,6 +31,28 @@ public class AggCommonHelper {
     private static final Logger logger = LoggerFactory.getLogger(AggCommonHelper.class);
+    /**
+     * 从TopHits中获取_source
+     *
+     * @param topHits
+     * @return
+     */
+    public static List<Map<String, Object>> getTopHitResultsWithScore(TopHits topHits) {
+        List<Map<String, Object>> topHitList = new ArrayList<>();
+        if (topHits == null) {
+            return topHitList;
+        }
+        SearchHits hits = topHits.getHits();
+        for (SearchHit hit : hits.getHits()) {
+            Map<String, Object> source = hit.getSource();
+            float _score = hit.getScore();
+            source.put("_score", _score);
+            topHitList.add(source);
+        }
+        return topHitList;
+    }
+
+
     public static List<Integer> getIdsFromAggMaps(Map<String, Aggregation> aggMaps, String aggName) {
         if (!aggMaps.containsKey(aggName)) {
             return new ArrayList<>();
@@ -53,7 +75,7 @@ public class AggCommonHelper {
      *
      * @param simpleFieldAggs
      * @param topHitOrder
-	 * @param topHitFieldCount
+     * @param topHitCount
      * @return
      */
     public static AbstractAggregationBuilder<?> getTopHitAggregation(List<SimpleFieldAgg> simpleFieldAggs, String topHitOrder, int topHitCount) {
@@ -85,12 +107,6 @@ public class AggCommonHelper {
     /**
      * 获取聚合出来的商品列表，并按特定顺序截取
-	 * 
-	 * @param aggregation
-	 * @param viewNum
-	 * @param sortField
-	 * @param sortOrder
-	 * @return
      */
     public static List<Map<String, Object>> getTopHitList(Map<String, Aggregation> aggMaps, List<SimpleFieldAgg> simpleFieldAggs, String topHitOrder, int totalCount) {
         List<Map<String, Object>> results = new ArrayList<Map<String, Object>>();
@@ -99,6 +115,7 @@ public class AggCommonHelper {
         return results;
     }
+
     private static void getTopHitResults(List<Map<String, Object>> results, Map<String, Aggregation> aggMaps, List<SimpleFieldAgg> simpleFieldAggs, int index) {
         try {
             String aggName = simpleFieldAggs.get(index++).getAggName();
@@ -113,15 +130,8 @@ public class AggCommonHelper {
                 Map<String, Aggregation> aggMap = bucket.getAggregations().getAsMap();
                 if (aggMap.containsKey("topHit")) {
                     TopHits topHits = bucket.getAggregations().get("topHit");
-					if (topHits != null) {
-						SearchHits hits = topHits.getHits();
-						for (SearchHit hit : hits.getHits()) {
-							Map<String, Object> source = hit.getSource();
-							float _score = hit.getScore();
-							source.put("_score", _score);
-							results.add(source);
-						}
-					}
+                    List<Map<String, Object>> topHitList = getTopHitResultsWithScore(topHits);
+                    results.addAll(topHitList);
                 } else {
                     getTopHitResults(results, aggMap, simpleFieldAggs, index);
                 }
@@ -133,7 +143,7 @@ public class AggCommonHelper {
     private static List<Map<String, Object>> sortListBySortField(List<Map<String, Object>> productList, String topHitOrder, int viewNum) {
         if (productList == null || productList.isEmpty()) {
-			return new ArrayList<Map<String, Object>>();
+            return new ArrayList<>();
         }
         // 再按照某个字段对商品排序
         FieldSortOrder fieldSortOrder = new FieldSortOrder(topHitOrder);
@@ -173,38 +183,27 @@ public class AggCommonHelper {
     /**
      * 按聚合的key获取聚合出来的商品列表[目前只支持一个field,多重field的话，key不好处理]
      *
-	 * @param aggregation
-	 * @param viewNum
-	 * @param sortField
-	 * @param sortOrder
      * @return
      */
     public static List<KeyTopHitModel> getTopHitListGroupByKey(Map<String, Aggregation> aggMaps, SimpleFieldAgg simpleFieldAgg) {
         String aggName = simpleFieldAgg.getAggName();
         if (!aggMaps.containsKey(aggName)) {
-			return new ArrayList<KeyTopHitModel>();
+            return new ArrayList<>();
         }
         List<? extends Bucket> bucketList = ((MultiBucketsAggregation) aggMaps.get(aggName)).getBuckets();
         if (bucketList.isEmpty()) {
-			return new ArrayList<KeyTopHitModel>();
+            return new ArrayList<>();
         }
         List<KeyTopHitModel> results = new ArrayList<KeyTopHitModel>();
         for (Bucket bucket : bucketList) {
             String key = bucket.getKeyAsString();
             long count = bucket.getDocCount();
-			List<Map<String, Object>> topHitList = new ArrayList<Map<String, Object>>();
+            List<Map<String, Object>> topHitList = new ArrayList<>();
             Map<String, Aggregation> aggMap = bucket.getAggregations().getAsMap();
             if (aggMap.containsKey("topHit")) {
                 TopHits topHits = bucket.getAggregations().get("topHit");
-				if (topHits != null) {
-					SearchHits hits = topHits.getHits();
-					for (SearchHit hit : hits.getHits()) {
-						Map<String, Object> source = hit.getSource();
-						float _score = hit.getScore();
-						source.put("_score", _score);
-						topHitList.add(source);
-					}
-				}
+                List<Map<String, Object>> tempTopHitList = getTopHitResultsWithScore(topHits);
+                topHitList.addAll(tempTopHitList);
             }
             KeyTopHitModel keyTopHitModel = new KeyTopHitModel(key, count, topHitList);
             results.add(keyTopHitModel);
--- a/service/src/main/java/com/yoho/search/service/helper/AggProductListHelper.java deleted 100644 → 0
View file @803d1b5
+++ b/service/src/main/java/com/yoho/search/service/helper/AggProductListHelper.java deleted 100644 → 0
View file @803d1b5
-package com.yoho.search.service.helper;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
-import org.elasticsearch.search.sort.SortOrder;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import com.yoho.search.base.utils.CollectionUtils;
-import com.yoho.search.service.base.index.ProductIndexBaseService;
-
-@Component
-public class AggProductListHelper {
-
-	@Autowired
-	private ProductIndexBaseService productIndexBaseService;
-
-	/**
-	 * 获取聚合出来的商品列表
-	 * 
-	 * @param aggregation
-	 * @return
-	 */
-	public List<Map<String, Object>> getProductListFromAggregation(final MultiBucketsAggregation aggregation) {
-		Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
-		List<Map<String, Object>> productEsSourceList = new ArrayList<Map<String, Object>>();
-		while (itAgg.hasNext()) {
-			Bucket lt = itAgg.next();
-			if (lt.getAggregations().getAsMap().containsKey("product")) {
-				TopHits topHits = lt.getAggregations().get("product");
-				if (topHits != null) {
-					SearchHits hits = topHits.getHits();
-					for (SearchHit hit : hits.getHits()) {
-						productEsSourceList.add(hit.getSource());
-					}
-				}
-			}
-		}
-		if (productEsSourceList == null || productEsSourceList.isEmpty()) {
-			return new ArrayList<Map<String, Object>>();
-		}
-		return productIndexBaseService.getProductListWithPricePlan(productEsSourceList);
-	}
-
-	/**
-	 * 获取聚合出来的商品列表，并按特定顺序截取
-	 * 
-	 * @param aggregation
-	 * @param viewNum
-	 * @param sortField
-	 * @param sortOrder
-	 * @return
-	 */
-	public List<Map<String, Object>> getProductListFromAggregation(final MultiBucketsAggregation aggregation, int viewNum, String sortField, SortOrder sortOrder) {
-		Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
-		// 获取品牌聚合出来的商品
-		List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
-		while (itAgg.hasNext()) {
-			Bucket lt = itAgg.next();
-			if (lt.getAggregations().getAsMap().containsKey("product")) {
-				TopHits topHits = lt.getAggregations().get("product");
-				if (topHits == null) {
-					continue;
-				}
-				SearchHits hits = topHits.getHits();
-				for (SearchHit hit : hits.getHits()) {
-					Map<String, Object> source = hit.getSource();
-					float _score = hit.getScore();
-					source.put("_score", _score);
-					dataList.add(source);
-				}
-			}
-		}
-		dataList = this.sortListBySortField(dataList, viewNum, sortField, sortOrder);
-		return productIndexBaseService.getProductListWithPricePlan(dataList);
-	}
-
-	private List<Map<String, Object>> sortListBySortField(List<Map<String, Object>> productList, int viewNum, String orderField, SortOrder sortOrder) {
-		if (productList == null || productList.isEmpty()) {
-			return new ArrayList<Map<String, Object>>();
-		}
-		// 再按照某个字段对商品排序
-		boolean isDesc = sortOrder == SortOrder.DESC ? true : false;
-		Collections.sort(productList, new Comparator<Map<String, Object>>() {
-			public int compare(Map<String, Object> o1, Map<String, Object> o2) {
-				double value1 = getDouble(o1.get(orderField));
-				double value2 = getDouble(o2.get(orderField));
-				if (isDesc) {
-					return value1 > value2 ? -1 : value1 < value2 ? 1 : 0;
-				} else {
-					return value1 > value2 ? 1 : value1 < value2 ? -1 : 0;
-				}
-			}
-		});
-		if (productList.size() > viewNum) {
-			productList = CollectionUtils.safeSubList(productList, 0, viewNum);
-		}
-		return productList;
-	}
-
-	private double getDouble(Object value) {
-		try {
-			return Double.parseDouble(value.toString());
-		} catch (Exception e) {
-			return 0;
-		}
-	}
-}
--- a/service/src/main/java/com/yoho/search/service/scene/AggProductListService.java
View file @057b7b1
+++ b/service/src/main/java/com/yoho/search/service/scene/AggProductListService.java
View file @057b7b1
@@ -14,6 +14,7 @@ import com.yoho.search.models.SearchApiResult;
 import com.yoho.search.models.SearchSort;
 import com.yoho.search.service.base.SearchCommonService;
 import com.yoho.search.service.base.index.ProductIndexBaseService;
+import com.yoho.search.service.helper.AggCommonHelper;
 import com.yoho.search.service.helper.SearchCommonHelper;
 import com.yoho.search.service.helper.SearchParamHelper;
 import com.yoho.search.service.helper.SearchSortHelper;
@@ -207,7 +208,7 @@ public class AggProductListService implements ApplicationEventPublisherAware {
 	 *            【父聚合的字段名称】
 	 * @param firstAggBucketCount
 	 *            【父聚合的桶大小】
-	 * @param aggOrderSearchSort
+	 * @param aggSort
 	 *            【聚合排序规则，即按什么排序推荐】
 	 * @param secondSearchSort
 	 *            【次要排序条件，即最终商品按什么排序】
@@ -291,15 +292,7 @@ public class AggProductListService implements ApplicationEventPublisherAware {
 			Bucket lt = itAgg.next();
 			if (lt.getAggregations().getAsMap().containsKey("product")) {
 				TopHits topHits = lt.getAggregations().get("product");
-				if (topHits != null) {
-					SearchHits hits = topHits.getHits();
-					for (SearchHit hit : hits.getHits()) {
-						Map<String, Object> source = hit.getSource();
-						float _score = hit.getScore();
-						source.put("_score", _score);
-						productList.add(source);
-					}
-				}
+				productList.addAll(AggCommonHelper.getTopHitResultsWithScore(topHits));
 			}
 		}
 		return productList;
--- a/service/src/main/java/com/yoho/search/service/service/impl/ShopListServiceImpl.java
View file @057b7b1
+++ b/service/src/main/java/com/yoho/search/service/service/impl/ShopListServiceImpl.java
View file @057b7b1
@@ -15,6 +15,7 @@ import com.yoho.search.service.base.SearchCommonService;
 import com.yoho.search.service.base.SearchRequestParams;
 import com.yoho.search.service.base.index.BrandIndexBaseService;
 import com.yoho.search.service.base.index.ShopsIndexBaseService;
+import com.yoho.search.service.helper.AggCommonHelper;
 import com.yoho.search.service.helper.SearchCommonHelper;
 import com.yoho.search.service.service.IShopListService;
 import org.apache.commons.lang.StringUtils;
@@ -151,20 +152,13 @@ public class ShopListServiceImpl implements IShopListService {
 			Bucket lt = itAgg.next();
 			if (lt.getAggregations().getAsMap().containsKey("product")) {
 				TopHits topHits = lt.getAggregations().get("product");
-				if (topHits != null) {
-					SearchHits hits = topHits.getHits();
-					for (SearchHit hit : hits.getHits()) {
-						Map<String, Object> source = hit.getSource();
-						float _score = hit.getScore();
-						source.put("_score", _score);
-						productSourceList.add(source);
-					}
-				}
+				productSourceList.addAll(AggCommonHelper.getTopHitResultsWithScore(topHits));
 			}
 		}
 		if (productSourceList == null || productSourceList.isEmpty()) {
 			return new ArrayList<Map<String, Object>>();
 		}
+
 		List<Integer> yohoShopIds = new ArrayList<Integer>();
 		List<Integer> globalBrandIds = new ArrayList<Integer>();
--- a/service/src/main/java/com/yoho/search/service/service/impl/WebNewShelveServiceImpl.java
View file @057b7b1
+++ b/service/src/main/java/com/yoho/search/service/service/impl/WebNewShelveServiceImpl.java
View file @057b7b1
@@ -6,8 +6,10 @@ import com.yoho.search.base.utils.CollectionUtils;
 import com.yoho.search.base.utils.EventReportEnum;
 import com.yoho.search.base.utils.ISearchConstants;
 import com.yoho.search.base.utils.ProductIndexEsField;
+import com.yoho.search.common.cache.CacheType;
 import com.yoho.search.common.cache.SearchCacheFactory;
 import com.yoho.search.common.cache.SearchCacheMatchLogger;
+import com.yoho.search.common.cache.aop.SearchCacheAble;
 import com.yoho.search.common.cache.model.SearchCache;
 import com.yoho.search.common.utils.SearchApiResultUtils;
 import com.yoho.search.core.es.model.SearchParam;
@@ -62,17 +64,12 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
     @Autowired
     private SearchServiceHelper searchServiceHelper;
     @Autowired
-    private SearchCacheService searchCacheService;
-    @Autowired
-    private SearchCacheFactory searchCacheFactory;
-    @Autowired
     private SearchCommonService searchCommonService;
     @Autowired
     private WebProductIndexBaseService webProductIndexBaseService;
     @Autowired
     private ProductListSortService productListSortService;
-    private SearchCache searchCache;
     private ApplicationEventPublisher publisher;
     @Override
@@ -80,12 +77,8 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
         this.publisher = applicationEventPublisher;
     }
-    @PostConstruct
-    void init() {
-        searchCache = searchCacheFactory.getBrandRelatedCache();
-    }
-
     @Override
+    @SearchCacheAble(cacheName = "WEB_NEW_SHELVE",cacheInMinute = 30,cacheType = CacheType.SEARCH_REDIS,excludeParams={ "uid","udid"})
     public SearchApiResult webNewShelveProductList(Map<String, String> paramMap) {
         try {
             logger.info("[func=aggProductsByBrandId][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
@@ -130,14 +123,7 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
             // 5、构造返回结果
             SearchApiResult searchApiResult = new SearchApiResult().setMessage("new-shelve list");
-            // 6、先从缓存中获取,如果能取到，则直接返回
-            JSONArray jsonArray = searchCacheService.getJSONArrayFromCache(this.searchCache, ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
-            if (jsonArray != null) {
-                SearchCacheMatchLogger.doSearchCacheMatchLog("/new-shelve.json", paramMap);
-                return searchApiResult.setData(jsonArray);
-            }
-
-            // 7、执行搜索,并构造返回结果
+            // 6、执行搜索,并构造返回结果
             final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
             SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
             if (searchResult == null || searchResult.getAggMaps() == null) {
@@ -148,7 +134,6 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
                 return searchApiResult.setData("");
             }
             JSONArray productList = this.getNewShelveResponseMap(((MultiBucketsAggregation) aggMaps.get("brandAgg")));
-            searchCacheService.addJSONArrayToCache(this.searchCache, indexName, searchParam, productList);
             return searchApiResult.setData(productList);
         } catch (Exception e) {
             publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_NEW_SHELVE.getEventName(), EventReportEnum.SEARCHCONTROLLER_NEW_SHELVE.getFunctionName(),
@@ -177,6 +162,7 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
         return this.transforToJSONArray(productListWithPricePlan);
     }
+
     private JSONArray transforToJSONArray(List<Map<String, Object>> productListWithPricePlan) {
         JSONArray results = new JSONArray();
         for (Map<String, Object> productWithPricePlan : productListWithPricePlan) {
@@ -186,6 +172,7 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
     }
     @Override
+    @SearchCacheAble(cacheName = "WEB_BATCH_NEW_SHELVE",cacheInMinute = 30,cacheType = CacheType.SEARCH_REDIS,excludeParams={ "uid","udid"})
     public SearchApiResult webNewShelveBatchProductList(Map<String, String> paramMap) {
         try {
             logger.info("[func=webNewShelvesProductList][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());