yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 057b7b1ff0e4b8248c34db6f9134e7b0b9e9f1cd 1 parent 803d1b5c

添加缓存，聚合优化

Inline Side-by-side
Showing 5 changed files with 44 additions and 189 deletions
... ... @@ -31,6 +31,28 @@ public class AggCommonHelper {
private static final Logger logger = LoggerFactory.getLogger(AggCommonHelper.class);
/**
* 从TopHits中获取_source
*
* @param topHits
* @return
*/
public static List<Map<String, Object>> getTopHitResultsWithScore(TopHits topHits) {
List<Map<String, Object>> topHitList = new ArrayList<>();
if (topHits == null) {
return topHitList;
}
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
topHitList.add(source);
}
return topHitList;
}
public static List<Integer> getIdsFromAggMaps(Map<String, Aggregation> aggMaps, String aggName) {
if (!aggMaps.containsKey(aggName)) {
return new ArrayList<>();
... ... @@ -53,7 +75,7 @@ public class AggCommonHelper {
*
* @param simpleFieldAggs
* @param topHitOrder
* @param topHitFieldCount
* @param topHitCount
* @return
*/
public static AbstractAggregationBuilder<?> getTopHitAggregation(List<SimpleFieldAgg> simpleFieldAggs, String topHitOrder, int topHitCount) {
... ... @@ -85,12 +107,6 @@ public class AggCommonHelper {
/**
* 获取聚合出来的商品列表，并按特定顺序截取
*
* @param aggregation
* @param viewNum
* @param sortField
* @param sortOrder
* @return
*/
public static List<Map<String, Object>> getTopHitList(Map<String, Aggregation> aggMaps, List<SimpleFieldAgg> simpleFieldAggs, String topHitOrder, int totalCount) {
List<Map<String, Object>> results = new ArrayList<Map<String, Object>>();
... ... @@ -99,6 +115,7 @@ public class AggCommonHelper {
return results;
}
private static void getTopHitResults(List<Map<String, Object>> results, Map<String, Aggregation> aggMaps, List<SimpleFieldAgg> simpleFieldAggs, int index) {
try {
String aggName = simpleFieldAggs.get(index++).getAggName();
... ... @@ -113,15 +130,8 @@ public class AggCommonHelper {
Map<String, Aggregation> aggMap = bucket.getAggregations().getAsMap();
if (aggMap.containsKey("topHit")) {
TopHits topHits = bucket.getAggregations().get("topHit");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
results.add(source);
}
}
List<Map<String, Object>> topHitList = getTopHitResultsWithScore(topHits);
results.addAll(topHitList);
} else {
getTopHitResults(results, aggMap, simpleFieldAggs, index);
}
... ... @@ -133,7 +143,7 @@ public class AggCommonHelper {
private static List<Map<String, Object>> sortListBySortField(List<Map<String, Object>> productList, String topHitOrder, int viewNum) {
if (productList == null || productList.isEmpty()) {
return new ArrayList<Map<String, Object>>();
return new ArrayList<>();
}
// 再按照某个字段对商品排序
FieldSortOrder fieldSortOrder = new FieldSortOrder(topHitOrder);
... ... @@ -173,38 +183,27 @@ public class AggCommonHelper {
/**
* 按聚合的key获取聚合出来的商品列表[目前只支持一个field,多重field的话，key不好处理]
*
* @param aggregation
* @param viewNum
* @param sortField
* @param sortOrder
* @return
*/
public static List<KeyTopHitModel> getTopHitListGroupByKey(Map<String, Aggregation> aggMaps, SimpleFieldAgg simpleFieldAgg) {
String aggName = simpleFieldAgg.getAggName();
if (!aggMaps.containsKey(aggName)) {
return new ArrayList<KeyTopHitModel>();
return new ArrayList<>();
}
List<? extends Bucket> bucketList = ((MultiBucketsAggregation) aggMaps.get(aggName)).getBuckets();
if (bucketList.isEmpty()) {
return new ArrayList<KeyTopHitModel>();
return new ArrayList<>();
}
List<KeyTopHitModel> results = new ArrayList<KeyTopHitModel>();
for (Bucket bucket : bucketList) {
String key = bucket.getKeyAsString();
long count = bucket.getDocCount();
List<Map<String, Object>> topHitList = new ArrayList<Map<String, Object>>();
List<Map<String, Object>> topHitList = new ArrayList<>();
Map<String, Aggregation> aggMap = bucket.getAggregations().getAsMap();
if (aggMap.containsKey("topHit")) {
TopHits topHits = bucket.getAggregations().get("topHit");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
topHitList.add(source);
}
}
List<Map<String, Object>> tempTopHitList = getTopHitResultsWithScore(topHits);
topHitList.addAll(tempTopHitList);
}
KeyTopHitModel keyTopHitModel = new KeyTopHitModel(key, count, topHitList);
results.add(keyTopHitModel);
... ...
package com.yoho.search.service.helper;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.CollectionUtils;
import com.yoho.search.service.base.index.ProductIndexBaseService;
@Component
public class AggProductListHelper {
@Autowired
private ProductIndexBaseService productIndexBaseService;
/**
* 获取聚合出来的商品列表
*
* @param aggregation
* @return
*/
public List<Map<String, Object>> getProductListFromAggregation(final MultiBucketsAggregation aggregation) {
Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
List<Map<String, Object>> productEsSourceList = new ArrayList<Map<String, Object>>();
while (itAgg.hasNext()) {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
TopHits topHits = lt.getAggregations().get("product");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
productEsSourceList.add(hit.getSource());
}
}
}
}
if (productEsSourceList == null || productEsSourceList.isEmpty()) {
return new ArrayList<Map<String, Object>>();
}
return productIndexBaseService.getProductListWithPricePlan(productEsSourceList);
}
/**
* 获取聚合出来的商品列表，并按特定顺序截取
*
* @param aggregation
* @param viewNum
* @param sortField
* @param sortOrder
* @return
*/
public List<Map<String, Object>> getProductListFromAggregation(final MultiBucketsAggregation aggregation, int viewNum, String sortField, SortOrder sortOrder) {
Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
// 获取品牌聚合出来的商品
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
while (itAgg.hasNext()) {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
TopHits topHits = lt.getAggregations().get("product");
if (topHits == null) {
continue;
}
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
dataList.add(source);
}
}
}
dataList = this.sortListBySortField(dataList, viewNum, sortField, sortOrder);
return productIndexBaseService.getProductListWithPricePlan(dataList);
}
private List<Map<String, Object>> sortListBySortField(List<Map<String, Object>> productList, int viewNum, String orderField, SortOrder sortOrder) {
if (productList == null || productList.isEmpty()) {
return new ArrayList<Map<String, Object>>();
}
// 再按照某个字段对商品排序
boolean isDesc = sortOrder == SortOrder.DESC ? true : false;
Collections.sort(productList, new Comparator<Map<String, Object>>() {
public int compare(Map<String, Object> o1, Map<String, Object> o2) {
double value1 = getDouble(o1.get(orderField));
double value2 = getDouble(o2.get(orderField));
if (isDesc) {
return value1 > value2 ? -1 : value1 < value2 ? 1 : 0;
} else {
return value1 > value2 ? 1 : value1 < value2 ? -1 : 0;
}
}
});
if (productList.size() > viewNum) {
productList = CollectionUtils.safeSubList(productList, 0, viewNum);
}
return productList;
}
private double getDouble(Object value) {
try {
return Double.parseDouble(value.toString());
} catch (Exception e) {
return 0;
}
}
}
... ... @@ -14,6 +14,7 @@ import com.yoho.search.models.SearchApiResult;
import com.yoho.search.models.SearchSort;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.index.ProductIndexBaseService;
import com.yoho.search.service.helper.AggCommonHelper;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.helper.SearchParamHelper;
import com.yoho.search.service.helper.SearchSortHelper;
... ... @@ -207,7 +208,7 @@ public class AggProductListService implements ApplicationEventPublisherAware {
* 【父聚合的字段名称】
* @param firstAggBucketCount
* 【父聚合的桶大小】
* @param aggOrderSearchSort
* @param aggSort
* 【聚合排序规则，即按什么排序推荐】
* @param secondSearchSort
* 【次要排序条件，即最终商品按什么排序】
... ... @@ -291,15 +292,7 @@ public class AggProductListService implements ApplicationEventPublisherAware {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
TopHits topHits = lt.getAggregations().get("product");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
productList.add(source);
}
}
productList.addAll(AggCommonHelper.getTopHitResultsWithScore(topHits));
}
}
return productList;
... ...
... ... @@ -15,6 +15,7 @@ import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.service.base.index.BrandIndexBaseService;
import com.yoho.search.service.base.index.ShopsIndexBaseService;
import com.yoho.search.service.helper.AggCommonHelper;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.IShopListService;
import org.apache.commons.lang.StringUtils;
... ... @@ -151,20 +152,13 @@ public class ShopListServiceImpl implements IShopListService {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
TopHits topHits = lt.getAggregations().get("product");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
productSourceList.add(source);
}
}
productSourceList.addAll(AggCommonHelper.getTopHitResultsWithScore(topHits));
}
}
if (productSourceList == null || productSourceList.isEmpty()) {
return new ArrayList<Map<String, Object>>();
}
List<Integer> yohoShopIds = new ArrayList<Integer>();
List<Integer> globalBrandIds = new ArrayList<Integer>();
... ...
... ... @@ -6,8 +6,10 @@ import com.yoho.search.base.utils.CollectionUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.CacheType;
import com.yoho.search.common.cache.SearchCacheFactory;
import com.yoho.search.common.cache.SearchCacheMatchLogger;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.common.cache.model.SearchCache;
import com.yoho.search.common.utils.SearchApiResultUtils;
import com.yoho.search.core.es.model.SearchParam;
... ... @@ -62,17 +64,12 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
@Autowired
private SearchServiceHelper searchServiceHelper;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private SearchCacheFactory searchCacheFactory;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private WebProductIndexBaseService webProductIndexBaseService;
@Autowired
private ProductListSortService productListSortService;
private SearchCache searchCache;
private ApplicationEventPublisher publisher;
@Override
... ... @@ -80,12 +77,8 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
this.publisher = applicationEventPublisher;
}
@PostConstruct
void init() {
searchCache = searchCacheFactory.getBrandRelatedCache();
}
@Override
@SearchCacheAble(cacheName = "WEB_NEW_SHELVE",cacheInMinute = 30,cacheType = CacheType.SEARCH_REDIS,excludeParams={ "uid","udid"})
public SearchApiResult webNewShelveProductList(Map<String, String> paramMap) {
try {
logger.info("[func=aggProductsByBrandId][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
... ... @@ -130,14 +123,7 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
// 5、构造返回结果
SearchApiResult searchApiResult = new SearchApiResult().setMessage("new-shelve list");
// 6、先从缓存中获取,如果能取到，则直接返回
JSONArray jsonArray = searchCacheService.getJSONArrayFromCache(this.searchCache, ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (jsonArray != null) {
SearchCacheMatchLogger.doSearchCacheMatchLog("/new-shelve.json", paramMap);
return searchApiResult.setData(jsonArray);
}
// 7、执行搜索,并构造返回结果
// 6、执行搜索,并构造返回结果
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null) {
... ... @@ -148,7 +134,6 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
return searchApiResult.setData("");
}
JSONArray productList = this.getNewShelveResponseMap(((MultiBucketsAggregation) aggMaps.get("brandAgg")));
searchCacheService.addJSONArrayToCache(this.searchCache, indexName, searchParam, productList);
return searchApiResult.setData(productList);
} catch (Exception e) {
publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_NEW_SHELVE.getEventName(), EventReportEnum.SEARCHCONTROLLER_NEW_SHELVE.getFunctionName(),
... ... @@ -177,6 +162,7 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
return this.transforToJSONArray(productListWithPricePlan);
}
private JSONArray transforToJSONArray(List<Map<String, Object>> productListWithPricePlan) {
JSONArray results = new JSONArray();
for (Map<String, Object> productWithPricePlan : productListWithPricePlan) {
... ... @@ -186,6 +172,7 @@ public class WebNewShelveServiceImpl implements IWebNewShelveService, Applicatio
}
@Override
@SearchCacheAble(cacheName = "WEB_BATCH_NEW_SHELVE",cacheInMinute = 30,cacheType = CacheType.SEARCH_REDIS,excludeParams={ "uid","udid"})
public SearchApiResult webNewShelveBatchProductList(Map<String, String> paramMap) {
try {
logger.info("[func=webNewShelvesProductList][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
... ...