yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 9841db78f1464182e745a84a298d8269c7d5d936 1 parent 0ae13c26

SimilarProductServiceImpl优化

Inline Side-by-side
Showing 1 changed file with 75 additions and 227 deletions
package com.yoho.search.service.service.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.SearchCacheFactory;
import com.yoho.search.common.cache.model.SearchCache;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.models.YohoFilterFunctionBuilders;
import com.yoho.search.service.aggregations.common.SimpleFieldAgg;
import com.yoho.search.service.base.SearchCacheService;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.service.base.index.ProductIndexBaseService;
import com.yoho.search.service.helper.AggCommonHelper;
import com.yoho.search.service.service.ISimilarProductService;
/**
* Created by wangnan on 2017/4/24.
*/
@Service
public class SimilarProductServiceImpl extends BaseService implements ISimilarProductService {
public class SimilarProductServiceImpl implements ISimilarProductService {
private static final String PRODUCT_SKN = "product_skn";
private static final String SMALL_SORT_AGG_NAME = "smallSortAgg";
... ... @@ -59,61 +41,34 @@ public class SimilarProductServiceImpl extends BaseService implements ISimilarPr
private static final String SMALL_SORT_IDS = "smallSortIds";
private static final int SKN_LENGTH_LIMIT = 100;
private static final String similarProductOrder = "salesNum:desc";
@Autowired
private ProductIndexBaseService productIndexBaseService;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private SearchCacheFactory searchCacheFactory;
private SearchCache searchLikeSearchCache;
@PostConstruct
void init() {
searchLikeSearchCache = searchCacheFactory.getSimilarProductSearchCache();
}
@Override
public SearchApiResult similarProductList(Map<String, String> paramMap) {
// 1.检测分页参数
int pageSize = this.getPageSize(paramMap);
int page = this.getPage(paramMap);
if (page < 1 || pageSize < 0) {
return new SearchApiResult().setCode(400).setMessage("分页参数不合法");
}
// 2 获取入参SKN列表
// 1、 获取入参SKN列表
List<String> productSknList = this.stringToList(paramMap.getOrDefault(PRODUCT_SKN, ""));
// 3.获取skn的品类列表和品牌列表
JSONObject sortAndBrandInfo = this.querySknSortAndBrand(productSknList);
// 4.再根据品类和品牌获取推荐SKN列表
List<String> recommendSknList = this.recommendSknListBySortAndBrandInfo(sortAndBrandInfo);
// 5.给推荐的skn加分，查询商品列表
SearchApiResult searchApiResult = this.searchProductList(paramMap, recommendSknList, page, pageSize);
return searchApiResult;
}
private List<String> stringToList(String source) {
if (source.length() > SKN_LENGTH_LIMIT) {
source = source.substring(0, SKN_LENGTH_LIMIT);
if (productSknList == null || productSknList.isEmpty()) {
return new SearchApiResult().setCode(400).setMessage("请传product_skn参数");
}
List<String> result = new ArrayList<>();
if (StringUtils.isBlank(source)) {
return result;
// 2.检测数量
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 30 : Integer.parseInt(paramMap.get("viewNum"));
if (pageSize > 50) {
pageSize = 50;
}
for (String part : source.split(",")) {
result.add(part);
}
return result;
// 3.获取skn的品类列表和品牌列表
Map<String, List<Integer>> sortAndBrandInfo = this.querySknSortAndBrand(productSknList);
// 4.根据品牌和品类，随机推荐X个商品
SearchApiResult searchApiResult = this.searchProductList(productSknList, sortAndBrandInfo, pageSize);
return searchApiResult;
}
/**
* 获取skn的品类列表和品牌列表
*/
private JSONObject querySknSortAndBrand(List<String> productSknList) {
private Map<String, List<Integer>> querySknSortAndBrand(List<String> productSknList) {
SearchParam searchParam = new SearchParam();
// 1、设置过滤条件
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
... ... @@ -129,197 +84,90 @@ public class SimilarProductServiceImpl extends BaseService implements ISimilarPr
// 3、设置分页
searchParam.setOffset(0);
searchParam.setSize(0);
// 4、先从缓存中获取,如果能取到，则直接返回
JSONObject sortAndBrandJSONObject = searchCacheService.getJSONObjectFromCache(searchLikeSearchCache, ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (sortAndBrandJSONObject != null) {
return sortAndBrandJSONObject;
}
// 5、执行搜索
// 4、执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null) {
return new JSONObject();
return new HashMap<String, List<Integer>>();
}
Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
// 6、从聚合结果中获取品牌和品类id
// 5、从聚合结果中获取品牌和品类id
List<Integer> smallSortIds = AggCommonHelper.getIdsFromAggMaps(aggMaps, SMALL_SORT_AGG_NAME);
List<Integer> brandIds = AggCommonHelper.getIdsFromAggMaps(aggMaps, BRAND_AGG_NAME);
// 7、构造返回结果并加入缓存
sortAndBrandJSONObject = new JSONObject();
sortAndBrandJSONObject.put(BRAND_IDS, brandIds);
sortAndBrandJSONObject.put(SMALL_SORT_IDS, smallSortIds);
searchCacheService.addJSONObjectToCache(searchLikeSearchCache, ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam, sortAndBrandJSONObject);
return sortAndBrandJSONObject;
// 6、构造返回结果并加入缓存
Map<String, List<Integer>> results = new HashMap<String, List<Integer>>();
results.put(BRAND_IDS, brandIds);
results.put(SMALL_SORT_IDS, smallSortIds);
return results;
}
/**
* 根据skn参数获取推荐出来的skn列表
* 根据品牌和品类随机推荐商品
*/
private List<String> recommendSknListBySortAndBrandInfo(JSONObject sortAndBrandInfo) {
// 1、获取品牌id和品类id
if (sortAndBrandInfo == null || sortAndBrandInfo.isEmpty()) {
return new ArrayList<>();
}
JSONArray brandIdJSONArray = sortAndBrandInfo.getJSONArray(BRAND_IDS);
JSONArray smallSortIdJSONArray = sortAndBrandInfo.getJSONArray(SMALL_SORT_IDS);
List<Integer> brandIds = this.jsonArrayToList(brandIdJSONArray, Integer.class);
List<Integer> smallSortIds = this.jsonArrayToList(smallSortIdJSONArray, Integer.class);
if (brandIds.isEmpty() && smallSortIds.isEmpty()) {
return new ArrayList<>();
}
// 2、构造filter
private SearchApiResult searchProductList(List<String> productSknList, Map<String, List<Integer>> sortAndBrandInfo, int pageSize) {
SearchParam searchParam = new SearchParam();
BoolQueryBuilder boolFilter = this.getDefaultBoolQueryBuilder();
BoolQueryBuilder brandAndSortFilter = new BoolQueryBuilder();
if (!brandIds.isEmpty()) {
brandAndSortFilter.should(QueryBuilders.termsQuery(ProductIndexEsField.brandId, brandIds));
}
if (!smallSortIds.isEmpty()) {
brandAndSortFilter.should(QueryBuilders.termsQuery(ProductIndexEsField.smallSortId, smallSortIds));
}
boolFilter.must(brandAndSortFilter);
searchParam.setFiter(boolFilter);
// 3、构造query
searchParam.setQuery(QueryBuilders.matchAllQuery());
// 4、设置聚合条件[品类+品牌聚合]
List<SimpleFieldAgg> simpleFieldAggs = new ArrayList<SimpleFieldAgg>();
simpleFieldAggs.add(new SimpleFieldAgg(SMALL_SORT_AGG_NAME, ProductIndexEsField.smallSortId, 100));
simpleFieldAggs.add(new SimpleFieldAgg(BRAND_AGG_NAME, ProductIndexEsField.brandId, 100));
AbstractAggregationBuilder<?> smallSortAndBrandAgg = AggCommonHelper.getTopHitAggregation(simpleFieldAggs, similarProductOrder, 2);
searchParam.setAggregationBuilders(Arrays.asList(smallSortAndBrandAgg));
// 5、设置分页
// 1.构造filter
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
searchParam.setFiter(boolQueryBuilder);
// 1.1默认参数
boolQueryBuilder.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.isSeckill, "Y"));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(ProductIndexEsField.isFobbiden, 1));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(ProductIndexEsField.attribute, 2));
boolQueryBuilder.must(QueryBuilders.termQuery(ProductIndexEsField.status, 1));
boolQueryBuilder.must(QueryBuilders.termQuery(ProductIndexEsField.isOutlets, 2));
boolQueryBuilder.must(QueryBuilders.rangeQuery(ProductIndexEsField.storageNum).gte(1));
boolQueryBuilder.must(QueryBuilders.rangeQuery(ProductIndexEsField.breakSizePercent).lt(50));// 非断码
// 1.2不推参数中的skn
boolQueryBuilder.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, productSknList));
// 1.3过滤品牌和品类
BoolQueryBuilder shouldFilter = new BoolQueryBuilder();
List<Integer> brandIds = sortAndBrandInfo.get(BRAND_IDS);
List<Integer> smallSortIds = sortAndBrandInfo.get(SMALL_SORT_IDS);
if (brandIds != null && !brandIds.isEmpty()) {
shouldFilter.should(QueryBuilders.termsQuery(ProductIndexEsField.brandId, brandIds));
}
if (smallSortIds != null && !smallSortIds.isEmpty()) {
shouldFilter.should(QueryBuilders.termsQuery(ProductIndexEsField.smallSortId, smallSortIds));
}
if (shouldFilter.hasClauses()) {
boolQueryBuilder.must(shouldFilter);
}
// 2.构建分页参数
searchParam.setOffset(0);
searchParam.setSize(0);
// 6、先从缓存中获取,如果能取到，则直接返回
JSONArray recommendedSknJSONArray = searchCacheService.getJSONArrayFromCache(searchLikeSearchCache, ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (recommendedSknJSONArray != null) {
return this.jsonArrayToList(recommendedSknJSONArray, String.class);
}
// 7、执行搜索
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null) {
return new ArrayList<>();
}
// 8、获取搜索结果
List<Map<String, Object>> productList = AggCommonHelper.getTopHitList(searchResult.getAggMaps(), simpleFieldAggs, similarProductOrder, 100);
// 9、构造返回结果
recommendedSknJSONArray = new JSONArray();
for (Map<String, Object> map : productList) {
recommendedSknJSONArray.add(MapUtils.getString(map, ProductIndexEsField.productSkn, ""));
}
// 10、随机打乱即可
List<String> recommendedSknList = this.jsonArrayToList(recommendedSknJSONArray, String.class);
Collections.shuffle(recommendedSknList);
return recommendedSknList;
}
/**
* 给推荐的skn加分，查询商品列表
*/
private SearchApiResult searchProductList(Map<String, String> paramMap, List<String> recommendSknList, int page, int pageSize) {
// 1.构造搜索参数
SearchParam searchParam = new SearchParam();
searchParam.setFiter(this.getDefaultBoolQueryBuilder());
searchParam.setQuery(this.builderProductQueryBuilder(paramMap, recommendSknList));
searchParam.setAggregationBuilders(null);
searchParam.setOffset((page - 1) * pageSize);
searchParam.setSize(pageSize);
List<SortBuilder<?>> sortBuilders = new ArrayList<>();
sortBuilders.add(SortBuilders.scoreSort().order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.salesNum).order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.firstShelveTime).order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.id).order(SortOrder.DESC));
// 3.随机排序
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
sortBuilders.add(SortBuilders.scriptSort(new Script("Math.random()"), ScriptSortType.NUMBER).order(SortOrder.ASC));
searchParam.setSortBuilders(sortBuilders);
// 2.从缓存中获取数据
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
JSONObject cacheObject = searchCacheService.getJSONObjectFromCache(searchLikeSearchCache, indexName, searchParam);
if (cacheObject != null) {
return new SearchApiResult().setData(cacheObject);
}
// 3.查询ES
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
// 4.查询ES
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult == null) {
return new SearchApiResult().setCode(500).setMessage("exception");
}
// 4.构造返回结果
// 5.构造返回结果
JSONObject dataMap = new JSONObject();
dataMap.put("total", searchResult.getTotal());
dataMap.put("page", searchResult.getPage());
dataMap.put("page_size", searchParam.getSize());
dataMap.put("page_total", searchResult.getTotalPage());
dataMap.put("total", searchResult.getResultList().size());
dataMap.put("page", 1);
dataMap.put("page_size", pageSize);
dataMap.put("page_total", 1);
dataMap.put("product_list", productIndexBaseService.getProductListWithPricePlan(searchResult.getResultList()));
// 5.将结果存进缓存
searchCacheService.addJSONObjectToCache(searchLikeSearchCache, indexName, searchParam, dataMap);
return new SearchApiResult().setData(dataMap);
}
/**
* 过滤规则
*/
private BoolQueryBuilder getDefaultBoolQueryBuilder() {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.isSeckill, "Y"));
boolFilter.mustNot(QueryBuilders.termQuery(ProductIndexEsField.isFobbiden, 1));
boolFilter.mustNot(QueryBuilders.termQuery(ProductIndexEsField.attribute, 2));
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.status, 1));
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.isOutlets, 2));
boolFilter.must(QueryBuilders.rangeQuery(ProductIndexEsField.storageNum).gte(1));
// 非断码
boolFilter.must(QueryBuilders.rangeQuery(ProductIndexEsField.breakSizePercent).lt(50));
return boolFilter;
}
/**
* 分组
*/
private Map<Integer, List<String>> splitProductSknList(List<String> recommendedSknList, int maxCountPerGroup) {
int maxSize = recommendedSknList.size();
int groupSize = maxSize / maxCountPerGroup;
if (maxSize % maxCountPerGroup > 0) {
groupSize = groupSize + 1;
}
Map<Integer, List<String>> result = new HashMap<>();
for (int i = 0; i < groupSize; i++) {
int fromIndex = i * maxCountPerGroup;
int toIndex = (i + 1) * maxCountPerGroup;
result.put(i, recommendedSknList.subList(fromIndex, toIndex > maxSize ? maxSize : toIndex));
}
return result;
}
private QueryBuilder builderProductQueryBuilder(Map<String, String> paramMap, List<String> recommendedSknList) {
YohoFilterFunctionBuilders filterFunctionBuilders = new YohoFilterFunctionBuilders();
// 如果参数中包含firstProductSkn，则放在第一个
if (paramMap.containsKey(SearchRequestParams.PARAM_SEARCH_FIRST_PRODUCRSKN)) {
filterFunctionBuilders.add(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, paramMap.getOrDefault(SearchRequestParams.PARAM_SEARCH_FIRST_PRODUCRSKN, "")),
ScoreFunctionBuilders.weightFactorFunction(100000));
private List<String> stringToList(String source) {
if (source.length() > SKN_LENGTH_LIMIT) {
source = source.substring(0, SKN_LENGTH_LIMIT);
}
// 针对推荐出来的SKN做加分
if (recommendedSknList != null && !recommendedSknList.isEmpty()) {
Map<Integer, List<String>> recommendSknMap = this.splitProductSknList(recommendedSknList, 2);
float currentGroupScore = 1000;
for (Map.Entry<Integer, List<String>> entry : recommendSknMap.entrySet()) {
filterFunctionBuilders.add(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, entry.getValue()),
ScoreFunctionBuilders.weightFactorFunction(currentGroupScore));
currentGroupScore = currentGroupScore - 10;
}
List<String> result = new ArrayList<>();
if (StringUtils.isBlank(source)) {
return result;
}
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.matchAllQuery(), filterFunctionBuilders.getFilterFunctionBuilders());
functionScoreQueryBuilder.boostMode(CombineFunction.MULTIPLY);
return functionScoreQueryBuilder;
}
private <T> List<T> jsonArrayToList(JSONArray jsonArray, Class<T> clazz) {
if (jsonArray == null) {
return new ArrayList<>();
for (String part : source.split(",")) {
result.add(part);
}
return JSON.parseArray(jsonArray.toJSONString(), clazz);
return result;
}
}
... ...