yoho-search / yoho-search-service · Commits

Go to a project

Authored by 胡古飞
Commit 843c352e93379c810604af6a7b1c38bcb7909f9e 2 parents 391966f2 6eaf79db

Merge branch 'master' into wn_robot_return_all_field

Inline Side-by-side
Showing 28 changed files with 1609 additions and 913 deletions
package com.yoho.search.service.aggregations.impls;
import com.yoho.search.core.es.agg.AbstractAggregation;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* 单个字段进行聚合的抽象类
*/
public abstract class AbstractSingleFieldAggregation extends AbstractAggregation {
private int count;
public AbstractSingleFieldAggregation(int count) {
this.count = count;
}
@Override
public AbstractAggregationBuilder getBuilder() {
return AggregationBuilders.terms(aggName()).field(getField()).size(count);
}
@Override
public Object getAggregationResponseMap(Map<String, Aggregation> aggMaps) {
List<String> valueList = new ArrayList<String>();
MultiBucketsAggregation aggregation = this.getAggregation(aggMaps);
if (aggregation == null) {
return valueList;
}
Iterator<? extends Bucket> itSizeAgg = aggregation.getBuckets().iterator();
while (itSizeAgg.hasNext()) {
Bucket ltSize = itSizeAgg.next();
valueList.add(ltSize.getKeyAsString());
}
return valueList;
}
abstract protected String getField();
}
... ...
package com.yoho.search.service.aggregations.impls;
public class BrandNameAggregation extends AbstractSingleFieldAggregation {
public BrandNameAggregation(int count) {
super(count);
}
@Override
public String aggName() {
return "brandNameAgg";
}
@Override
protected String getField() {
return "brandName.brandName_keyword";
}
}
... ...
package com.yoho.search.service.aggregations.impls;
public class SmallSortNameAggregation extends AbstractSingleFieldAggregation {
public SmallSortNameAggregation(int count) {
super(count);
}
@Override
public String aggName() {
return "smallSortNameAgg";
}
@Override
protected String getField() {
return "smallSort.smallSort_keyword";
}
}
... ...
package com.yoho.search.service.aggregations.impls;
public class StyleNameAggregation extends AbstractSingleFieldAggregation {
public StyleNameAggregation(int count) {
super(count);
}
@Override
public String aggName() {
return "styleNameAgg";
}
@Override
protected String getField() {
return "style";
}
}
... ...
package com.yoho.search.service.restapi;
import com.yoho.search.service.downgrade.aop.DownGradeAble;
import com.yoho.search.service.servicenew.ISearchRecommendService;
import com.yoho.search.service.servicenew.ISuggestService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.vo.SearchApiResult;
... ... @@ -20,6 +21,9 @@ public class SuggestController {
@Autowired
private ISuggestService suggestService;
@Autowired
private ISearchRecommendService searchRecommendService;
/**
* 搜索建议接口
*
... ... @@ -34,7 +38,7 @@ public class SuggestController {
}
/**
* 搜索建议接口
* 搜索提示接口 提供给内部使用
*
* @return
*/
... ... @@ -42,6 +46,18 @@ public class SuggestController {
@ResponseBody
public SearchApiResult suggestConversion(HttpServletRequest request) {
Map<String, String> paramMap = HttpServletRequestUtils.transParamType(request);
return suggestService.suggestConversion(paramMap);
return searchRecommendService.suggestConversionList(paramMap);
}
/**
* 搜索建议词接口 提供给内部使用
*
* @return
*/
@RequestMapping(method = RequestMethod.GET, value = "tools/suggestList")
@ResponseBody
public SearchApiResult suggestList(HttpServletRequest request) {
Map<String, String> paramMap = HttpServletRequestUtils.transParamType(request);
return suggestService.suggestList(paramMap);
}
}
... ...
... ... @@ -2,6 +2,7 @@ package com.yoho.search.service.restapi;
import com.yoho.search.service.cache.LocalCacheService;
import com.yoho.search.service.personalized.PersonalVectorFeatureSearch;
import com.yoho.search.service.service.SearchDynamicConfigService;
import com.yoho.search.service.service.SearchKeyWordService;
import com.yoho.search.service.servicenew.IProductListService;
import com.yoho.search.service.vo.KeyWordWithCount;
... ... @@ -9,12 +10,14 @@ import com.yoho.search.service.vo.SearchApiResult;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.util.Assert;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
... ... @@ -31,6 +34,9 @@ public class ToolsController {
@Autowired
private IProductListService productListService;
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
/**
* 获取热搜词结果
*
... ... @@ -104,6 +110,7 @@ public class ToolsController {
/**
* 删除redis上的key
*
* @param redisKey
* @return
*/
... ... @@ -141,6 +148,23 @@ public class ToolsController {
return new SearchApiResult().setMessage("清除本地缓存成功");
}
@RequestMapping(value = "/dynamicParameterValue")
@ResponseBody
public Map<String, Object> dynamicParameterValue(String key) {
Map<String, Object> rtnMap = new HashMap<String, Object>();
try {
Assert.notNull(key);
rtnMap.put("code", 200);
rtnMap.put("value", searchDynamicConfigService.getDynamicParameterValue(key));
} catch (Exception e) {
rtnMap.put("code", 400);
rtnMap.put("msg", e.getMessage());
}
return rtnMap;
}
@RequestMapping(method = RequestMethod.GET, value = "/calVectorFeature")
@ResponseBody
public SearchApiResult calVectorFeature(@RequestParam String uid, @RequestParam String skns, @RequestParam String version) throws Exception {
... ...
... ... @@ -114,4 +114,11 @@ public class SearchDynamicConfigService {
public boolean isSearchSuggestionFromConversionOpen(){
return configReader.getBoolean("search.suggestion.tips.conversion.open", true);
}
/**
* 关于API查看动态参数值
*/
public String getDynamicParameterValue(String key){
return configReader.getString(key, "DEFAULT");
}
}
... ...
... ... @@ -183,6 +183,14 @@ public class SearchKeyWordService {
}
}
public Double getKeywordCount(String redisKeyTemplate, String queryWord){
try {
return yhNoSyncZSetOperations.score(RedisKeys.getRedisKey4Yesterday(redisKeyTemplate), queryWord);
}catch (Exception e){
return null;
}
}
// 获取【热搜】toplist
public Map<String, Object> getHotkeyWords(int limit, boolean isReturnTodayRecords) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, isReturnTodayRecords);
... ...
... ... @@ -42,9 +42,9 @@ public class FunctionScoreSearchHelper {
// 普通个性化的时间维度
private final int oneDaySecondCount = 24 * 60 * 60;
private FirstShelveTimeScore commonFirstShelveTimeScore = new FirstShelveTimeScore(90, 30, 60);
private FirstShelveTimeScore commonFirstShelveTimeScore = new FirstShelveTimeScore(60, 30, 30);
// 新品到着的个性化时间维度
private FirstShelveTimeScore newRecShelveTimeScore = new FirstShelveTimeScore(30, 10, 20);
private FirstShelveTimeScore newRecShelveTimeScore = new FirstShelveTimeScore(14, 7, 7);
private final float globalWeight = 0.50f;
private WeightBuilder genWeightFactorBuilder(float factor) {
... ... @@ -73,10 +73,6 @@ public class FunctionScoreSearchHelper {
functionScoreQueryBuilder.add(physicalChannelScore.getQueryBuilder(), ScoreFunctionBuilders.weightFactorFunction(physicalChannelScore.getWeight()));
}
}
// 针对屏蔽的品牌降分【目前没用】
if (searchCommonHelper.isNeedDeScoreBrandSearch(paramMap)) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("isForbiddenSortBrand", "1"), ScoreFunctionBuilders.weightFactorFunction(0));
}
functionScoreQueryBuilder.boostMode(CombineFunction.MULT);
return functionScoreQueryBuilder;
}
... ...
package com.yoho.search.service.service.helper;
import java.util.Map;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.SearchPageIdDefine;
import com.yoho.search.service.service.SearchDynamicConfigService;
import com.yoho.search.service.utils.SearchRequestParams;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
... ... @@ -9,10 +12,9 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.service.service.SearchDynamicConfigService;
import com.yoho.search.service.utils.SearchRequestParams;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Component
public class SearchCommonHelper {
... ... @@ -80,36 +82,6 @@ public class SearchCommonHelper {
}
/**
* 是否需要对品牌降分
*
* @param paramMap
* @return
*/
public boolean isNeedDeScoreBrandSearch(Map<String, String> paramMap) {
// 如果品牌降分总开关未开启，则直接返回
if (!dynamicConfig.deScoreBrandOpen()) {
return false;
}
// 如果是按售价或者折扣排序或品牌也或店铺页,则降分不生效
String sortField = paramMap.get("order");
if (StringUtils.isNotBlank(sortField)) {
if (sortField.contains("sales_price")) {
return false;
}
if (sortField.contains("discount")) {
return false;
}
if (StringUtils.isNotBlank(paramMap.get("shop")) || StringUtils.isNotBlank(paramMap.get("brand"))) {
return false;
}
}
if ("Y".equals(paramMap.get(SearchRequestParams.PARAM_SEARCH_GLOBAL_DESCORE_BRAND))) {
return true;
}
return false;
}
/**
* 是否需要对频道降分
*
* @param paramMap
... ... @@ -187,6 +159,39 @@ public class SearchCommonHelper {
}
/**
* 从搜索的关键词中提取出skn信息
*
* @param keyword
* @return
*/
public List<Integer> getQuerySknList(String keyword) {
List<Integer> sknList = new ArrayList<>();
try {
String[] keys = null;
if (keyword.contains(" ")) {
keys = keyword.split(" ");
} else if (keyword.contains(",")) {
keys = keyword.split(",");
} else if (keyword.contains("+")) {
keys = keyword.split("\\+");
}
if(keys != null && keys.length > 0){
for(String key : keys){
if (CharUtils.isNumeric(key) && key.length() >= 7) {
sknList.add(Integer.valueOf(key));
}
}
}
return sknList;
} catch (Exception e) {
return sknList;
}
}
/**
* 是否是新品到着页
*
* @param paramMap
... ... @@ -194,7 +199,7 @@ public class SearchCommonHelper {
*/
public boolean isNewRecPage(Map<String, String> paramMap) {
String pageId = paramMap.get("pageId");
if (StringUtils.isBlank(pageId) || !pageId.equals("4")) {
if (StringUtils.isBlank(pageId) || !pageId.equals(SearchPageIdDefine.PAGE_ID_NEW)) {
return false;
}
return true;
... ...
... ... @@ -89,7 +89,6 @@ public class SearchLikeHelper {
searchParam.setQuery(QueryBuilders.matchAllQuery());
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termsQuery("productSkn", productSkn));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
searchParam.setFiter(boolFilter);
searchParam.setAggregationBuilders(null);
searchParam.setPage(0);
... ... @@ -159,10 +158,10 @@ public class SearchLikeHelper {
* @param productSkn
* @return
*/
public BoolQueryBuilder genDefaultQueryBuilder(List<String> notProductSkns) {
public BoolQueryBuilder genDefaultQueryBuilder(List<String> notProductSkns,boolean isGlobal) {
isGlobal = false; //暂时写死，5.6再放开
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y"));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
boolFilter.must(QueryBuilders.termQuery("status", 1));
boolFilter.must(QueryBuilders.termQuery("isOutlets", 2));
boolFilter.must(QueryBuilders.termQuery("attribute", 1));
... ... @@ -170,6 +169,11 @@ public class SearchLikeHelper {
if (notProductSkns != null && !notProductSkns.isEmpty()) {
boolFilter.mustNot(QueryBuilders.termsQuery("productSkn", notProductSkns));
}
if(isGlobal){
boolFilter.must(QueryBuilders.termsQuery("isGlobal", "Y"));
}else{
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
}
return boolFilter;
}
... ...
... ... @@ -200,10 +200,13 @@ public class SearchServiceHelper {
}
// 全局降分品牌过滤
if ("Y".equals(paramMap.get(SearchRequestParams.PARAM_SEARCH_GLOBAL_FILTER_BRAND))) {
boolFilter.mustNot(QueryBuilders.termQuery("isForbiddenSortBrand", 1));
// if ("Y".equals(paramMap.get(SearchRequestParams.PARAM_SEARCH_GLOBAL_FILTER_BRAND))) {
// boolFilter.mustNot(QueryBuilders.termQuery("isForbiddenSortBrand", 1));
// }
if ("Y".equals(paramMap.get(SearchRequestParams.PARAM_SEARCH_GLOBAL_FILTER_BRAND)) && StringUtils.isNotBlank(paramMap.get("pageId"))){
boolFilter.mustNot(QueryBuilders.termQuery("forbiddenPageIds",paramMap.get("pageId")));
}
// 店铺
if (paramMap.containsKey(SearchRequestParams.PARAM_SEARCH_SHOP) && StringUtils.isNotBlank(paramMap.get(SearchRequestParams.PARAM_SEARCH_SHOP))
&& !SearchRequestParams.PARAM_SEARCH_SHOP.equals(filterParamName)) {
... ... @@ -736,6 +739,10 @@ public class SearchServiceHelper {
productMap.put(extendedField, map.get(extendedField));
}
}
// 增加小分类名称用于搜索推荐
productMap.put("small_sort_name", map.get("smallSort"));
return productMap;
}
}
... ...
... ... @@ -83,11 +83,15 @@ public class SearchSortHelper {
*/
public String getLegalOrder(Map<String, String> paramMap) {
String sortFields = paramMap.get("order");
if (StringUtils.isBlank(sortFields)) {
boolean isNewRecPage = searchCommonHelper.isNewRecPage(paramMap);
return this.getLegalOrder(sortFields,isNewRecPage);
}
public String getLegalOrder(String order,boolean isNewRecPage) {
if (StringUtils.isBlank(order)) {
return null;
}
boolean isNewRecPage = searchCommonHelper.isNewRecPage(paramMap);
String[] sortFieldArray = sortFields.split(",");
String[] sortFieldArray = order.split(",");
StringBuilder realOrder = new StringBuilder();
for (String sortField : sortFieldArray) {
if (!orderValues.contains(sortField)) {
... ... @@ -148,11 +152,7 @@ public class SearchSortHelper {
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
return true;
}
// 3、需要做降分处理时
if (searchCommonHelper.isNeedDeScoreBrandSearch(paramMap)) {
return true;
}
// 4、传了需要显示第一个SKN的参数过来时
// 3、传了需要显示第一个SKN的参数过来时
if (searchCommonHelper.isFirstProductSknSearch(paramMap)) {
return true;
}
... ...
package com.yoho.search.service.servicenew;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.service.vo.SearchApiResult;
import java.util.Map;
/**
* Created by ginozhang on 2017/3/22.
*/
public interface ISearchRecommendService {
/**
* 根据query词获取term建议和phrase建议。
* 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
* @param searchResult 搜索结果
* @param paramMap 搜索参数
* @return 包括term建议和phrase建议。
*/
JSONObject recommend(SearchApiResult searchResult, Map<String, String> paramMap);
/**
* 根据关键词和时间查询关键词转换关系
* @param paramMap 查询参数
* @return 满足查询要求的转换关系
*/
SearchApiResult suggestConversionList(Map<String, String> paramMap);
}
... ...
package com.yoho.search.service.servicenew;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SuggestApiResult;
... ... @@ -17,17 +16,24 @@ public interface ISuggestService {
public SuggestApiResult suggest(Map<String, String> paramMap);
/**
* 根据query词获取term建议和phrase建议。
* 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
* @param paramMap 搜索参数
* @return 包括term建议和phrase建议。
* 根据关键词查询suggest索引
*
* @param paramMap 查询参数
* @return 满足查询要求的建议词
*/
JSONObject suggestTips(Map<String, String> paramMap);
SearchApiResult suggestList(Map<String, String> paramMap);
/**
* 根据关键词和时间查询关键词转换关系
* @param paramMap 查询参数
* @return 满足查询要求的转换关系
* 到suggest索引进行拼写纠错处理。
* @param keyword 用户输入的关键词
* @return 纠错后的关键词。
*/
String getSpellingCorrectKeyword(String keyword);
/**
* 根据搜索条件判断使用到的suggest索引的count字段名。
* @param paramMap 搜索条件
* @return count字段名
*/
SearchApiResult suggestConversion(Map<String, String> paramMap);
String getCountField(Map<String, String> paramMap);
}
... ...
package com.yoho.search.service.servicenew.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
... ... @@ -22,6 +23,7 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Buck
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
... ... @@ -46,8 +48,8 @@ import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.service.helper.SearchSortHelper;
import com.yoho.search.service.servicenew.IAggProductListService;
import com.yoho.search.service.utils.SearchApiResultUtils;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SearchSort;
@Service
public class AggProductListServiceImpl implements IAggProductListService, ApplicationEventPublisherAware {
... ... @@ -85,230 +87,266 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
aggTypeToEsField.put("sort", "smallSortId");
aggTypeToEsField.put("brand", "brandId");
}
private QueryBuilder builderActivityQueryBuilder(Map<String, String> paramMap) {
// FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.matchAllQuery());
// functionScoreSearchHelper.addActivityPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
// return functionScoreQueryBuilder;
return searchServiceHelper.constructQueryBuilderForProductList(paramMap);
}
/**
* 获取每个桶的商品数量
*
* @param paramMap
* @return
*/
private int getTopHitsBucketSize(Map<String, String> paramMap) {
int topHitsSize = 1;
try {
topHitsSize = Integer.valueOf(paramMap.get("aggTypeSize"));
} catch (Exception e) {
}
if (topHitsSize <= 0) {
topHitsSize = 1;
}
return topHitsSize;
}
private SearchSort genSearchSort(String order) {
if (StringUtils.isBlank(order)) {
return null;
}
order = searchSortHelper.getLegalOrder(order, false);
if (StringUtils.isBlank(order)) {
return null;
}
String sortField = order.split(":")[0];
SortOrder sortOrder = order.split(":")[1].equals("desc") ? SortOrder.DESC : SortOrder.ASC;
return new SearchSort(sortField, sortOrder);
}
/**
* @aggType=brand[聚合类型]
* @brand=1,2,3,4,5,6,7,8,9,10[聚合的值]
* @viewNum=10[总数]
* @aggTypeSize=1|2|3[每种品牌最多的个数]
* @second_order=shelve_time:desc[次要排序条件]
*/
@Override
public SearchApiResult aggProductList(Map<String, String> paramMap) {
try {
logger.info("[func=aggProductList][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
logger.info("[func=aggProductList][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
// 1、参数校验
String aggType = paramMap.get("aggType");// 获取聚合类型
if (StringUtils.isBlank(aggType) || !aggTypeToEsField.containsKey(aggType)) {
return new SearchApiResult().setCode(400).setData("聚合类型非法");
}
if (!paramMap.containsKey(aggType)) {
return new SearchApiResult().setCode(400).setData("请传对应聚合类型的参数");
}
// 1、参数校验
String aggType = paramMap.get("aggType");
if (StringUtils.isBlank(aggType) || !aggTypeToEsField.containsKey(aggType)) {
return new SearchApiResult().setCode(400).setData("聚合类型非法");
}
if (!paramMap.containsKey(aggType)) {
return new SearchApiResult().setCode(400).setData("请传对应聚合类型的参数");
}
// 2、获取第一个聚合的字段
String firstAggField = aggTypeToEsField.get(aggType);
// 2、构造核心参数
String[] values = paramMap.get(aggType).split(",");
int viewNum = (values == null ? 10 : values.length);
String firstAggFiled = aggTypeToEsField.get(aggType);
// 3、获取第一个聚合的桶大小
String[] aggTypeValues = paramMap.get(aggType).split(",");// 获取聚合类型的值
int firstAggBucketCount = (aggTypeValues == null ? 10 : aggTypeValues.length);// 获取聚合的桶大小
// 3、构造查询条件
SearchParam searchParam = new SearchParam();
//paramMap.put("promotion", null);
searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap, null));//活动特定的个性化
searchParam.setQuery(this.builderActivityQueryBuilder(paramMap));
searchParam.setSize(0);
// 4、获取聚合排序条件
String aggOrder = "sales_num:desc";
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
aggOrder = "_score:desc";
}
SearchSort aggSort = this.genSearchSort(aggOrder);
String second_order = paramMap.get("second_order");
// 4、构造聚合条件
final String firstAggName = "firstAgg";
// 5、获取次要排序条件，主要用作顺序打乱
SearchSort secondSearchSort = this.genSearchSort(second_order);
List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>();
// 4.1）构造父聚合:品牌或品类聚合【同时按子聚合的sort字段排序】
TermsBuilder brandAggregationBuilder = AggregationBuilders.terms(firstAggName).field(firstAggFiled).order(Terms.Order.aggregation("sort", false)).size(viewNum);
// 4.2）添加子聚合:取得分最大的值
brandAggregationBuilder.subAggregation(AggregationBuilders.max("sort").field("_score"));
// 4.3）添加孙聚合：取打分最高的一个product
brandAggregationBuilder.subAggregation(AggregationBuilders.topHits("product").addSort(SortBuilders.fieldSort("_score").order(SortOrder.DESC)).setSize(1));
list.add(brandAggregationBuilder);
searchParam.setAggregationBuilders(list);
// 6、获取每个桶下面的商品数量
int topHitsBucketSize = this.getTopHitsBucketSize(paramMap);// 每个桶下面获取的商品数量
// 5、构造返回结果
SearchApiResult searchApiResult = new SearchApiResult().setMessage("agg productList list");
// 7、获取商品总数量
int totalViewNum = StringUtils.isBlank(paramMap.get("viewNum")) ? firstAggBucketCount * topHitsBucketSize : Integer.parseInt(paramMap.get("viewNum"));
// 6、先从缓存中获取,如果能取到，则直接返回
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (jsonObject != null) {
return searchApiResult.setData(jsonObject);
}
// 7、执行搜索,并构造返回结果
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null) {
return searchApiResult.setData("");
}
Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
if (!aggMaps.containsKey(firstAggName)) {
return searchApiResult.setData("");
}
List<Map<String, Object>> productList = this.getProductList(((MultiBucketsAggregation) aggMaps.get(firstAggName)));
// 8、执行查询
JSONObject jsonObject = this.getAggProductListJSONObject(paramMap, firstAggField, firstAggBucketCount, aggSort, topHitsBucketSize, secondSearchSort, totalViewNum);
if (jsonObject == null) {
jsonObject = new JSONObject();
jsonObject.put("total", productList == null ? 0 : productList.size());
jsonObject.put("page", 1);
jsonObject.put("page_size", viewNum);
jsonObject.put("page_total", 1);
jsonObject.put("product_list", productList);
searchCacheService.addJSONObjectToCache(indexName, searchParam, jsonObject);
return searchApiResult.setData(jsonObject);
} catch (Exception e) {
publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_AGG_PRODUCTLIST.getEventName(), EventReportEnum.SEARCHCONTROLLER_AGG_PRODUCTLIST
.getFunctionName(), EventReportEnum.SEARCHCONTROLLER_AGG_PRODUCTLIST.getMoudleName(), "exception", IgnoreSomeException.filterSomeException(e), null));
return SearchApiResultUtils.errorSearchApiResult("aggProductList", paramMap, e);
}
return new SearchApiResult().setMessage("agg productList list").setData(jsonObject);
}
private List<Map<String, Object>> getProductList(final MultiBucketsAggregation aggregation) {
Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
// 获取品牌聚合出来的商品
TopHits topHits;
SearchHits hits;
List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
List<String> sknStr = new ArrayList<String>();
while (itAgg.hasNext()) {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
topHits = lt.getAggregations().get("product");
if (topHits != null) {
hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
sknStr.add("" + hit.getSource().get("productSkn"));
dataList.add(hit.getSource());
}
}
}
/**
* @viewNum=10[总数]
* @aggTypeSize=1|2|3[每种个品牌下面最多的个数]
* @second_order=shelve_time:desc[次要排序条件]
*/
@Override
public SearchApiResult aggProductListByBrand(Map<String, String> paramMap) {
logger.info("[func=aggProductListByBrand][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
// 1、获取商品总数量参数
int totalViewNum = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
// 2、获取第一个聚合的字段
String firstAggField = "brandId";
// 3、获取第一个聚合的桶大小
int firstAggBucketCount = 200 + totalViewNum;
// 4、获取聚合排序条件
String aggOrder = "sales_num:desc";
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
aggOrder = "_score:desc";
}
try {
Map<String, List<Map<String, Object>>> productPricePlanMap = searchServiceHelper.searchProductPricePlan((String[]) sknStr.toArray(new String[sknStr.size()]));
for (Map<String, Object> m : dataList) {
result.add(searchServiceHelper.getProductMapWithPricePlan(m, productPricePlanMap));
}
} catch (Exception e) {
logger.error("[func=aggProductList][Exception={}][begin={}]", e, System.currentTimeMillis());
SearchSort aggOrderSearchSort = this.genSearchSort(aggOrder);
// 5、获取次要排序条件，主要用作顺序打乱
String second_order = paramMap.get("second_order");
SearchSort secondSearchSort = this.genSearchSort(second_order);
// 6、获取每个桶下面的商品数量
int topHitsBucketSize = this.getTopHitsBucketSize(paramMap);// 每个桶下面获取的商品数量
// 7、执行搜索
JSONObject jsonObject = this.getAggProductListJSONObject(paramMap, firstAggField, firstAggBucketCount, aggOrderSearchSort, topHitsBucketSize, secondSearchSort,
totalViewNum);
if (jsonObject == null) {
jsonObject = new JSONObject();
}
return result;
return new SearchApiResult().setMessage("agg productList by brand list").setData(jsonObject);
}
@Override
public SearchApiResult aggProductListByBrand(Map<String, String> paramMap) {
/**
* @param firstAggField
* 【父聚合的字段名称】
* @param firstAggBucketCount
* 【父聚合的桶大小】
* @param aggOrderSearchSort
* 【聚合排序规则，即按什么排序推荐】
* @param secondSearchSort
* 【次要排序条件，即最终商品按什么排序】
* @param totalViewNum
* 【最终返回的总数量】
* @return
*/
private JSONObject getAggProductListJSONObject(Map<String, String> paramMap, String firstAggField, int firstAggBucketCount, SearchSort aggSort, int topHitsBucketSize,
SearchSort secondSearchSort, int totalViewNum) {
try {
logger.info("[func=aggProductList][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
// 1、参数校验
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1 || pageSize < 0) {
throw new IllegalArgumentException("分页参数不合法");
}
if (pageSize > 60) {
pageSize = 60;
}
// 3、构造查询条件
// 1、构造查询条件
SearchParam searchParam = new SearchParam();
//paramMap.put("promotion", null);
searchParam.setQuery(this.builderActivityQueryBuilder(paramMap));// 活动特定的个性化
searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap, null));
searchParam.setQuery(this.builderActivityQueryBuilder(paramMap));// 活动特定的个性化
searchParam.setSize(0);
// 4、构造聚合条件
final String firstAggName = "firstAgg";
String order = "sales_num:desc";
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
order = "_score:desc";
// just for cache
searchParam.setOffset(totalViewNum);
if (secondSearchSort != null) {
searchParam.setSortBuilders(Arrays.asList(new FieldSortBuilder(secondSearchSort.getSortField()).order(secondSearchSort.getSortOrder())));
}
order = searchSortHelper.dealSortField(order,false);
String sortField = order.split(":")[0];
SortOrder sortOrder = order.split(":")[1].equals("desc") ? SortOrder.DESC : SortOrder.ASC;
// 2、构造聚合条件
final String firstAggName = "firstAgg";
List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>();
// 4.1）构造父聚合:品牌或品类聚合【同时按子聚合的sort字段排序】
TermsBuilder brandAggregationBuilder = AggregationBuilders.terms(firstAggName).field("brandId").order(Terms.Order.aggregation("sort", sortOrder.equals(SortOrder.ASC)))
.size(200 + pageSize);
// 4.2）添加子聚合:取得分最大的值
brandAggregationBuilder.subAggregation(AggregationBuilders.max("sort").field(sortField));
// 4.3）添加孙聚合：取打分最高的一个product
brandAggregationBuilder.subAggregation(AggregationBuilders.topHits("product").addSort(SortBuilders.fieldSort(sortField).order(sortOrder)).setSize(1));
list.add(brandAggregationBuilder);
// 2.1）构造父聚合:品牌或品类聚合【同时按子聚合的sort字段排序】
TermsBuilder firstAggregationBuilder = AggregationBuilders.terms(firstAggName).field(firstAggField).order(Terms.Order.aggregation("sort", aggSort.asc()))
.size(firstAggBucketCount);
// 2.2）添加子聚合:取得分最大的值
firstAggregationBuilder.subAggregation(AggregationBuilders.max("sort").field(aggSort.getSortField()));
// 2.3）添加孙聚合：取打分最高的一个product
firstAggregationBuilder.subAggregation(AggregationBuilders.topHits("product").addSort(SortBuilders.fieldSort(aggSort.getSortField()).order(aggSort.getSortOrder()))
.setSize(topHitsBucketSize));
list.add(firstAggregationBuilder);
searchParam.setAggregationBuilders(list);
// 5、构造返回结果
SearchApiResult searchApiResult = new SearchApiResult().setMessage("agg productList list");
// 6、先从缓存中获取,如果能取到，则直接返回
// 3、先从缓存中获取,如果能取到，则直接返回
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (jsonObject != null) {
return searchApiResult.setData(jsonObject);
return jsonObject;
}
// 7、执行搜索,并构造返回结果
// 4、执行搜索,并构造返回结果
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null) {
return searchApiResult.setData("");
return null;
}
Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
if (!aggMaps.containsKey(firstAggName)) {
return searchApiResult.setData("");
return null;
}
List<Map<String, Object>> productList = this.getProductListOrderByScore(((MultiBucketsAggregation) aggMaps.get(firstAggName)), pageSize, sortField, sortOrder);
// 5、构造返回结果
List<Map<String, Object>> productList = this.getAggProductListResult(((MultiBucketsAggregation) aggMaps.get(firstAggName)), totalViewNum, aggSort, secondSearchSort);
jsonObject = new JSONObject();
jsonObject.put("total", pageSize);
jsonObject.put("total", productList == null ? 0 : productList.size());
jsonObject.put("page", 1);
jsonObject.put("page_size", pageSize);
jsonObject.put("page_size", totalViewNum);
jsonObject.put("page_total", 1);
jsonObject.put("product_list", productList);
searchCacheService.addJSONObjectToCache(indexName, searchParam, jsonObject);
return searchApiResult.setData(jsonObject);
return jsonObject;
} catch (Exception e) {
publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_AGG_PRODUCTLIST.getEventName(), EventReportEnum.SEARCHCONTROLLER_AGG_PRODUCTLIST
.getFunctionName(), EventReportEnum.SEARCHCONTROLLER_AGG_PRODUCTLIST.getMoudleName(), "exception", IgnoreSomeException.filterSomeException(e), null));
return SearchApiResultUtils.errorSearchApiResult("aggProductList", paramMap, e);
return null;
}
}
private List<Map<String, Object>> getProductListOrderByScore(final MultiBucketsAggregation aggregation, int viewNum, String sortField, SortOrder sortOrder) {
private List<Map<String, Object>> getAggProductListResult(final MultiBucketsAggregation aggregation, int viewNum, SearchSort firstOrder, SearchSort secondOrder) {
// 1、从聚合结果中获取原生数据
List<Map<String, Object>> productList = this.getProductListWithFromAggregation(aggregation);
// 2、按指定的首要规则排序
productList = this.sortListBySortField(productList, viewNum, firstOrder);
// 3、按指定的次要规则排序
productList = this.sortListBySortField(productList, Integer.MAX_VALUE, secondOrder);
// 4、获取最终结果
return this.getProductListResults(productList);
}
/**
* 从聚合结果中获取原生的商品列表
*
* @param aggregation
* @return
*/
private List<Map<String, Object>> getProductListWithFromAggregation(final MultiBucketsAggregation aggregation) {
Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
// 获取品牌聚合出来的商品
TopHits topHits;
SearchHits hits;
List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
List<Map<String, Object>> productList = new ArrayList<Map<String, Object>>();
while (itAgg.hasNext()) {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
topHits = lt.getAggregations().get("product");
TopHits topHits = lt.getAggregations().get("product");
if (topHits != null) {
hits = topHits.getHits();
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> source = hit.getSource();
float _score = hit.getScore();
source.put("_score", _score);
dataList.add(source);
productList.add(source);
}
}
}
}
dataList = this.sortListBySortField(dataList, viewNum, sortField, sortOrder);
return productList;
}
/**
* 组合productPlan
*
* @param productList
* @return
*/
private List<Map<String, Object>> getProductListResults(List<Map<String, Object>> productList) {
List<Map<String, Object>> results = new ArrayList<Map<String, Object>>();
try {
List<String> sknStr = new ArrayList<String>();
for (Map<String, Object> data : dataList) {
for (Map<String, Object> data : productList) {
sknStr.add("" + data.get("productSkn"));
}
Map<String, List<Map<String, Object>>> productPricePlanMap = searchServiceHelper.searchProductPricePlan((String[]) sknStr.toArray(new String[sknStr.size()]));
for (Map<String, Object> m : dataList) {
result.add(searchServiceHelper.getProductMapWithPricePlan(m, productPricePlanMap));
for (Map<String, Object> m : productList) {
results.add(searchServiceHelper.getProductMapWithPricePlan(m, productPricePlanMap));
}
} catch (Exception e) {
logger.error("[func=aggProductList][Exception={}][begin={}]", e, System.currentTimeMillis());
}
return result;
return results;
}
private double getDouble(Object value) {
... ... @@ -333,25 +371,24 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
return 0;
}
private List<Map<String, Object>> sortListBySortField(List<Map<String, Object>> productList, int viewNum, String orderField, SortOrder sortOrder) {
if (productList == null || productList.isEmpty()) {
return new ArrayList<Map<String, Object>>();
private List<Map<String, Object>> sortListBySortField(List<Map<String, Object>> productList, int viewNum, SearchSort searchSort) {
if (productList == null || productList.isEmpty() || searchSort == null) {
return productList;
}
// 再按照某个字段对商品排序
boolean isDesc = sortOrder.equals(SortOrder.DESC) ? true : false;
boolean asc = searchSort.asc();
Collections.sort(productList, new Comparator<Map<String, Object>>() {
public int compare(Map<String, Object> o1, Map<String, Object> o2) {
try {
double value1 = getDouble(o1.get(orderField));
double value2 = getDouble(o2.get(orderField));
double value1 = getDouble(o1.get(searchSort.getSortField()));
double value2 = getDouble(o2.get(searchSort.getSortField()));
if (value1 == value2) {
return 0;
}
if (isDesc) {
return value1 - value2 > 0 ? -1 : 1;
} else {
if (asc) {
return value1 - value2 > 0 ? 1 : -1;
} else {
return value1 - value2 > 0 ? -1 : 1;
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
... ...
... ... @@ -26,6 +26,8 @@ import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
... ... @@ -43,12 +45,14 @@ import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.service.helper.SearchSortHelper;
import com.yoho.search.service.servicenew.IGoodProductsService;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.GoodProductBO;
import com.yoho.search.service.vo.SearchApiResult;
@Service
public class GoodProductListService implements IGoodProductsService {
private static final Logger logger = LoggerFactory.getLogger(GoodProductListService.class);
@Autowired
private SearchCommonService searchCommonService;
@Autowired
... ... @@ -64,14 +68,12 @@ public class GoodProductListService implements IGoodProductsService {
@Autowired
private FunctionScoreSearchHelper functionScoreSearchHelper;
private final int maxSmallSortCount = 20;
private final int maxProductSknCountPerSort = 5;
private final int maxCountPerGroup = 3;
// private final float firstSknScore = 300;
private final float recommendedSknMaxScore = 200;
@Override
public SearchApiResult goodProductList(Map<String, String> paramMap) {
boolean openLog = "13420925".equals(paramMap.getOrDefault("uid", "0"));
if (openLog) {
logger.warn("goodProductList param is : [{}]", paramMap.toString());
}
// 1、检测分页参数
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 30 : Integer.parseInt(paramMap.get("viewNum"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
... ... @@ -79,9 +81,14 @@ public class GoodProductListService implements IGoodProductsService {
return new SearchApiResult().setCode(400).setMessage("分页参数不合法");
}
// 2、先获取用户浏览的SKN对应的品类列表
List<Integer> smallSortIds = this.getProductSknSmallSortIds(paramMap, maxSmallSortCount);
// 3、再每个品类下获取5个SKN[每次随机打乱]
List<String> recommondSkns = this.getRecommondedSkns(smallSortIds, maxProductSknCountPerSort, paramMap);
JSONObject sortAndBrandInfo = this.querySknSortAndBrand(paramMap.getOrDefault("product_skn", ""));
// 3、再根据品类和品牌推荐SKN
List<String> recommondSkns = this.recommondSknsBySortAndBrandInfo(sortAndBrandInfo, paramMap);
if (paramMap.getOrDefault("uid", "0").equals("13420925")) {
logger.warn("goodProductList recommondSkns is : [{}]", recommondSkns);
}
// 4、构造搜索参数
SearchParam searchParam = new SearchParam();
... ... @@ -103,6 +110,11 @@ public class GoodProductListService implements IGoodProductsService {
CacheEnum cacheEnum = CacheEnum.EHCACHE;
JSONObject cacheObject = searchCacheService.getJSONObjectFromCache(cacheEnum, indexName, searchParam);
if (cacheObject != null) {
JSONArray product_list = cacheObject.getJSONArray("product_list");
if(product_list!=null){
// Collections.shuffle(product_list);
// cacheObject.put("product_list", product_list);
}
return new SearchApiResult().setData(cacheObject);
}
// 6)查询ES
... ... @@ -123,43 +135,15 @@ public class GoodProductListService implements IGoodProductsService {
return new SearchApiResult().setData(dataMap);
}
private QueryBuilder builderGoodProductQueryBuilder(Map<String, String> paramMap, List<String> recommendedSknList) {
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder);
// // 针对参数里第一个SKN加分
// String productSkns =
// paramMap.get(SearchRequestParams.PARAM_SYNC_SKN);
// if (!StringUtils.isBlank(productSkns)) {
// functionScoreQueryBuilder.add(QueryBuilders.termsQuery("productSkn",
// productSkns.split(",")[0]),
// ScoreFunctionBuilders.weightFactorFunction(firstSknScore));
// }
// 针对推荐出来的SKN做加分
if (recommendedSknList != null && !recommendedSknList.isEmpty()) {
Map<Integer, List<String>> recommondSknMap = this.splitProductSkns(recommendedSknList, maxCountPerGroup);
float currentGroupScore = recommendedSknMaxScore;
for (Map.Entry<Integer, List<String>> entry : recommondSknMap.entrySet()) {
functionScoreQueryBuilder.add(QueryBuilders.termsQuery("productSkn", entry.getValue()), ScoreFunctionBuilders.weightFactorFunction(currentGroupScore));
currentGroupScore = currentGroupScore - 10;
}
}
// 加上个性化打分
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
functionScoreSearchHelper.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
}
return functionScoreQueryBuilder;
}
/**
* 获取SKN相关的小分类
* 获取skn的品类列表和品牌列表
*
* @param productSkns
* @return
*/
private List<Integer> getProductSknSmallSortIds(Map<String, String> paramMap, int maxCount) {
String productSkns = paramMap.get(SearchRequestParams.PARAM_SYNC_SKN);
private JSONObject querySknSortAndBrand(String productSkns) {
if (StringUtils.isBlank(productSkns)) {
return new ArrayList<Integer>();
return new JSONObject();
}
String[] productSknArray = productSkns.split(",");
SearchParam searchParam = new SearchParam();
... ... @@ -167,102 +151,108 @@ public class GoodProductListService implements IGoodProductsService {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termsQuery("productSkn", productSknArray));
searchParam.setFiter(boolFilter);
// 2、设置聚合条件
final String aggName = "smallSortAgg";
TermsBuilder smallSortIdAgg = AggregationBuilders.terms(aggName).field("smallSortId").size(maxCount);
searchParam.setAggregationBuilders(Arrays.asList(smallSortIdAgg));
// 2、设置聚合条件，获取所有的品类和品牌
TermsBuilder smallSortIdAgg = AggregationBuilders.terms("smallSortAgg").field("smallSortId").size(100);
TermsBuilder brandIdAgg = AggregationBuilders.terms("brandAgg").field("brandId").size(100);
List<AbstractAggregationBuilder> aggregationBuilders = new ArrayList<AbstractAggregationBuilder>();
aggregationBuilders.add(smallSortIdAgg);
aggregationBuilders.add(brandIdAgg);
searchParam.setAggregationBuilders(aggregationBuilders);
// 3、设置分页
searchParam.setPage(0);
searchParam.setSize(0);
searchParam.setOffset(0);
// 4、先从缓存中获取,如果能取到，则直接返回
JSONArray sknSmallSortIdJSONArray = searchCacheService.getJSONArrayFromCache(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (sknSmallSortIdJSONArray != null) {
return this.jsonArrayToList(sknSmallSortIdJSONArray, Integer.class);
JSONObject sortAndBrandJSONObject = searchCacheService.getJSONObjectFromCache(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (sortAndBrandJSONObject != null) {
return sortAndBrandJSONObject;
}
// 5、执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null || searchResult.getAggMaps().get("smallSortAgg") == null) {
if (searchResult == null || searchResult.getAggMaps() == null) {
return new JSONObject();
}
Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
// 6、从聚合结果中获取品牌和品类id
List<Integer> smallSortIds = this.getIdsFromAggMaps(aggMaps, "smallSortAgg");
List<Integer> brandIds = this.getIdsFromAggMaps(aggMaps, "brandAgg");
// 7、构造返回结果并加入缓存
sortAndBrandJSONObject = new JSONObject();
sortAndBrandJSONObject.put("brandIds", brandIds);
sortAndBrandJSONObject.put("smallSortIds", smallSortIds);
searchCacheService.addJSONObjectToCache(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam, sortAndBrandJSONObject);
return sortAndBrandJSONObject;
}
private List<Integer> getIdsFromAggMaps(Map<String, Aggregation> aggMaps, String aggName) {
if (!aggMaps.containsKey(aggName)) {
return new ArrayList<Integer>();
}
sknSmallSortIdJSONArray = new JSONArray();
MultiBucketsAggregation aggregation = (MultiBucketsAggregation) searchResult.getAggMaps().get(aggName);
Iterator<? extends Bucket> smallSortIdIterator = aggregation.getBuckets().iterator();
while (smallSortIdIterator.hasNext()) {
Bucket smallSortIdBucket = smallSortIdIterator.next();
if (StringUtils.isNumeric(smallSortIdBucket.getKeyAsString())) {
sknSmallSortIdJSONArray.add(Integer.valueOf(smallSortIdBucket.getKeyAsString()));
MultiBucketsAggregation aggregation = (MultiBucketsAggregation) aggMaps.get(aggName);
List<Integer> results = new ArrayList<Integer>();
Iterator<? extends Bucket> iterator = aggregation.getBuckets().iterator();
while (iterator.hasNext()) {
Bucket bucket = iterator.next();
if (StringUtils.isNumeric(bucket.getKeyAsString())) {
results.add(Integer.valueOf(bucket.getKeyAsString()));
}
}
searchCacheService.addJSONArrayToCache(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam, sknSmallSortIdJSONArray);
return this.jsonArrayToList(sknSmallSortIdJSONArray, Integer.class);
return results;
}
/**
* 有好货默认的过滤规则
* 根据skn参数获取推荐出来的skn列表
*
* @return
*/
private BoolQueryBuilder getDefaultBoolQueryBuilder() {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y"));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
boolFilter.must(QueryBuilders.termQuery("status", 1));
boolFilter.must(QueryBuilders.termQuery("isOutlets", 2));
boolFilter.must(QueryBuilders.termQuery("attribute", 1));
// 默认推库存>2,非断码，并且短评存在的数据
boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(2));
boolFilter.must(QueryBuilders.rangeQuery("breakingRate").lt(50));
boolFilter.must(QueryBuilders.rangeQuery("basePinRatio").lt(3.5));
boolFilter.must(QueryBuilders.termQuery("isPhraseExist", "Y"));
return boolFilter;
}
/**
* 每个品类为用户推荐maxSize个SKN
*
* @param smallSortIds
* @param maxSize
* @param paramMap
* @return
*/
private List<String> getRecommondedSkns(List<Integer> smallSortIds, int maxSizePerSort, Map<String, String> paramMap) {
// 1、如果品类为空，则直接返回
if (smallSortIds == null || smallSortIds.isEmpty()) {
private List<String> recommondSknsBySortAndBrandInfo(JSONObject sortAndBrandInfo, Map<String, String> paramMap) {
// 1、获取品牌id和品类id
if (sortAndBrandInfo == null || sortAndBrandInfo.isEmpty()) {
return new ArrayList<String>();
}
JSONArray brandIdJSONArray = sortAndBrandInfo.getJSONArray("brandIds");
JSONArray smallSortIdJSONArray = sortAndBrandInfo.getJSONArray("smallSortIds");
List<Integer> brandIds = this.jsonArrayToList(brandIdJSONArray, Integer.class);
List<Integer> smallSortIds = this.jsonArrayToList(smallSortIdJSONArray, Integer.class);
if (brandIds.isEmpty() && smallSortIds.isEmpty()) {
return new ArrayList<String>();
}
SearchParam searchParam = new SearchParam();
// 2、构造filter
SearchParam searchParam = new SearchParam();
BoolQueryBuilder boolFilter = this.getDefaultBoolQueryBuilder();
boolFilter.must(QueryBuilders.termsQuery("smallSortId", smallSortIds));
BoolQueryBuilder brandAndSortFilter = new BoolQueryBuilder();
if (!brandIds.isEmpty()) {
brandAndSortFilter.should(QueryBuilders.termsQuery("brandId", brandIds));
}
if (!smallSortIds.isEmpty()) {
brandAndSortFilter.should(QueryBuilders.termsQuery("smallSortId", smallSortIds));
}
boolFilter.must(brandAndSortFilter);
searchParam.setFiter(boolFilter);
// 3、构造query
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.matchAllQuery());
// 针对看过的SKN做加分
String productSkns = paramMap.get(SearchRequestParams.PARAM_SYNC_SKN);
if (!StringUtils.isBlank(productSkns)) {
functionScoreQueryBuilder.add(QueryBuilders.termsQuery("productSkn", productSkns.split(",")), ScoreFunctionBuilders.weightFactorFunction(100));
}
// 强制加上个性化打分
functionScoreSearchHelper.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
searchParam.setQuery(functionScoreQueryBuilder);
// 4、设置聚合条件
final String firstAggName = "firstAgg";
String order = "_score:desc";
String sortField = order.split(":")[0];
SortOrder sortOrder = order.split(":")[1].equals("desc") ? SortOrder.DESC : SortOrder.ASC;
String sortField = "_score";
SortOrder sortOrder = SortOrder.DESC;
List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>();
// 4.1）构造父聚合:品牌或品类聚合【同时按子聚合的sort字段排序】
TermsBuilder parentAggregationBuilder = AggregationBuilders.terms(firstAggName).field("smallSortId").size(smallSortIds.size());
// 4.2）添加子聚合:取得分最大的值
parentAggregationBuilder.subAggregation(AggregationBuilders.max("sort").field(sortField));
// 4.3）添加孙聚合：取打分最高的一个product
parentAggregationBuilder.subAggregation(AggregationBuilders.topHits("product").addSort(SortBuilders.fieldSort(sortField).order(sortOrder)).setSize(maxSizePerSort));
list.add(parentAggregationBuilder);
// 4.1）构造父聚合:品类聚合【同时按子聚合的sort字段排序】
TermsBuilder parentAggregationBuilder = AggregationBuilders.terms("smallSortAgg").field("smallSortId").size(smallSortIds.size());
// 4.2）构造子聚合:品牌或聚合【同时按子聚合的sort字段排序】
TermsBuilder sonAggregationBuilder = AggregationBuilders.terms("brandAgg").field("brandId").size(brandIds.size());
// 4.3）为子聚合添加孙聚合:取得分最大的值
sonAggregationBuilder.subAggregation(AggregationBuilders.max("sort").field(sortField));
// 4.4）为子聚合孙聚合：取打分最高的一个product
sonAggregationBuilder.subAggregation(AggregationBuilders.topHits("product").addSort(SortBuilders.fieldSort(sortField).order(sortOrder)).setSize(1));
list.add(parentAggregationBuilder.subAggregation(sonAggregationBuilder));
searchParam.setAggregationBuilders(list);
// 5、设置分页
... ... @@ -275,26 +265,123 @@ public class GoodProductListService implements IGoodProductsService {
if (recommendedSknJSONArray != null) {
return this.jsonArrayToList(recommendedSknJSONArray, String.class);
}
// 7、执行搜索,并构造返回结果
// 7、执行搜索
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null || searchResult.getAggMaps() == null) {
return new ArrayList<String>();
}
// 8、获取搜索结果
Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
if (!aggMaps.containsKey(firstAggName)) {
if (!aggMaps.containsKey("smallSortAgg")) {
return new ArrayList<String>();
}
List<String> recommendedSknList = this.getRecommendedSknList((MultiBucketsAggregation) aggMaps.get(firstAggName));
List<GoodProductBO> goodProductLists = new ArrayList<GoodProductBO>();
Iterator<? extends Bucket> smallSortIterator = ((MultiBucketsAggregation) aggMaps.get("smallSortAgg")).getBuckets().iterator();
while (smallSortIterator.hasNext()) {
Bucket smallSortBucket = smallSortIterator.next();
Map<String, Aggregation> smallSortAggMap = smallSortBucket.getAggregations().getAsMap();
if (!smallSortAggMap.containsKey("brandAgg")) {
continue;
}
Iterator<? extends Bucket> brandIdIterator = ((MultiBucketsAggregation) smallSortAggMap.get("brandAgg")).getBuckets().iterator();
List<String> productSkns = new ArrayList<String>();
while (brandIdIterator.hasNext()) {
Bucket brandIdBucket = brandIdIterator.next();
if (brandIdBucket.getAggregations().getAsMap().containsKey("product")) {
TopHits topHits = brandIdBucket.getAggregations().get("product");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
productSkns.add("" + hit.getSource().get("productSkn"));
}
}
}
}
if (!productSkns.isEmpty()) {
Collections.shuffle(productSkns);
goodProductLists.add(new GoodProductBO(smallSortBucket.getKeyAsString(), productSkns));
}
}
Collections.shuffle(goodProductLists);
// 9、构造返回结果
recommendedSknJSONArray = new JSONArray();
for (String recommendedSkn : recommendedSknList) {
recommendedSknJSONArray.add(recommendedSkn);
// 9.1、获取推荐出来的商品
recommendedSknJSONArray.addAll(this.getProductSkns(goodProductLists));
// 9.2、再加入用户看过的商品前面
for (String skn : paramMap.getOrDefault("product_skn", "").split(",")) {
if (!recommendedSknJSONArray.contains(skn)) {
recommendedSknJSONArray.add(skn);
}
}
searchCacheService.addJSONArrayToCache(indexName, searchParam, recommendedSknJSONArray);
return this.jsonArrayToList(recommendedSknJSONArray, String.class);
}
private List<String> getProductSkns(List<GoodProductBO> goodProductLists) {
if (goodProductLists == null || goodProductLists.isEmpty()) {
return new ArrayList<String>();
}
int maxSknCount = 0;
for (GoodProductBO goodProductBO : goodProductLists) {
int productSknSize = goodProductBO.getProductSkn().size();
if (maxSknCount < productSknSize) {
maxSknCount = productSknSize;
}
}
List<String> results = new ArrayList<String>();
for (int i = 0; i < maxSknCount; i++) {
for (GoodProductBO goodProductBO : goodProductLists) {
int randomCount = 1 + (int) (Math.random() * 2);
results.addAll(goodProductBO.randomGetProductSkn(randomCount));
}
}
return results;
}
private QueryBuilder builderGoodProductQueryBuilder(Map<String, String> paramMap, List<String> recommendedSknList) {
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder);
// 针对推荐出来的SKN做加分
if (recommendedSknList != null && !recommendedSknList.isEmpty()) {
Map<Integer, List<String>> recommondSknMap = this.splitProductSkns(recommendedSknList, 2);
float currentGroupScore = 1000;
for (Map.Entry<Integer, List<String>> entry : recommondSknMap.entrySet()) {
functionScoreQueryBuilder.add(QueryBuilders.termsQuery("productSkn", entry.getValue()), ScoreFunctionBuilders.weightFactorFunction(currentGroupScore));
currentGroupScore = currentGroupScore - 10;
}
}
// 加上个性化打分
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
functionScoreSearchHelper.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
}
return functionScoreQueryBuilder;
}
/**
* 有好货默认的过滤规则
*
* @return
*/
private BoolQueryBuilder getDefaultBoolQueryBuilder() {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y"));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
boolFilter.must(QueryBuilders.termQuery("status", 1));
boolFilter.must(QueryBuilders.termQuery("isOutlets", 2));
boolFilter.must(QueryBuilders.termQuery("attribute", 1));
// 默认推库存>2,非断码，并且短评存在的数据
boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(2));
boolFilter.must(QueryBuilders.rangeQuery("breakingRate").lt(50));
boolFilter.must(QueryBuilders.rangeQuery("basePinRatio").lt(3.5));
boolFilter.must(QueryBuilders.termQuery("isPhraseExist", "Y"));
return boolFilter;
}
private <T> List<T> jsonArrayToList(JSONArray jsonArray, Class<T> clazz) {
if (jsonArray == null) {
return new ArrayList<T>();
}
return JSONObject.parseArray(jsonArray.toJSONString(), clazz);
}
... ... @@ -320,40 +407,11 @@ public class GoodProductListService implements IGoodProductsService {
return result;
}
/**
* 获取聚合出来的商品
*
* @param aggregation
* @return
*/
private List<String> getRecommendedSknList(MultiBucketsAggregation aggregation) {
Iterator<? extends Bucket> itAgg = aggregation.getBuckets().iterator();
List<String> recommendedSknList = new ArrayList<String>();
while (itAgg.hasNext()) {
Bucket lt = itAgg.next();
if (lt.getAggregations().getAsMap().containsKey("product")) {
TopHits topHits = lt.getAggregations().get("product");
if (topHits != null) {
SearchHits hits = topHits.getHits();
for (SearchHit hit : hits.getHits()) {
recommendedSknList.add("" + hit.getSource().get("productSkn"));
}
}
}
public static List<String> getNewList(List<String> oldList) {
List<String> newList = new ArrayList<String>();
for (String element : oldList) {
newList.add(element);
}
Collections.shuffle(recommendedSknList);
return recommendedSknList;
return newList;
}
public static void main(String[] args) {
List<String> list = new ArrayList<String>();
for (int i = 1; i <= 99; i++) {
list.add(i + "");
}
Map<Integer, List<String>> results = new GoodProductListService().splitProductSkns(list, 20);
for (Map.Entry<Integer, List<String>> entry : results.entrySet()) {
System.out.println(entry.getKey() + "_" + entry.getValue());
}
}
}
... ...
... ... @@ -14,13 +14,11 @@ import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.service.helper.SearchSortHelper;
import com.yoho.search.service.servicenew.IProductListService;
import com.yoho.search.service.servicenew.ISuggestService;
import com.yoho.search.service.servicenew.ISearchRecommendService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.utils.SearchApiResultUtils;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SuggestApiResult;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
... ... @@ -54,7 +52,7 @@ public class ProductListServiceImpl implements IProductListService {
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private ISuggestService suggestService;
private ISearchRecommendService searchRecommendService;
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
... ... @@ -65,7 +63,7 @@ public class ProductListServiceImpl implements IProductListService {
if (needTermSuggestion(searchResult, paramMap)) {
// 当商品数量太少或无结果时 支持智能推荐Term搜索
JSONObject dataMap = ((JSONObject) searchResult.getData());
dataMap.put("suggestion", suggestService.suggestTips(paramMap));
dataMap.put("suggestion", searchRecommendService.recommend(searchResult, paramMap));
}
return searchResult;
... ... @@ -76,7 +74,8 @@ public class ProductListServiceImpl implements IProductListService {
}
private SearchApiResult innerProductList(Map<String, String> paramMap) throws Exception {
logger.info("[func=productList][param={}][begin={}]", paramMap, System.currentTimeMillis());
long begin = System.currentTimeMillis();
logger.info("[func=productList][param={}][begin={}]", paramMap, begin);
// 1)构造搜索参数
SearchParam searchParam = buildProductListSearchParam(paramMap);
... ... @@ -121,6 +120,7 @@ public class ProductListServiceImpl implements IProductListService {
// 10)将结果存进缓存
searchCacheService.addJSONObjectToCache(cacheEnum, indexName, searchParam, dataMap);
logger.info("[func=productList][cost={}]", System.currentTimeMillis() - begin);
return new SearchApiResult().setData(dataMap);
}
... ... @@ -128,13 +128,11 @@ public class ProductListServiceImpl implements IProductListService {
if (searchResult == null || searchResult.getCode() != 200 || searchResult.getData() == null) {
return false;
}
// 1. 判断是否需要进行term推荐
// 1.1 query不为空且不是查询SKN
// 1.1 query不为空
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
if (StringUtils.isEmpty(queryWord) || searchCommonHelper.isQuerySkn(queryWord)) {
return false;
}
if (queryWord.length() > 30) {
if (StringUtils.isEmpty(queryWord) || (queryWord.length() > 30 && !searchCommonHelper.isQuerySkn(queryWord))) {
return false;
}
... ...
... ... @@ -95,7 +95,9 @@ public class SearchLikeInShopServiceImpl implements ISearchLikeInShopService {
searchParam.setQuery(queryBuilder);
// 5、设置filter
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultQueryBuilder(Arrays.asList(productSkn));
String isGlobalInEs = productInfoInEs.getString("isGlobal");
boolean isGlobal = "Y".equalsIgnoreCase(isGlobalInEs);
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultQueryBuilder(Arrays.asList(productSkn),isGlobal);
// 1)设置此SKN相关的性别过滤条件
String gender = productInfoInEs.getString("gender");
List<String> genderList = searchLikeHelper.getGenderInfo(gender);
... ...
... ... @@ -81,10 +81,14 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
if (pageSize > 60 || pageSize <= 0) {
pageSize = 60;
}
//3.1是否是全球购
String isGlobalInEs = productInfoInEs.getString("isGlobal");
boolean isGlobal = "Y".equalsIgnoreCase(isGlobalInEs);
// 4、获取同一个品牌下最多5个商品
int maxCountInBrand = 5;
JSONArray inBrandProductList = this.getProductListInBrand(productInfoInEs, paramMap, pageSize>maxCountInBrand?maxCountInBrand:pageSize);
JSONArray inBrandProductList = this.getProductListInBrand(productInfoInEs, paramMap, pageSize > maxCountInBrand ? maxCountInBrand : pageSize,isGlobal);
// 5、获取不同品牌下的余下数量的商品[通过聚合的方式来查找]
List<String> notInProductSkns = new ArrayList<String>();
... ... @@ -93,7 +97,7 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
notInProductSkns.add(product.getString("product_skn"));
}
notInProductSkns.add(productSkn);
JSONArray notInBrandProductList = this.getProductListNotInBrand(productInfoInEs, paramMap, notInProductSkns, pageSize - inBrandProductList.size());
JSONArray notInBrandProductList = this.getProductListNotInBrand(productInfoInEs, paramMap, notInProductSkns, pageSize - inBrandProductList.size(),isGlobal);
// 6、构造返回结果
Map<String, Object> dataMap = new HashMap<String, Object>();
... ... @@ -118,14 +122,14 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
* @param limit
* @return
*/
private JSONArray getProductListInBrand(JSONObject productInfoInEs, Map<String, String> paramMap, int limit) {
private JSONArray getProductListInBrand(JSONObject productInfoInEs, Map<String, String> paramMap, int limit,boolean isGlobal) {
SearchParam searchParam = new SearchParam();
// 1、构建Query
QueryBuilder queryBuilder = this.genQueryBuilder(productInfoInEs, paramMap, true);
QueryBuilder queryBuilder = this.genQueryBuilder(productInfoInEs, paramMap, true,isGlobal);
searchParam.setQuery(queryBuilder);
// 2、设置过滤条件
BoolQueryBuilder booleanQueryBuilder = this.genBoolQueryBuilder(productInfoInEs, paramMap, Arrays.asList(paramMap.get(SearchRequestParams.PARAM_SYNC_SKN)), true);
BoolQueryBuilder booleanQueryBuilder = this.genBoolQueryBuilder(productInfoInEs, paramMap, Arrays.asList(paramMap.get(SearchRequestParams.PARAM_SYNC_SKN)), true,isGlobal);
searchParam.setFiter(booleanQueryBuilder);
// 3、设置排序规则[按打分排序]
... ... @@ -164,17 +168,17 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
* @param limit
* @return
*/
private JSONArray getProductListNotInBrand(JSONObject productInfoInEs, Map<String, String> paramMap, List<String> notInProductSkns, int limit) {
if(limit<=0){
private JSONArray getProductListNotInBrand(JSONObject productInfoInEs, Map<String, String> paramMap, List<String> notInProductSkns, int limit,boolean isGlobal) {
if (limit <= 0) {
return new JSONArray();
}
SearchParam searchParam = new SearchParam();
// 1、构建Query
QueryBuilder queryBuilder = this.genQueryBuilder(productInfoInEs, paramMap, false);
QueryBuilder queryBuilder = this.genQueryBuilder(productInfoInEs, paramMap, false,isGlobal);
searchParam.setQuery(queryBuilder);
// 2、设置过滤条件
BoolQueryBuilder booleanQueryBuilder = this.genBoolQueryBuilder(productInfoInEs, paramMap, notInProductSkns, false);
BoolQueryBuilder booleanQueryBuilder = this.genBoolQueryBuilder(productInfoInEs, paramMap, notInProductSkns, false,isGlobal);
searchParam.setFiter(booleanQueryBuilder);
// 3、设置聚合条件
... ... @@ -220,31 +224,41 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
return productJSONArray;
}
private QueryBuilder genQueryBuilder(JSONObject productInfoInEs, Map<String, String> paramMap, boolean isInBrand) {
// 1、设置查询的值
String productName = productInfoInEs.getString("productName");
String standardOnlyNames = productInfoInEs.getString("standardOnlyNames");
String style = productInfoInEs.getString("style");
String pattern = productInfoInEs.getString("pattern");
String attributeNames = productInfoInEs.getString("attributeNames");
private QueryBuilder genQueryBuilder(JSONObject productInfoInEs, Map<String, String> paramMap, boolean isInBrand, boolean isGlobal) {
StringBuilder query = new StringBuilder();
// 1、如果是全球购,则直接用商品名称+品类名称去查
if (isGlobal) {
this.append(query, productInfoInEs.getString("productName"));
this.append(query, productInfoInEs.getString("smallSort"));
this.append(query, productInfoInEs.getString("middleSort"));
this.append(query, productInfoInEs.getString("maxSort"));
return searchLikeHelper.genSearchLikeQueryBuilder(query.toString(), "25%", null);
}
// 2、设置有货的查询的值
this.append(query, productInfoInEs.getString("productName"));
this.append(query, productInfoInEs.getString("style"));
this.append(query, productInfoInEs.getString("pattern"));
this.append(query, productInfoInEs.getString("attributeNames"));
this.append(query, productInfoInEs.getString("standardOnlyNames"));
String brandName = productInfoInEs.getString("brandName");
String productFeatureFactor = productInfoInEs.getString("productFeatureFactor");
String query = standardOnlyNames + "," + style + "," + pattern + "," + attributeNames;
String queryString = query.toString();
if (isInBrand) {
query = brandName + "," + productName + "," + query;
queryString = queryString + " " + brandName;
} else {
query = productName.replaceAll(brandName, "") + "," + query;
queryString = queryString.replaceAll(brandName, "");
}
// 2、设置匹配度
String minimumShouldMatch = "30%";
if (!isInBrand) {
minimumShouldMatch = "40%";
// 2、生成QueryBuilder
return searchLikeHelper.genSearchLikeQueryBuilder(queryString, isInBrand ? "40%" : "30%", productInfoInEs.getString("productFeatureFactor"));
}
private void append(StringBuilder stringBuilder, String word) {
if (StringUtils.isNotBlank(word)) {
stringBuilder.append(word).append(" ");
}
return searchLikeHelper.genSearchLikeQueryBuilder(query, minimumShouldMatch, productFeatureFactor);
}
private BoolQueryBuilder genBoolQueryBuilder(JSONObject productInfoInEs, Map<String, String> paramMap, List<String> notProductSkns, boolean isInBrand) {
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultQueryBuilder(notProductSkns);
private BoolQueryBuilder genBoolQueryBuilder(JSONObject productInfoInEs, Map<String, String> paramMap, List<String> notProductSkns, boolean isInBrand,boolean isGlobal) {
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultQueryBuilder(notProductSkns, isGlobal);
// 1)设置此SKN相关的过滤条件
String gender = productInfoInEs.getString("gender");
List<String> genderList = searchLikeHelper.getGenderInfo(gender);
... ... @@ -255,14 +269,21 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
BoolQueryBuilder sortFilter = QueryBuilders.boolQuery();
Integer middle_sort_id = productInfoInEs.getInteger("middleSortId");
Integer small_sort_id = productInfoInEs.getInteger("smallSortId");
sortFilter.should(QueryBuilders.termQuery("middleSortId", middle_sort_id));
sortFilter.should(QueryBuilders.termQuery("smallSortId", small_sort_id));
boolFilter.must(sortFilter);
if (middle_sort_id != null && middle_sort_id>0) {
sortFilter.should(QueryBuilders.termQuery("middleSortId", middle_sort_id));
}
if (small_sort_id != null && small_sort_id>0) {
sortFilter.should(QueryBuilders.termQuery("smallSortId", small_sort_id));
}
if (sortFilter.hasClauses()) {
boolFilter.must(sortFilter);
}
// 3)设置品牌
Integer brandId = productInfoInEs.getInteger("brandId");
if (isInBrand) {
if (brandId != null && brandId>0 && isInBrand) {
boolFilter.must(QueryBuilders.termQuery("brandId", brandId));
} else {
}
if (brandId != null && brandId>0 && !isInBrand) {
boolFilter.mustNot(QueryBuilders.termQuery("brandId", brandId));
}
return boolFilter;
... ...
package com.yoho.search.service.servicenew.impl;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.RedisKeys;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.service.SearchKeyWordService;
import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.servicenew.ISearchRecommendService;
import com.yoho.search.service.servicenew.ISuggestService;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import java.util.*;
import java.util.stream.Collectors;
/**
* Created by ginozhang on 2017/3/22.
*/
@Component
public class SearchRecommendServiceImpl implements ISearchRecommendService {
private static final Logger logger = LoggerFactory.getLogger(SearchRecommendServiceImpl.class);
private static final Logger CACHE_MATCH_REQUEST = LoggerFactory.getLogger("CACHE_MATCH_REQUEST");
// 返回智能搜索词的数量
private static final int SMART_SUGGESTION_TERM_COUNT = 3;
// 返回智能搜索词限制关联商品数量限制
private static final int SMART_SUGGESTION_COUNT_LIMIT = 20;
private static final List<String> DEFAULT_SUGGEST_TIPS = Arrays.asList("潮流", "时尚", "休闲");
private static final Map<String, String> CUSTOM_SUGGEST_CONVERSIONS = new HashMap<>();
private static final int VALID_STATUS = 1;
@Autowired
private ISuggestService suggestService;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private SearchKeyWordService searchKeyWordService;
@Autowired
private SearchServiceHelper searchServiceHelper;
static {
// it是一个停用词 所以直接配置转换关系
CUSTOM_SUGGEST_CONVERSIONS.put("it", ":CHOCOOLATE,izzue,5cm");
}
/**
* 根据query关键词和搜索结果获取搜索推荐词。
* 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
*
* @param searchResult 搜索结果
* @param paramMap 商品列表搜索参数
* @return 包括term建议和phrase建议。
*/
@Override
public JSONObject recommend(SearchApiResult searchResult, Map<String, String> paramMap) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
long begin = System.currentTimeMillis();
logger.info("[func=recommend][queryWord={}][begin={}]", queryWord, begin);
try {
Assert.isTrue(StringUtils.isNotEmpty(queryWord), "query keyword cannot be empty.");
Assert.isTrue(searchResult != null && searchResult.getData() != null, "SearchResult is invalid.");
String keywordsToSearch = queryWord;
// 搜索推荐分两部分
// 1) 第一部分是最常见的情况，包括有结果、根据SKN搜索、关键词未出现在空结果Redis ZSet里
if (containsProductInSearchResult(searchResult)) {
// 1.1） 搜索有结果的 优先从搜索结果聚合出品牌等关键词进行查询
JSONObject dataMap = ((JSONObject) searchResult.getData());
List<Map<String, Object>> productList = (List<Map<String, Object>>) dataMap.get("product_list");
String aggKeywords = aggKeywordsByProductList(productList);
keywordsToSearch = keywordsToSearch + " " + aggKeywords;
} else if (isQuerySkn(queryWord)) {
// 1.2） 如果是查询SKN 没有查询到的 后续的逻辑也无法推荐 所以直接到ES里去获取关键词
keywordsToSearch = aggKeywordsBySkns(queryWord);
if (StringUtils.isEmpty(keywordsToSearch)) {
return defaultSuggestRecommendation();
}
}
Double count = searchKeyWordService.getKeywordCount(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);
if (count == null || queryWord.length() >= 5) {
// 1.3) 如果该关键词一次都没有出现在空结果列表或者长度大于5 则该词很有可能是可以搜索出结果的 因此优先取suggest去搜索一把 减少后面的查询动作
JSONObject recommendResult = recommendBySuggestIndex(paramMap, keywordsToSearch, false);
if (recommendResult == null) {
return defaultSuggestRecommendation();
} else if (CollectionUtils.isNotEmpty((List) recommendResult.get("terms_suggestion"))) {
logger.info("[func=recommend][queryWord={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return recommendResult;
}
}
// 2) 第二部分是通过Conversion和拼写纠错去获取关键词 由于很多品牌的拼写可能比较相近 因此先走Conversion然后再拼写检查
String spellingCorrentWord = null, dest = null;
if (allowGetingDest(queryWord) && StringUtils.isNotEmpty((dest = getSuggestConversionDestBySource(queryWord)))) {
// 2.1) 爬虫和自定义的Conversion处理
keywordsToSearch = dest;
} else if (allowSpellingCorrent(queryWord) && StringUtils.isNotEmpty((spellingCorrentWord = suggestService.getSpellingCorrectKeyword(queryWord)))) {
// 2.2) 执行拼写检查 由于在搜索建议的时候会进行拼写检查 所以缓存命中率高
keywordsToSearch = spellingCorrentWord;
} else {
// 2.3) 如果两者都没有 则直接返回
return defaultSuggestRecommendation();
}
JSONObject recommendResult = recommendBySuggestIndex(paramMap, keywordsToSearch, dest != null);
if (recommendResult == null || CollectionUtils.isEmpty((List) recommendResult.get("terms_suggestion"))) {
recommendResult = defaultSuggestRecommendation();
}
logger.info("[func=recommend][queryWord={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return recommendResult;
} catch (Exception e) {
logger.error("[func=recommend][queryWord=" + queryWord + "]", e);
return defaultSuggestRecommendation();
}
}
private boolean allowGetingDest(String queryWord) {
// 对于conversion长度要求在[2,10]之间
if (queryWord.length() < 2 || queryWord.length() > 10) {
return false;
}
// 不能有空格、加号、逗号等符号
if (queryWord.contains(" ") || queryWord.contains("+") || queryWord.contains(",") || queryWord.contains("%")) {
return false;
}
return true;
}
private boolean allowSpellingCorrent(String queryWord) {
// 对于拼写检查长度要求在[4,20]之间
if (queryWord.length() < 4 || queryWord.length() > 20) {
return false;
}
// 不能有空格、加号、逗号等符号
if (queryWord.contains(" ") || queryWord.contains("+") || queryWord.contains(",") || queryWord.contains("%")) {
return false;
}
return true;
}
/**
* 根据计算获取的推荐词列表到suggest索引获取推荐词。
* 多个词的情况下优先会给匹配到更多term的词加分。
*
* @param paramMap 搜索参数，用于判断取哪个count
* @param keywordsToSearch 计算获取的推荐词列表
* @param isLimitKeywords 是否限制只能是推荐词列表里面的词
* @return 搜推荐结果
*/
private JSONObject recommendBySuggestIndex(Map<String, String> paramMap, String keywordsToSearch, boolean isLimitKeywords) {
String srcQueryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
long begin = System.currentTimeMillis();
logger.info("[func=recommendBySuggestIndex][srcQueryWord={}][keywordsToSearch={}][begin={}]", srcQueryWord, keywordsToSearch, begin);
// 1) 如果是查询SKN 直接返回 前面已经处理过了
if (StringUtils.isEmpty(keywordsToSearch)) {
return null;
}
// 2) 先对keywordsToSearch进行分词
List<String> terms = null;
if (isLimitKeywords) {
terms = Arrays.stream(keywordsToSearch.split(",")).filter(term -> term != null && term.length() > 1).distinct().collect(Collectors.toList());
} else {
terms = searchKeyWordService.getAnalyzeTerms(keywordsToSearch, "ik_smart", true);
}
if (CollectionUtils.isEmpty(terms)) {
return new JSONObject();
}
Set<String> termSet = terms.stream().collect(Collectors.toSet());
logger.info("[func=recommendBySuggestIndex][termSet={}]", termSet);
// 3) 根据terms搜索构造搜索请求
final String countField = suggestService.getCountField(paramMap);
SearchParam searchParam = new SearchParam();
QueryBuilder queryBuilder = isLimitKeywords ? buildQueryBuilderByLimit(terms) : buildQueryBuilder(keywordsToSearch, termSet);
searchParam.setQuery(queryBuilder);
searchParam.setPage(1);
searchParam.setSize(SMART_SUGGESTION_TERM_COUNT);
// 3.5) 设置过滤条件
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termQuery("status", VALID_STATUS));
boolFilter.must(QueryBuilders.rangeQuery(countField).gte(SMART_SUGGESTION_COUNT_LIMIT));
boolFilter.mustNot(QueryBuilders.termQuery("standardKeyword", CharUtils.standardized(srcQueryWord)));
if (isLimitKeywords) {
boolFilter.must(QueryBuilders.termsQuery("standardKeyword", termSet.stream().map(term -> CharUtils.standardized(term)).collect(Collectors.toList())));
}
searchParam.setFiter(boolFilter);
// 3.6) 按照得分、权重、数量的规则降序排序
List<SortBuilder> sortBuilders = new ArrayList<>(3);
sortBuilders.add(SortBuilders.fieldSort("_score").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort("weight").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(countField).order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
// 4) 先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject suggestResult = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggestResult != null) {
CACHE_MATCH_REQUEST.info("match cache for product list search recommendation by suggest, keyword = {}.", keywordsToSearch);
return suggestResult;
}
// 5) 调用ES执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, searchParam);
if (searchResult == null) {
return null;
}
// 6) 构建结果加入缓存
suggestResult = new JSONObject();
List<String> resultTerms = searchResult.getResultList().stream().map(map -> (String) map.get("keyword")).collect(Collectors.toList());
suggestResult.put("terms_suggestion", resultTerms);
searchCacheService.addJSONObjectToCache(indexName, searchParam, suggestResult);
logger.info("[func=recommendBySuggestIndex][srcQueryWord={}][keywordsToSearch={}][resultTerms={}][cost={}]",
srcQueryWord, keywordsToSearch, resultTerms, System.currentTimeMillis() - begin);
return suggestResult;
}
private QueryBuilder buildQueryBuilderByLimit(List<String> terms) {
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.matchAllQuery());
// 给品类类型的关键词加分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("type", Integer.valueOf(2)),
ScoreFunctionBuilders.weightFactorFunction(3));
// 按词出现的顺序加分
for (int i = 0; i < terms.size(); i++) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("standardKeyword", CharUtils.standardized(terms.get(i))),
ScoreFunctionBuilders.weightFactorFunction(terms.size() - i));
}
functionScoreQueryBuilder.boostMode(CombineFunction.SUM);
return functionScoreQueryBuilder;
}
private QueryBuilder buildQueryBuilder(String keywordsToSearch, Set<String> termSet) {
// 3.1) 对于suggest的multi-fields至少要有一个字段匹配到 匹配打分为常量1
MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keywordsToSearch.trim().toLowerCase(),
"keyword", "keyword.keyword_ik", "keyword.keyword_pinyin", "keyword.keyword_jianpin", "keyword.keyword_lowercase")
.analyzer("ik_smart")
.type(MultiMatchQueryBuilder.Type.BEST_FIELDS)
.operator(MatchQueryBuilder.Operator.OR)
.minimumShouldMatch("1");
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.constantScoreQuery(queryBuilder));
for (String term : termSet) {
// 3.2) 对于完全匹配Term的加1分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("standardKeyword", CharUtils.standardized(term)),
ScoreFunctionBuilders.weightFactorFunction(1));
// 3.3) 对于匹配到一个Term的加2分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("keyword.keyword_ik", term),
ScoreFunctionBuilders.weightFactorFunction(2));
}
// 3.4) 处理性别相关的关键词
if (termSet.contains("男") && !termSet.contains("女")) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("keyword.keyword_ik", "女"),
ScoreFunctionBuilders.weightFactorFunction(-5));
} else if (!termSet.contains("男") && termSet.contains("女")) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("keyword.keyword_ik", "男"),
ScoreFunctionBuilders.weightFactorFunction(-5));
} else if (!termSet.contains("男") && !termSet.contains("女")) {
functionScoreQueryBuilder.add(QueryBuilders.termsQuery("keyword.keyword_ik", Arrays.asList("男", "女")),
ScoreFunctionBuilders.weightFactorFunction(-2));
}
functionScoreQueryBuilder.boostMode(CombineFunction.SUM);
return functionScoreQueryBuilder;
}
private boolean containsProductInSearchResult(SearchApiResult searchResult) {
JSONObject dataMap = ((JSONObject) searchResult.getData());
return CollectionUtils.isNotEmpty((List) dataMap.get("product_list"));
}
/**
* 根据商品列表搜索结果聚合去关键词
* 当推荐词为3个时 品牌和品类最多各两个
*
* @param productList 商品列表搜索结果
* @return 关键词列表
*/
private String aggKeywordsByProductList(List<Map<String, Object>> productList) {
Map<String, Double> brandMap = new HashMap<>(productList.size());
Map<String, Double> sortNameMap = new HashMap<>(productList.size());
for (Map<String, Object> product : productList) {
String brandName = (String) product.get("brand_name");
if (StringUtils.isNotEmpty(brandName)) {
brandMap.compute(brandName, (k, v) -> v == null ? 1.0D : v + 1.0D);
}
String sortName = (String) product.get("small_sort_name");
if (StringUtils.isNotEmpty(sortName)) {
sortNameMap.compute(sortName, (k, v) -> v == null ? 1.0D : v + 1.0D);
}
}
// 分别获取前两个品牌、分类
List<String> sortNameList = com.yoho.search.base.utils.CollectionUtils.getSortedKeys(sortNameMap, false)
.stream().limit(SMART_SUGGESTION_TERM_COUNT - 1).collect(Collectors.toList());
List<String> brandNameList = com.yoho.search.base.utils.CollectionUtils.getSortedKeys(brandMap, false)
.stream().limit(SMART_SUGGESTION_TERM_COUNT - 1).collect(Collectors.toList());
List<String> keywords = new ArrayList<>();
keywords.addAll(sortNameList);
keywords.addAll(brandNameList);
return keywords.isEmpty() ? "" : keywords.stream().collect(Collectors.joining(" "));
}
/**
* 根据SKN到productindex索引获取搜索结果，然后聚合出推荐词列表。
*
* @param queryWord 搜索的关键词
* @return 推荐词列表
*/
private String aggKeywordsBySkns(String queryWord) {
String sknStr = queryWord.toLowerCase().trim();
if (sknStr.startsWith("skn")) {
sknStr = sknStr.substring("skn".length()).trim();
}
List<Integer> sknList = searchCommonHelper.getQuerySknList(sknStr);
if (CollectionUtils.isEmpty(sknList)) {
return null;
}
int size = sknList.size() > SMART_SUGGESTION_COUNT_LIMIT ? SMART_SUGGESTION_COUNT_LIMIT : sknList.size();
SearchParam productIndexSearchParam = new SearchParam();
productIndexSearchParam.setSize(size);
productIndexSearchParam.setFiter(QueryBuilders.termsQuery("productSkn", sknList));
// 缓存和列表的很难命中 所以单独自己调 而没有走ProductListServiceImpl的代码
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(indexName, productIndexSearchParam);
if (jsonObject != null) {
CACHE_MATCH_REQUEST.info("match cache for product list search recommendation by skns, sknList = {}.", sknList);
return jsonObject.getString("aggKeyword");
}
SearchResult productIndexSearchResult = searchCommonService.doSearch(indexName, productIndexSearchParam);
if (productIndexSearchResult == null) {
return null;
}
List<Map<String, Object>> productList = productIndexSearchResult.getResultList().stream().map(map -> searchServiceHelper.getProductMap(map)).collect(Collectors.toList());
String aggKeyword = aggKeywordsByProductList(productList);
jsonObject = new JSONObject();
jsonObject.put("aggKeyword", aggKeyword);
return aggKeyword;
}
private boolean isQuerySkn(String queryWord) {
String checkStr = queryWord.toLowerCase().trim();
if (checkStr.startsWith("skn")) {
checkStr = checkStr.substring("skn".length()).trim();
}
return searchCommonHelper.isQuerySkn(checkStr);
}
private JSONObject defaultSuggestRecommendation() {
JSONObject emptySuggestResult = new JSONObject();
emptySuggestResult.put("terms_suggestion", DEFAULT_SUGGEST_TIPS);
return emptySuggestResult;
}
/**
* 根据搜索的关键词到conversion索引查询关联的关键词。
* 该索引包含了爬虫提取的关键词、品牌的关键词定义和自定义的关键词。
*
* @param queryWord 搜索的关键词
* @return 推荐关键词列表
*/
private String getSuggestConversionDestBySource(String queryWord) {
long begin = System.currentTimeMillis();
logger.info("[func=getSuggestConversionDestBySource][query={}][begin={}]", queryWord, begin);
// 1) 先检查自定义的配置
if (CUSTOM_SUGGEST_CONVERSIONS.containsKey(queryWord.trim().toLowerCase())) {
return CUSTOM_SUGGEST_CONVERSIONS.get(queryWord.trim().toLowerCase());
}
// 2) 根据terms搜索构造搜索请求 要求完全匹配source
SearchParam searchParam = new SearchParam();
searchParam.setQuery(QueryBuilders.termQuery("source.source_keyword", queryWord.trim().toLowerCase()));
searchParam.setPage(1);
searchParam.setSize(1);
searchParam.setFiter(QueryBuilders.termQuery("status", 1));
// 3) 先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_CONVERSION;
JSONObject suggestResult = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggestResult != null) {
CACHE_MATCH_REQUEST.info("match cache for product list search recommendation by conversion, keyword = {}.", queryWord);
return suggestResult.getString("dest");
}
// 4) 调用ES执行搜索
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null) {
return null;
}
String dest = "";
if (CollectionUtils.isNotEmpty(searchResult.getResultList())) {
// 5) 从conversion获取的keyword列表
String source = (String) searchResult.getResultList().get(0).get("source");
dest = (String) searchResult.getResultList().get(0).get("dest");
logger.info("[func=getSuggestConversionDestBySource][source={}][dest={}]", source, dest);
}
// 6) 加入缓存
suggestResult = new JSONObject();
suggestResult.put("dest", dest);
searchCacheService.addJSONObjectToCache(indexName, searchParam, suggestResult);
logger.info("[func=getSuggestConversionDestBySource][query={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return dest;
}
/**
* 根据关键词和时间查询关键词转换关系
*
* @param paramMap 查询参数
* @return 满足查询要求的转换关系
*/
@Override
public SearchApiResult suggestConversionList(Map<String, String> paramMap) {
try {
long begin = System.currentTimeMillis();
logger.info("[func=suggestConversionList][param={}][begin={}]", paramMap, begin);
String queryWord = paramMap.get("query");
int updateTime = StringUtils.isBlank(paramMap.get("updateTime")) ? 0 : Integer.parseInt(paramMap.get("updateTime"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1) {
throw new IllegalArgumentException("分页参数不合法");
}
// 1) 构建ES请求
SearchParam searchParam = new SearchParam();
QueryBuilder queryBuilder = StringUtils.isNotEmpty(queryWord) ? QueryBuilders.matchQuery("source", queryWord) : QueryBuilders.matchAllQuery();
searchParam.setQuery(queryBuilder);
searchParam.setPage(page);
searchParam.setOffset((page - 1) * 10);
searchParam.setSize(10);
if (updateTime > 0) {
searchParam.setFiter(QueryBuilders.rangeQuery("updateTime").gte(updateTime));
}
if (StringUtils.isEmpty(queryWord)) {
// 没有传入query时按照时间降序排序
searchParam.setSortBuilders(Arrays.asList(SortBuilders.fieldSort("updateTime").order(SortOrder.DESC)));
}
// 2) 调用ES查询
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_CONVERSION, searchParam);
if (searchResult == null) {
return null;
}
// 3) 返回结果
Map<String, Object> dataMap = new HashMap<>();
dataMap.put("total", searchResult.getTotal());
dataMap.put("page", searchResult.getPage());
dataMap.put("page_size", searchParam.getSize());
dataMap.put("page_total", searchResult.getTotalPage());
dataMap.put("conversion_list", searchResult.getResultList());
logger.info("[func=suggestConversionList][cost={}]", System.currentTimeMillis() - begin);
return new SearchApiResult().setData(dataMap);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new SearchApiResult().setCode(500).setMessage(e.getMessage()).setData(null);
}
}
}
... ...
... ... @@ -88,7 +88,7 @@ public class SearchSortGroupService implements ISearchSortGroupService, Applicat
// 性别
if (paramMap.containsKey(SearchRequestParams.PARAM_SEARCH_GENDER)) {
String[] genders = paramMap.get(SearchRequestParams.PARAM_SEARCH_GENDER).split(",");
boolFilter.must(QueryBuilders.termQuery("gender", genders));
boolFilter.must(QueryBuilders.termsQuery("gender", genders));
}
// 如果contain_global！=Y，则过滤掉全球购的商品[全球购商品融合需求]
if (!searchCommonHelper.containGlobal(paramMap)) {
... ...
package com.yoho.search.service.servicenew.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.alibaba.fastjson.JSONObject;
import com.yoho.error.event.SearchEvent;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.es.utils.IgnoreSomeException;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.servicenew.ISuggestService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SuggestApiResult;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
... ... @@ -30,485 +32,271 @@ import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import com.yoho.error.event.SearchEvent;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.es.utils.IgnoreSomeException;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.service.SearchDynamicConfigService;
import com.yoho.search.service.service.SearchKeyWordService;
import com.yoho.search.service.servicenew.ISuggestService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SuggestApiResult;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class SuggestServiceImpl implements ISuggestService, ApplicationEventPublisherAware {
private static final Logger logger = LoggerFactory.getLogger(SuggestServiceImpl.class);
private static final Logger CACHE_MATCH_REQUEST = LoggerFactory.getLogger("CACHE_MATCH_REQUEST");
private static final String SUGGEST_PARAM_APPTYPE = "app_type";
private static final String SUGGEST_PARAM_GLOBAL = "contain_global";
// 返回智能搜索词的数量
private static final int SMART_SUGGESTION_TERM_COUNT = 3;
// 返回智能搜索词限制关联商品数量限制
private static final int SMART_SUGGESTION_COUNT_LIMIT = 20;
private static final List<String> DEFAULT_SUGGEST_TIPS = Arrays.asList("潮流", "时尚", "休闲");
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
@Autowired
private SearchKeyWordService searchKeyWordService;
private ApplicationEventPublisher publisher;
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.publisher = applicationEventPublisher;
}
@Override
public SuggestApiResult suggest(Map<String, String> paramMap) {
// 已使用SuggestSearchHandler进行重构
try {
logger.info("[func=suggest][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
// 1)关键参数验证
String keyword = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
if (StringUtils.isBlank(keyword)) {
return new SuggestApiResult().setCode(400).setMessage("关键字[query]参数为空！");
}
keyword = keyword.toLowerCase();
if (keyword.length() >= 30) {
return new SuggestApiResult().setCode(400).setMessage("关键字[query]参数非法！");
}
// 2）构建查询参数
SearchParam searchParam = buildSuggestSearchParam(paramMap);
// 3)先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject suggest = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggest != null) {
CACHE_MATCH_REQUEST.info("match cache , url is :/suggest.json?" + HttpServletRequestUtils.genParamString(paramMap));
return new SuggestApiResult().setCode(200).setMessage("suggest List.").setSuggest(suggest);
}
// 4)进行ES检索
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
// 将searchResult转化为map返回--需要把aggregation转化为需要的结构
if (searchResult == null) {
return null;
}
if (CollectionUtils.isEmpty(searchResult.getResultList())) {
// 5) suggest 纠错
SearchResult newSearchResult = getSpellCorrectResult(paramMap);
if (newSearchResult != null) {
searchResult = newSearchResult;
}
}
suggest = new JSONObject();
List<Map<String, Object>> itemList = new ArrayList<Map<String, Object>>();
String countEsField = getCountField(paramMap);
for (Map<String, Object> map : searchResult.getResultList()) {
Map<String, Object> item = new HashMap<String, Object>();
item.put("item", map.get("keyword"));
item.put("frequency", map.get(countEsField));
item.put("type", map.get("type"));
itemList.add(item);
}
suggest.put("items", itemList);
searchCacheService.addJSONObjectToCache(indexName, searchParam, suggest);
return new SuggestApiResult().setCode(200).setMessage("suggest List.").setSuggest(suggest);
} catch (Exception e) {
logger.error(e.getMessage(), e);
publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_SUGGEST.getEventName(), EventReportEnum.SEARCHCONTROLLER_SUGGEST.getFunctionName(),
EventReportEnum.SEARCHCONTROLLER_SUGGEST.getMoudleName(), "exception", IgnoreSomeException.filterSomeException(e), null));
return new SuggestApiResult().setCode(500).setMessage(e.getMessage()).setSuggest(null);
}
}
private String getCountField(Map<String, String> paramMap) {
// suggest 支持PC、APP、BLK走不同的count字段 字段同SuggestIndexBO里保持一直
if (paramMap.containsKey(SUGGEST_PARAM_APPTYPE) && "1".equals(paramMap.get(SUGGEST_PARAM_APPTYPE))) {
return "countForBlk";
} else if (paramMap.containsKey(SUGGEST_PARAM_GLOBAL) && "Y".equals(paramMap.get(SUGGEST_PARAM_GLOBAL))) {
return "countForApp";
} else {
return "count";
}
}
private SearchResult getSpellCorrectResult(Map<String, String> paramMap) {
String keyword = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
String newKeyword = getSpellCorrectKeyword(keyword);
if (StringUtils.isEmpty(newKeyword)) {
return null;
}
try {
logger.info("Switch the suggest keyword from [{}] to [{}].", keyword, newKeyword);
Map<String, String> newParamMap = new HashMap<>(paramMap.size());
newParamMap.putAll(paramMap);
newParamMap.put(SearchRequestParams.PARAM_SEARCH_KEYWORD, newKeyword);
SearchParam newSearchParam = buildSuggestSearchParam(newParamMap);
return searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, newSearchParam);
} catch (Exception e) {
logger.error("Get new suggest result by keyword [" + newKeyword + "] failed!", e);
return null;
}
}
private SearchParam buildSuggestSearchParam(Map<String, String> paramMap) {
// 2）构建查询参数
String keyword = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
SearchParam searchParam = new SearchParam();
int size = SearchRequestParams.RESULT_SIZE_SUGGEST;
if (paramMap.containsKey(SearchRequestParams.PARAM_SEARCH_SIZE)) {
size = Integer.parseInt(paramMap.get(SearchRequestParams.PARAM_SEARCH_SIZE));
}
if (size > 10) {
size = 10;
}
QueryBuilder query = QueryBuilders.boolQuery().should(QueryBuilders.prefixQuery("keyword", keyword)).should(QueryBuilders.prefixQuery("keyword.keyword_pinyin", keyword))
.should(QueryBuilders.prefixQuery("keyword.keyword_jianpin", keyword));
searchParam.setQuery(query);
searchParam.setPage(1);
searchParam.setSize(size);
// count数量要>=1
String countEsField = getCountField(paramMap);
searchParam.setFiter(QueryBuilders.rangeQuery(countEsField).gte(2));
// 3）设置排序字段
List<SortBuilder> sortBuilders = new ArrayList<SortBuilder>();
sortBuilders.add(SortBuilders.fieldSort("weight").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(countEsField).order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
return searchParam;
}
private String getSpellCorrectKeyword(String keyword) {
try {
SearchParam suggestSearchParam = new SearchParam();
suggestSearchParam.setSize(0);
suggestSearchParam.setSuggestionBuilder(SuggestBuilders.termSuggestion("keyword_suggestion").text(keyword).field("keyword").size(1));
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, suggestSearchParam);
if (searchResult != null && searchResult.getSuggestionMap() != null && searchResult.getSuggestionMap().get("keyword_suggestion") != null) {
TermSuggestion suggestion = (TermSuggestion) searchResult.getSuggestionMap().get("keyword_suggestion");
List<TermSuggestion.Entry> entries = suggestion.getEntries();
if (CollectionUtils.isNotEmpty(entries)) {
List<TermSuggestion.Entry.Option> entryList = entries.get(0).getOptions();
if (CollectionUtils.isNotEmpty(entryList)) {
TermSuggestion.Entry.Option option = entryList.get(0);
return option.getText().string();
}
}
}
return null;
} catch (Exception e) {
logger.error("Get suggestion by keyword [" + keyword + "] failed!", e);
return null;
}
}
/**
* 根据query词获取term建议和phrase建议。 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
*
* @param paramMap
* 商品列表搜索参数
* @return 包括term建议和phrase建议。
*/
public JSONObject suggestTips(Map<String, String> paramMap) {
// 1) 对query进行判断 为空时不处理
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
if (StringUtils.isEmpty(queryWord)) {
return null;
}
try {
// 2) 先调用suggest获取搜索提示词 只要能匹配到20%的term即可
JSONObject suggestTipResult = suggestTipsBySuggestIndex(paramMap);
if (suggestTipResult == null) {
// 2.1) 可能是ES发生异常 如果搜索不到相关是应该是集合为空而不是对象为null
return null;
}
List<String> resultTerms = (List<String>) suggestTipResult.get("terms_suggestion");
if (CollectionUtils.isNotEmpty(resultTerms)) {
// 2.2) 如果能直接从suggest获取到则直接返回
return suggestTipResult;
}
// 3) 通过conversion转换后再到suggest获取提示词
suggestTipResult = suggestTipsByConversionIndex(paramMap);
if (suggestTipResult != null && CollectionUtils.isNotEmpty((List<String>) suggestTipResult.get("terms_suggestion"))) {
return suggestTipResult;
}
return defaultSuggestTips();
} catch (Exception e) {
logger.error("[func=suggestTips]Get suggestion by keyword [" + queryWord + "] failed!", e);
return null;
}
}
public JSONObject suggestTipsBySuggestIndex(Map<String, String> paramMap) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
long begin = System.currentTimeMillis();
logger.info("[func=suggestTipsBySuggestIndex][query={}][begin={}]", queryWord, begin);
// 1) 先对query进行分词
List<String> terms = searchKeyWordService.getAnalyzeTerms(queryWord, "ik_smart", true);
if (CollectionUtils.isEmpty(terms)) {
return null;
}
Set<String> termSet = terms.stream().collect(Collectors.toSet());
logger.info("[func=suggestTipsBySuggestIndex][termSet={}]", termSet);
// 2) 根据terms搜索构造搜索请求
final String countField = getCountField(paramMap);
SearchParam searchParam = new SearchParam();
// 2.1) 对于suggest的multi-fields至少要有一个字段匹配到20% 匹配打分为常量1
MultiMatchQueryBuilder queryBuilder = QueryBuilders
.multiMatchQuery(queryWord.trim().toLowerCase(), "keyword", "keyword.keyword_ik", "keyword.keyword_pinyin", "keyword.keyword_jianpin", "keyword.keyword_lowercase")
.analyzer("ik_smart").type(MultiMatchQueryBuilder.Type.BEST_FIELDS).operator(MatchQueryBuilder.Operator.OR).minimumShouldMatch("20%");
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.constantScoreQuery(queryBuilder));
for (String term : termSet) {
// 2.2) 对于完全匹配Term的加1分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("standardKeyword", CharUtils.standardized(term)), ScoreFunctionBuilders.weightFactorFunction(1));
// 2.3) 对于匹配到一个Term的加2分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("keyword.keyword_ik", term), ScoreFunctionBuilders.weightFactorFunction(2));
}
functionScoreQueryBuilder.boostMode(CombineFunction.SUM);
searchParam.setQuery(functionScoreQueryBuilder);
searchParam.setPage(1);
searchParam.setSize(SMART_SUGGESTION_TERM_COUNT);
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.rangeQuery(countField).gte(SMART_SUGGESTION_COUNT_LIMIT));
boolFilter.mustNot(QueryBuilders.termQuery("standardKeyword", CharUtils.standardized(queryWord)));
searchParam.setFiter(boolFilter);
// 2.4) 按照得分、权重、数量的规则降序排序
List<SortBuilder> sortBuilders = new ArrayList<>(3);
sortBuilders.add(SortBuilders.fieldSort("_score").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort("weight").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(countField).order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
// 3) 先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject suggestResult = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggestResult != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion, keyword = {}.", queryWord);
return suggestResult;
}
// 4) 调用ES执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, searchParam);
if (searchResult == null) {
return null;
}
// 5) 加入缓存
suggestResult = new JSONObject();
List<String> resultTerms = searchResult.getResultList().stream().map(map -> (String) map.get("keyword")).collect(Collectors.toList());
suggestResult.put("terms_suggestion", resultTerms);
searchCacheService.addJSONObjectToCache(indexName, searchParam, suggestResult);
logger.info("[func=suggestTipsBySuggestIndex][query={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return suggestResult;
}
private JSONObject suggestTipsByConversionIndex(Map<String, String> paramMap) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
long begin = System.currentTimeMillis();
logger.info("[func=suggestTipsByConversionIndex][query={}][begin={}]", queryWord, begin);
// 1) 判断是否支持从conversion获取推荐词
if (!searchDynamicConfigService.isSearchSuggestionFromConversionOpen()) {
return null;
}
// 2) 异步上报query关键词 用于统计和分析
searchKeyWordService.recordSuggestTip(queryWord);
// 3) 从conversion索引中获取转换后的关键词列表
String dest = getSuggestConversionDestBySource(queryWord);
if (StringUtils.isEmpty(dest)) {
return null;
}
// 4) 查询suggest索引 过滤和排序
// 4.1) 根据查询条件获得不同的count字段
final String countFiled = getCountField(paramMap);
SearchParam searchParam = new SearchParam();
List<String> keywordsInDest = Arrays.stream(dest.split(",")).map(String::toLowerCase).map(String::trim).collect(Collectors.toList());
keywordsInDest.remove(queryWord);
// 4.2) 设置keyword列表
QueryBuilder queryBuilder = QueryBuilders.termsQuery("keyword.keyword_lowercase", keywordsInDest);
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder);
// 4.3) 确保品类的关键词排在前面
functionScoreQueryBuilder.add(QueryBuilders.termQuery("type", 2), ScoreFunctionBuilders.weightFactorFunction(keywordsInDest.size() + 1));
// 4.4) 根据顺序从高到低加分
for (int index = 0; index < keywordsInDest.size(); index++) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("keyword.keyword_lowercase", keywordsInDest.get(index)),
ScoreFunctionBuilders.weightFactorFunction(keywordsInDest.size() - index));
}
searchParam.setQuery(functionScoreQueryBuilder);
searchParam.setPage(1);
searchParam.setSize(SMART_SUGGESTION_TERM_COUNT);
// 4.5) 过滤关联的商品数量小于20的关键词
searchParam.setFiter(QueryBuilders.rangeQuery(countFiled).gte(SMART_SUGGESTION_COUNT_LIMIT));
// 5) 先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (jsonObject != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion conversion, keyword = {}.", queryWord);
return jsonObject;
}
// 6) 调用ES获取结果
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null) {
return null;
}
// 7) 将结果放入缓存
jsonObject = new JSONObject();
List<String> resultTerms = searchResult.getResultList().stream().map(map -> (String) map.get("keyword")).collect(Collectors.toList());
jsonObject.put("terms_suggestion", resultTerms);
searchCacheService.addJSONObjectToCache(indexName, searchParam, jsonObject);
logger.info("[func=suggestTipsByConversionIndex][query={}][cost={}]", queryWord, System.currentTimeMillis() - /**/begin);
return jsonObject;
}
private String getSuggestConversionDestBySource(String queryWord) {
long begin = System.currentTimeMillis();
logger.info("[func=getSuggestConversionDestBySource][query={}][begin={}]", queryWord, begin);
// 2) 根据terms搜索构造搜索请求
SearchParam searchParam = new SearchParam();
MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("source", queryWord);
searchParam.setQuery(queryBuilder);
searchParam.setPage(1);
searchParam.setSize(1);
searchParam.setFiter(QueryBuilders.termQuery("status", 1));
// 3) 先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_CONVERSION;
JSONObject suggestResult = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggestResult != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms conversion, keyword = {}.", queryWord);
return suggestResult.getString("dest");
}
// 4) 调用ES执行搜索
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
if (searchResult == null) {
return null;
}
String dest = "";
if (CollectionUtils.isNotEmpty(searchResult.getResultList())) {
// 5) 从conversion获取的keyword列表
dest = (String) searchResult.getResultList().get(0).get("dest");
logger.info("[func=getSuggestConversionDestBySource][source={}][dest={}]", searchResult.getResultList().get(0).get("source"), dest);
}
// 6) 加入缓存
suggestResult = new JSONObject();
suggestResult.put("dest", dest);
searchCacheService.addJSONObjectToCache(indexName, searchParam, suggestResult);
logger.info("[func=getSuggestConversionDestBySource][query={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return dest;
}
private JSONObject defaultSuggestTips() {
JSONObject defSuggestResult = new JSONObject();
defSuggestResult.put("terms_suggestion", DEFAULT_SUGGEST_TIPS);
return defSuggestResult;
}
/**
* 根据关键词和时间查询关键词转换关系
*
* @param paramMap
* 查询参数
* @return 满足查询要求的转换关系
*/
@Override
public SearchApiResult suggestConversion(Map<String, String> paramMap) {
try {
long begin = System.currentTimeMillis();
logger.info("[func=suggestConversion][param={}][begin={}]", paramMap, begin);
String queryWord = paramMap.get("query");
int updateTime = StringUtils.isBlank(paramMap.get("updateTime")) ? 0 : Integer.parseInt(paramMap.get("updateTime"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1) {
throw new IllegalArgumentException("分页参数不合法");
}
// 1) 构建ES请求
SearchParam searchParam = new SearchParam();
QueryBuilder queryBuilder = StringUtils.isNotEmpty(queryWord) ? QueryBuilders.matchQuery("source", queryWord) : QueryBuilders.matchAllQuery();
searchParam.setQuery(queryBuilder);
searchParam.setPage(page);
searchParam.setOffset((page - 1) * 10);
searchParam.setSize(10);
if (updateTime > 0) {
searchParam.setFiter(QueryBuilders.rangeQuery("updateTime").gte(updateTime));
}
// 2) 调用ES查询
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_CONVERSION, searchParam);
if (searchResult == null) {
return null;
}
// 3) 返回结果
Map<String, Object> dataMap = new HashMap<>();
dataMap.put("total", searchResult.getTotal());
dataMap.put("page", searchResult.getPage());
dataMap.put("page_size", searchParam.getSize());
dataMap.put("page_total", searchResult.getTotalPage());
dataMap.put("conversion_list", searchResult.getResultList());
logger.info("[func=suggestConversion][cost={}]", System.currentTimeMillis() - begin);
return new SearchApiResult().setData(dataMap);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new SearchApiResult().setCode(500).setMessage(e.getMessage()).setData(null);
}
}
private static final Logger logger = LoggerFactory.getLogger(SuggestServiceImpl.class);
private static final Logger CACHE_MATCH_REQUEST = LoggerFactory.getLogger("CACHE_MATCH_REQUEST");
private static final String SUGGEST_PARAM_APPTYPE = "app_type";
private static final String SUGGEST_PARAM_GLOBAL = "contain_global";
private static final int VALID_STATUS = 1;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchCacheService searchCacheService;
private ApplicationEventPublisher publisher;
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.publisher = applicationEventPublisher;
}
@Override
public SuggestApiResult suggest(Map<String, String> paramMap) {
// 已使用SuggestSearchHandler进行重构
try {
logger.info("[func=suggest][param={}][begin={}]", paramMap.toString(), System.currentTimeMillis());
// 1)关键参数验证
String keyword = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
if (StringUtils.isBlank(keyword)) {
return new SuggestApiResult().setCode(400).setMessage("关键字[query]参数为空！");
}
keyword = keyword.toLowerCase();
if (keyword.length() > 30) {
return new SuggestApiResult().setCode(400).setMessage("关键字[query]参数非法！");
}
// 2）构建查询参数
SearchParam searchParam = buildSuggestSearchParam(paramMap);
// 3)先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject suggest = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggest != null) {
CACHE_MATCH_REQUEST.info("match cache , url is :/suggest.json?" + HttpServletRequestUtils.genParamString(paramMap));
return new SuggestApiResult().setCode(200).setMessage("suggest List.").setSuggest(suggest);
}
// 4)进行ES检索
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
// 将searchResult转化为map返回--需要把aggregation转化为需要的结构
if (searchResult == null) {
return null;
}
if (CollectionUtils.isEmpty(searchResult.getResultList())) {
// 5) suggest 纠错
SearchResult newSearchResult = getSpellCorrectResult(paramMap);
if (newSearchResult != null) {
searchResult = newSearchResult;
}
}
suggest = new JSONObject();
List<Map<String, Object>> itemList = new ArrayList<Map<String, Object>>();
String countEsField = getCountField(paramMap);
for (Map<String, Object> map : searchResult.getResultList()) {
Map<String, Object> item = new HashMap<String, Object>();
item.put("item", map.get("keyword"));
item.put("frequency", map.get(countEsField));
item.put("type", map.get("type"));
itemList.add(item);
}
suggest.put("items", itemList);
searchCacheService.addJSONObjectToCache(indexName, searchParam, suggest);
return new SuggestApiResult().setCode(200).setMessage("suggest List.").setSuggest(suggest);
} catch (Exception e) {
logger.error(e.getMessage(), e);
publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_SUGGEST.getEventName(), EventReportEnum.SEARCHCONTROLLER_SUGGEST.getFunctionName(),
EventReportEnum.SEARCHCONTROLLER_SUGGEST.getMoudleName(), "exception", IgnoreSomeException.filterSomeException(e), null));
return new SuggestApiResult().setCode(500).setMessage(e.getMessage()).setSuggest(null);
}
}
private SearchParam buildSuggestSearchParam(Map<String, String> paramMap) {
// 2）构建查询参数
String keyword = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
SearchParam searchParam = new SearchParam();
int size = SearchRequestParams.RESULT_SIZE_SUGGEST;
if (paramMap.containsKey(SearchRequestParams.PARAM_SEARCH_SIZE)) {
size = Integer.parseInt(paramMap.get(SearchRequestParams.PARAM_SEARCH_SIZE));
}
if (size > 10) {
size = 10;
}
QueryBuilder query = QueryBuilders.boolQuery().should(QueryBuilders.prefixQuery("keyword", keyword)).should(QueryBuilders.prefixQuery("keyword.keyword_pinyin", keyword))
.should(QueryBuilders.prefixQuery("keyword.keyword_jianpin", keyword));
searchParam.setQuery(query);
searchParam.setPage(1);
searchParam.setSize(size);
// count数量要>=2
final String countEsField = getCountField(paramMap);
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termQuery("status", VALID_STATUS));
boolFilter.must(QueryBuilders.rangeQuery(countEsField).gte(2));
searchParam.setFiter(boolFilter);
// 3）设置排序字段
List<SortBuilder> sortBuilders = new ArrayList<SortBuilder>();
sortBuilders.add(SortBuilders.fieldSort("weight").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(countEsField).order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
return searchParam;
}
/**
* 根据搜索条件判断使用到的suggest索引的count字段名。
*
* @param paramMap 搜索条件
* @return count字段名
*/
@Override
public String getCountField(Map<String, String> paramMap) {
// suggest 支持PC、APP、BLK走不同的count字段 字段同SuggestIndexBO里保持一直
if (paramMap.containsKey(SUGGEST_PARAM_APPTYPE) && "1".equals(paramMap.get(SUGGEST_PARAM_APPTYPE))) {
return "countForBlk";
} else if (paramMap.containsKey(SUGGEST_PARAM_GLOBAL) && "Y".equals(paramMap.get(SUGGEST_PARAM_GLOBAL))) {
return "countForApp";
} else {
return "count";
}
}
private SearchResult getSpellCorrectResult(Map<String, String> paramMap) {
String keyword = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
String newKeyword = getSpellingCorrectKeyword(keyword);
if (StringUtils.isEmpty(newKeyword)) {
return null;
}
try {
logger.info("Switch the suggest keyword from [{}] to [{}].", keyword, newKeyword);
Map<String, String> newParamMap = new HashMap<>(paramMap.size());
newParamMap.putAll(paramMap);
newParamMap.put(SearchRequestParams.PARAM_SEARCH_KEYWORD, newKeyword);
SearchParam newSearchParam = buildSuggestSearchParam(newParamMap);
return searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, newSearchParam);
} catch (Exception e) {
logger.error("Get new suggest result by keyword [" + newKeyword + "] failed!", e);
return null;
}
}
/**
* 到suggest索引进行拼写纠错处理。
*
* @param keyword 用户输入的关键词
* @return 纠错后的关键词。
*/
@Override
public String getSpellingCorrectKeyword(String keyword) {
try {
SearchParam suggestSearchParam = new SearchParam();
suggestSearchParam.setSize(0);
suggestSearchParam.setSuggestionBuilder(SuggestBuilders.termSuggestion("keyword_suggestion")
.text(keyword.trim().toLowerCase()).field("keyword.keyword_lowercase").size(1));
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(ISearchConstants.INDEX_NAME_SUGGEST, suggestSearchParam);
if (jsonObject != null) {
CACHE_MATCH_REQUEST.info("match cache for suggest spelling correct, keyword={}", keyword);
return jsonObject.getString("spellingCorrentKeyword");
}
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, suggestSearchParam);
if (searchResult == null) {
return null;
}
String spellingCorrentKeyword = "";
if (searchResult.getSuggestionMap() != null && searchResult.getSuggestionMap().get("keyword_suggestion") != null) {
TermSuggestion suggestion = (TermSuggestion) searchResult.getSuggestionMap().get("keyword_suggestion");
List<TermSuggestion.Entry> entries = suggestion.getEntries();
if (CollectionUtils.isNotEmpty(entries)) {
List<TermSuggestion.Entry.Option> entryList = entries.get(0).getOptions();
if (CollectionUtils.isNotEmpty(entryList)) {
TermSuggestion.Entry.Option option = entryList.get(0);
spellingCorrentKeyword = option.getText().string();
}
}
}
jsonObject = new JSONObject();
jsonObject.put("spellingCorrentKeyword", spellingCorrentKeyword);
searchCacheService.addJSONObjectToCache(ISearchConstants.INDEX_NAME_SUGGEST, suggestSearchParam, jsonObject);
return spellingCorrentKeyword;
} catch (Exception e) {
logger.error("Get spelling correct keyword by [" + keyword + "] failed!", e);
return null;
}
}
/**
* 根据关键词查询suggest索引
*
* @param paramMap 查询参数
* @return 满足查询要求的建议词
*/
@Override
public SearchApiResult suggestList(Map<String, String> paramMap) {
try {
long begin = System.currentTimeMillis();
logger.info("[func=suggestList][param={}][begin={}]", paramMap, begin);
String queryWord = paramMap.get("query");
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1) {
throw new IllegalArgumentException("分页参数不合法");
}
// 1) 构建ES请求
SearchParam searchParam = new SearchParam();
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
if (StringUtils.isNotEmpty(queryWord)) {
// 是否精确匹配
String accurateQuery = paramMap.get("accurate");
queryBuilder = "Y".equalsIgnoreCase(accurateQuery) ? QueryBuilders.matchQuery("standardKeyword", CharUtils.standardized(queryWord)) : QueryBuilders.matchQuery("keyword.keyword_ik", queryWord);
}
searchParam.setQuery(queryBuilder);
searchParam.setPage(page);
searchParam.setOffset((page - 1) * 10);
searchParam.setSize(10);
// 2) 调用ES查询
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, searchParam);
if (searchResult == null) {
return null;
}
// 3) 返回结果
Map<String, Object> dataMap = new HashMap<>();
dataMap.put("total", searchResult.getTotal());
dataMap.put("page", searchResult.getPage());
dataMap.put("page_size", searchParam.getSize());
dataMap.put("page_total", searchResult.getTotalPage());
dataMap.put("suggest_list", searchResult.getResultList());
logger.info("[func=suggestList][cost={}]", System.currentTimeMillis() - begin);
return new SearchApiResult().setData(dataMap);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new SearchApiResult().setCode(500).setMessage(e.getMessage()).setData(null);
}
}
}
... ...
... ... @@ -62,7 +62,6 @@ public class SearchRequestParams {
public static final String PARAM_SEARCH_ACT_REC = "act_rec"; // 是否推荐
public static final String PARAM_SEARCH_ACT_STATUS = "act_status"; // 状态
public static final String PARAM_SEARCH_GLOBAL_DESCORE_BRAND = "isAdjustBrandScore"; // 是否要对全局降分品牌进行降分
public static final String PARAM_SEARCH_GLOBAL_FILTER_BRAND = "isFilterDescoreBrand"; // 是否过滤掉全局降分品牌的商品
public static final String PARAM_TERM_SUGGESTION = "needSuggestion"; //当商品数量小于特定数值（如20条）是否返回分词推荐
... ...
package com.yoho.search.service.vo;
import java.util.ArrayList;
import java.util.List;
public class GoodProductBO {
private String smallSortId;
private List<String> productSkn;
public GoodProductBO(String smallSortId, List<String> productSkn) {
super();
this.smallSortId = smallSortId;
this.productSkn = productSkn;
}
public String getSmallSortId() {
return smallSortId;
}
public List<String> getProductSkn() {
return productSkn;
}
public List<String> randomGetProductSkn(int count) {
List<String> results = new ArrayList<String>();
if (productSkn == null || productSkn.isEmpty()) {
return results;
}
for (int i = 0; i < count; i++) {
if (!productSkn.isEmpty()) {
results.add(productSkn.remove(0));
}
}
return results;
}
}
... ...
package com.yoho.search.service.vo;
import org.elasticsearch.search.sort.SortOrder;
public class SearchSort {
private String sortField;
private SortOrder sortOrder;
public SearchSort(String sortField, SortOrder sortOrder) {
super();
this.sortField = sortField;
this.sortOrder = sortOrder;
}
public String getSortField() {
return sortField;
}
public SortOrder getSortOrder() {
return sortOrder;
}
public boolean asc() {
if (sortOrder == null) {
return false;
}
return SortOrder.ASC.equals(sortOrder);
}
}
... ...
... ... @@ -30,7 +30,7 @@ search.index.translog.flush_threshold_ops=5000
#search
search.minimum.should.match=75%
search.operator=and
search.default.field=brandName.brandName_lowercase^2500,smallSort^1000,smallSort.smallSort_pinyin^1000,middleSort^950,middleSort.middleSort_pinyin^950,maxSort^900,maxSort.maxSort_pinyin^900,brandName^900,brandNameCn^850,brandNameCn.brandNameCn_pinyin^850,brandNameEn^800,brandDomain^800,specialSearchField^700,productName.productName_ansj^300,standardOnlyNames.standardOnlyNames_pinyin^250,standardOnlyNames.standardOnlyNames_ansj^250,productKeyword^50,brandKeyword^30,genderS^20,salesPhrase^50,marketPhrase^50,searchField_ansj^10,searchField,productSkn.productSkn_ansj
search.default.field=brandName.brandName_lowercase^4000,smallSort^1000,smallSort.smallSort_pinyin^1000,middleSort^950,middleSort.middleSort_pinyin^950,maxSort^900,maxSort.maxSort_pinyin^900,brandName^900,brandNameCn^850,brandNameCn.brandNameCn_pinyin^850,brandNameEn^800,brandDomain^800,specialSearchField^700,productName.productName_ansj^300,standardOnlyNames.standardOnlyNames_pinyin^250,standardOnlyNames.standardOnlyNames_ansj^250,productKeyword^50,brandKeyword^30,genderS^20,salesPhrase^50,marketPhrase^50,searchField_ansj^10,searchField,productSkn.productSkn_ansj,productSkn.productSkn_ik
search.script.score=_score+doc['sortWeight'].value*0.003+(100-doc['breakingRate'].value)/100 * doc['salesWithDateDiff'].value/pow((now-doc['shelveTime'].value)/3600+2,1.8)
search.script.lang=groovy
... ...
... ... @@ -30,7 +30,7 @@ search.index.translog.flush_threshold_ops=${search.index.translog.flush_threshol
#search
search.minimum.should.match=75%
search.operator=and
search.default.field=brandName.brandName_lowercase^2500,smallSort^1000,smallSort.smallSort_pinyin^1000,middleSort^950,middleSort.middleSort_pinyin^950,maxSort^900,maxSort.maxSort_pinyin^900,brandName^900,brandNameCn^850,brandNameCn.brandNameCn_pinyin^850,brandNameEn^800,brandDomain^800,specialSearchField^700,productName.productName_ansj^300,standardOnlyNames.standardOnlyNames_pinyin^250,standardOnlyNames.standardOnlyNames_ansj^250,productKeyword^50,brandKeyword^30,genderS^20,salesPhrase^50,marketPhrase^50,searchField_ansj^10,searchField,productSkn.productSkn_ansj
search.default.field=brandName.brandName_lowercase^4000,smallSort^1000,smallSort.smallSort_pinyin^1000,middleSort^950,middleSort.middleSort_pinyin^950,maxSort^900,maxSort.maxSort_pinyin^900,brandName^900,brandNameCn^850,brandNameCn.brandNameCn_pinyin^850,brandNameEn^800,brandDomain^800,specialSearchField^700,productName.productName_ansj^300,standardOnlyNames.standardOnlyNames_pinyin^250,standardOnlyNames.standardOnlyNames_ansj^250,productKeyword^50,brandKeyword^30,genderS^20,salesPhrase^50,marketPhrase^50,searchField_ansj^10,searchField,productSkn.productSkn_ansj,productSkn.productSkn_ik
search.script.score=_score+doc['sortWeight'].value*0.003+(100-doc['breakingRate'].value)/100 * doc['salesWithDateDiff'].value/pow((now-doc['shelveTime'].value)/3600+2,1.8)
search.script.lang=groovy
... ...