yoho-search / yoho-search-service · Commits

Go to a project

Authored by Gino Zhang
Commit e581531c37acc31fd17fbadc55249d1398e3bc36 1 parent 1234a204

搜索提示支持：1.根据搜索结果聚合出提示词； 2.支持代码自定义的转换关系，如it

Inline Side-by-side
Showing 7 changed files with 211 additions and 10 deletions
  1 +package com.yoho.search.service.aggregations.impls;
  2 +
  3 +import com.yoho.search.core.es.agg.AbstractAggregation;
  4 +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
  5 +import org.elasticsearch.search.aggregations.Aggregation;
  6 +import org.elasticsearch.search.aggregations.AggregationBuilders;
  7 +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
  8 +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
  9 +
  10 +import java.util.ArrayList;
  11 +import java.util.Iterator;
  12 +import java.util.List;
  13 +import java.util.Map;
  14 +
  15 +/**
  16 + * 单个字段进行聚合的抽象类
  17 + */
  18 +public abstract class AbstractSingleFieldAggregation extends AbstractAggregation {
  19 +
  20 + private int count;
  21 +
  22 + public AbstractSingleFieldAggregation(int count) {
  23 + this.count = count;
  24 + }
  25 +
  26 + @Override
  27 + public AbstractAggregationBuilder getBuilder() {
  28 + return AggregationBuilders.terms(aggName()).field(getField()).size(count);
  29 + }
  30 +
  31 + @Override
  32 + public Object getAggregationResponseMap(Map<String, Aggregation> aggMaps) {
  33 + MultiBucketsAggregation aggregation = this.getAggregation(aggMaps);
  34 + if (aggregation == null) {
  35 + return null;
  36 + }
  37 +
  38 + List<String> valueList = new ArrayList<String>();
  39 + Iterator<? extends Bucket> itSizeAgg = aggregation.getBuckets().iterator();
  40 + while (itSizeAgg.hasNext()) {
  41 + Bucket ltSize = itSizeAgg.next();
  42 + valueList.add(ltSize.getKeyAsString());
  43 + }
  44 +
  45 + return valueList;
  46 + }
  47 +
  48 + abstract protected String getField();
  49 +}
  1 +package com.yoho.search.service.aggregations.impls;
  2 +
  3 +public class BrandNameAggregation extends AbstractSingleFieldAggregation {
  4 +
  5 + public BrandNameAggregation(int count) {
  6 + super(count);
  7 + }
  8 +
  9 + @Override
  10 + public String aggName() {
  11 + return "brandNameAgg";
  12 + }
  13 +
  14 + @Override
  15 + protected String getField() {
  16 + return "brandName.brandName_keyword";
  17 + }
  18 +}
  1 +package com.yoho.search.service.aggregations.impls;
  2 +
  3 +public class SmallSortNameAggregation extends AbstractSingleFieldAggregation {
  4 +
  5 + public SmallSortNameAggregation(int count) {
  6 + super(count);
  7 + }
  8 +
  9 + @Override
  10 + public String aggName() {
  11 + return "smallSortNameAgg";
  12 + }
  13 +
  14 + @Override
  15 + protected String getField() {
  16 + return "smallSortName";
  17 + }
  18 +}
  1 +package com.yoho.search.service.aggregations.impls;
  2 +
  3 +public class StyleNameAggregation extends AbstractSingleFieldAggregation {
  4 +
  5 + public StyleNameAggregation(int count) {
  6 + super(count);
  7 + }
  8 +
  9 + @Override
  10 + public String aggName() {
  11 + return "styleNameAgg";
  12 + }
  13 +
  14 + @Override
  15 + protected String getField() {
  16 + return "style";
  17 + }
  18 +}
@@ -19,10 +19,11 @@ public interface ISuggestService { @@ -19,10 +19,11 @@ public interface ISuggestService {
19 /** 19 /**
20 * 根据query词获取term建议和phrase建议。 20 * 根据query词获取term建议和phrase建议。
21 * 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。 21 * 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
  22 + * @param searchResult 搜索结果
22 * @param paramMap 搜索参数 23 * @param paramMap 搜索参数
23 * @return 包括term建议和phrase建议。 24 * @return 包括term建议和phrase建议。
24 */ 25 */
25 - JSONObject suggestTips(Map<String, String> paramMap); 26 + JSONObject suggestTips(SearchApiResult searchResult, Map<String, String> paramMap);
26 27
27 /** 28 /**
28 * 根据关键词和时间查询关键词转换关系 29 * 根据关键词和时间查询关键词转换关系
@@ -19,8 +19,6 @@ import com.yoho.search.service.utils.HttpServletRequestUtils; @@ -19,8 +19,6 @@ import com.yoho.search.service.utils.HttpServletRequestUtils;
19 import com.yoho.search.service.utils.SearchApiResultUtils; 19 import com.yoho.search.service.utils.SearchApiResultUtils;
20 import com.yoho.search.service.utils.SearchRequestParams; 20 import com.yoho.search.service.utils.SearchRequestParams;
21 import com.yoho.search.service.vo.SearchApiResult; 21 import com.yoho.search.service.vo.SearchApiResult;
22 -import com.yoho.search.service.vo.SuggestApiResult;  
23 -  
24 import org.apache.commons.lang.StringUtils; 22 import org.apache.commons.lang.StringUtils;
25 import org.elasticsearch.index.query.QueryBuilders; 23 import org.elasticsearch.index.query.QueryBuilders;
26 import org.elasticsearch.search.builder.SearchSourceBuilder; 24 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -65,7 +63,7 @@ public class ProductListServiceImpl implements IProductListService { @@ -65,7 +63,7 @@ public class ProductListServiceImpl implements IProductListService {
65 if (needTermSuggestion(searchResult, paramMap)) { 63 if (needTermSuggestion(searchResult, paramMap)) {
66 // 当商品数量太少或无结果时 支持智能推荐Term搜索 64 // 当商品数量太少或无结果时 支持智能推荐Term搜索
67 JSONObject dataMap = ((JSONObject) searchResult.getData()); 65 JSONObject dataMap = ((JSONObject) searchResult.getData());
68 - dataMap.put("suggestion", suggestService.suggestTips(paramMap)); 66 + dataMap.put("suggestion", suggestService.suggestTips(searchResult, paramMap));
69 } 67 }
70 68
71 return searchResult; 69 return searchResult;
@@ -5,9 +5,13 @@ import com.yoho.error.event.SearchEvent; @@ -5,9 +5,13 @@ import com.yoho.error.event.SearchEvent;
5 import com.yoho.search.base.utils.CharUtils; 5 import com.yoho.search.base.utils.CharUtils;
6 import com.yoho.search.base.utils.EventReportEnum; 6 import com.yoho.search.base.utils.EventReportEnum;
7 import com.yoho.search.base.utils.ISearchConstants; 7 import com.yoho.search.base.utils.ISearchConstants;
  8 +import com.yoho.search.core.es.agg.IAggregation;
8 import com.yoho.search.core.es.model.SearchParam; 9 import com.yoho.search.core.es.model.SearchParam;
9 import com.yoho.search.core.es.model.SearchResult; 10 import com.yoho.search.core.es.model.SearchResult;
10 import com.yoho.search.core.es.utils.IgnoreSomeException; 11 import com.yoho.search.core.es.utils.IgnoreSomeException;
  12 +import com.yoho.search.service.aggregations.impls.BrandNameAggregation;
  13 +import com.yoho.search.service.aggregations.impls.SmallSortNameAggregation;
  14 +import com.yoho.search.service.aggregations.impls.StyleNameAggregation;
11 import com.yoho.search.service.service.SearchCacheService; 15 import com.yoho.search.service.service.SearchCacheService;
12 import com.yoho.search.service.service.SearchCommonService; 16 import com.yoho.search.service.service.SearchCommonService;
13 import com.yoho.search.service.service.SearchDynamicConfigService; 17 import com.yoho.search.service.service.SearchDynamicConfigService;
@@ -23,6 +27,7 @@ import org.elasticsearch.common.lucene.search.function.CombineFunction; @@ -23,6 +27,7 @@ import org.elasticsearch.common.lucene.search.function.CombineFunction;
23 import org.elasticsearch.index.query.*; 27 import org.elasticsearch.index.query.*;
24 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; 28 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
25 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; 29 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
  30 +import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
26 import org.elasticsearch.search.sort.SortBuilder; 31 import org.elasticsearch.search.sort.SortBuilder;
27 import org.elasticsearch.search.sort.SortBuilders; 32 import org.elasticsearch.search.sort.SortBuilders;
28 import org.elasticsearch.search.sort.SortOrder; 33 import org.elasticsearch.search.sort.SortOrder;
@@ -57,6 +62,8 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -57,6 +62,8 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
57 62
58 private static final List<String> DEFAULT_SUGGEST_TIPS = Arrays.asList("潮流", "时尚", "休闲"); 63 private static final List<String> DEFAULT_SUGGEST_TIPS = Arrays.asList("潮流", "时尚", "休闲");
59 64
  65 + private static final Map<String, String> CUSTOM_SUGGEST_CONVERSIONS = new HashMap<>();
  66 +
60 private static final int VALID_STATUS = 1; 67 private static final int VALID_STATUS = 1;
61 68
62 @Autowired 69 @Autowired
@@ -73,6 +80,9 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -73,6 +80,9 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
73 @Override 80 @Override
74 public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { 81 public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
75 this.publisher = applicationEventPublisher; 82 this.publisher = applicationEventPublisher;
  83 +
  84 + // it是一个停用词 所以直接配置转换关系
  85 + CUSTOM_SUGGEST_CONVERSIONS.put("it", ":CHOCOOLATE,izzue,5cm");
76 } 86 }
77 87
78 @Override 88 @Override
@@ -237,10 +247,11 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -237,10 +247,11 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
237 * 根据query词获取term建议和phrase建议。 247 * 根据query词获取term建议和phrase建议。
238 * 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。 248 * 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
239 * 249 *
  250 + * @param searchResult 搜索结果
240 * @param paramMap 商品列表搜索参数 251 * @param paramMap 商品列表搜索参数
241 * @return 包括term建议和phrase建议。 252 * @return 包括term建议和phrase建议。
242 */ 253 */
243 - public JSONObject suggestTips(Map<String, String> paramMap) { 254 + public JSONObject suggestTips(SearchApiResult searchResult, Map<String, String> paramMap) {
244 // 1) 对query进行判断 为空时不处理 255 // 1) 对query进行判断 为空时不处理
245 String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD); 256 String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
246 if (StringUtils.isEmpty(queryWord)) { 257 if (StringUtils.isEmpty(queryWord)) {
@@ -248,7 +259,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -248,7 +259,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
248 } 259 }
249 260
250 try { 261 try {
251 - // 2) 先调用suggest获取搜索提示词 只要能匹配到20%的term即可 262 + // 2) 先调用suggest获取搜索提示词 支持拼写纠错
252 JSONObject suggestTipResult = suggestTipsBySuggestIndex(paramMap, false); 263 JSONObject suggestTipResult = suggestTipsBySuggestIndex(paramMap, false);
253 if (suggestTipResult == null) { 264 if (suggestTipResult == null) {
254 // 2.1) 可能是ES发生异常 如果搜索不到相关是应该是集合为空而不是对象为null 265 // 2.1) 可能是ES发生异常 如果搜索不到相关是应该是集合为空而不是对象为null
@@ -261,6 +272,12 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -261,6 +272,12 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
261 return suggestTipResult; 272 return suggestTipResult;
262 } 273 }
263 274
  275 + // 3) 根据搜索结果提取关键词
  276 + suggestTipResult = suggestTipsByProductListResult(searchResult, paramMap);
  277 + if (suggestTipResult != null) {
  278 + return suggestTipResult;
  279 + }
  280 +
264 // 3) 通过conversion转换后再到suggest获取提示词 281 // 3) 通过conversion转换后再到suggest获取提示词
265 suggestTipResult = suggestTipsByConversionIndex(paramMap); 282 suggestTipResult = suggestTipsByConversionIndex(paramMap);
266 if (suggestTipResult != null && CollectionUtils.isNotEmpty((List<String>) suggestTipResult.get("terms_suggestion"))) { 283 if (suggestTipResult != null && CollectionUtils.isNotEmpty((List<String>) suggestTipResult.get("terms_suggestion"))) {
@@ -274,14 +291,16 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -274,14 +291,16 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
274 } 291 }
275 } 292 }
276 293
277 - public JSONObject suggestTipsBySuggestIndex(final Map<String, String> paramMap, boolean hasChangedKeyword) { 294 + private JSONObject suggestTipsBySuggestIndex(final Map<String, String> paramMap, boolean hasChangedKeyword) {
278 String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD); 295 String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
279 long begin = System.currentTimeMillis(); 296 long begin = System.currentTimeMillis();
280 logger.info("[func=suggestTipsBySuggestIndex][query={}][begin={}]", queryWord, begin); 297 logger.info("[func=suggestTipsBySuggestIndex][query={}][begin={}]", queryWord, begin);
281 // 1) 先对query进行分词 298 // 1) 先对query进行分词
282 List<String> terms = searchKeyWordService.getAnalyzeTerms(queryWord, "ik_smart", true); 299 List<String> terms = searchKeyWordService.getAnalyzeTerms(queryWord, "ik_smart", true);
283 if (CollectionUtils.isEmpty(terms)) { 300 if (CollectionUtils.isEmpty(terms)) {
284 - return null; 301 + JSONObject emptySuggestResult = new JSONObject();
  302 + emptySuggestResult.put("terms_suggestion", new ArrayList<>());
  303 + return emptySuggestResult;
285 } 304 }
286 305
287 Set<String> termSet = terms.stream().collect(Collectors.toSet()); 306 Set<String> termSet = terms.stream().collect(Collectors.toSet());
@@ -392,7 +411,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -392,7 +411,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
392 break; 411 break;
393 } 412 }
394 413
395 - if(!correnctSpellingKeyword.equalsIgnoreCase(item)){ 414 + if (!correnctSpellingKeyword.equalsIgnoreCase(item)) {
396 newResultTerms.add(item); 415 newResultTerms.add(item);
397 } 416 }
398 } 417 }
@@ -403,6 +422,85 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -403,6 +422,85 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
403 return resultTerms; 422 return resultTerms;
404 } 423 }
405 424
  425 + private JSONObject suggestTipsByProductListResult(SearchApiResult searchResult, Map<String, String> paramMap) {
  426 + if (searchResult == null || searchResult.getData() == null) {
  427 + return null;
  428 + }
  429 +
  430 + JSONObject dataMap = (JSONObject) searchResult.getData();
  431 + List<Map<String, Object>> productList = (List<Map<String, Object>>) dataMap.get("product_list");
  432 + if (CollectionUtils.isEmpty(productList)) {
  433 + return null;
  434 + }
  435 +
  436 + String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
  437 + long begin = System.currentTimeMillis();
  438 + logger.info("[func=suggestTipsByProductListResult][query={}][size={}][begin={}]", queryWord, productList.size(), begin);
  439 +
  440 + // 结果里只有品牌的信息 所以需要到ES里获取品类和风格的信息
  441 + List<Integer> sknList = productList.stream().map(map -> (Integer) map.get("product_skn")).filter(skn -> skn != null).collect(Collectors.toList());
  442 + if (CollectionUtils.isEmpty(sknList)) {
  443 + return null;
  444 + }
  445 +
  446 + SearchParam productIndexSearchParam = new SearchParam();
  447 + productIndexSearchParam.setSize(0);
  448 + productIndexSearchParam.setFiter(QueryBuilders.termsQuery("productSkn", sknList));
  449 + List<AbstractAggregationBuilder> aggregationBuilderList = new ArrayList<>();
  450 + IAggregation brandNameAgg = new BrandNameAggregation(1);
  451 + IAggregation smallSortNameAgg = new SmallSortNameAggregation(1);
  452 + IAggregation styleNameAgg = new StyleNameAggregation(1);
  453 +
  454 + aggregationBuilderList.add(brandNameAgg.getBuilder());
  455 + aggregationBuilderList.add(smallSortNameAgg.getBuilder());
  456 + aggregationBuilderList.add(styleNameAgg.getBuilder());
  457 + productIndexSearchParam.setAggregationBuilders(aggregationBuilderList);
  458 +
  459 + final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
  460 + JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(indexName, productIndexSearchParam);
  461 + if (jsonObject != null) {
  462 + CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion by aggregation, keyword = {}.", queryWord);
  463 + return jsonObject;
  464 + }
  465 +
  466 + SearchResult productIndexSearchResult = searchCommonService.doSearch(indexName, productIndexSearchParam);
  467 + if (productIndexSearchResult == null) {
  468 + return null;
  469 + }
  470 +
  471 + List<String> resultTerms = new ArrayList<>();
  472 + List<String> list = (List<String>) smallSortNameAgg.getAggregationResponseMap(productIndexSearchResult.getAggMaps());
  473 + if (CollectionUtils.isNotEmpty(list)) {
  474 + resultTerms.addAll(list);
  475 + }
  476 +
  477 + list = (List<String>) brandNameAgg.getAggregationResponseMap(productIndexSearchResult.getAggMaps());
  478 + if (CollectionUtils.isNotEmpty(list)) {
  479 + resultTerms.addAll(list);
  480 + }
  481 +
  482 + list = (List<String>) styleNameAgg.getAggregationResponseMap(productIndexSearchResult.getAggMaps());
  483 + if (CollectionUtils.isNotEmpty(list)) {
  484 + resultTerms.addAll(list);
  485 + }
  486 +
  487 + // 移除queryWord 避免重复
  488 + String standardQueryWord = CharUtils.standardized(queryWord);
  489 + for (String tipsWord : resultTerms) {
  490 + if (standardQueryWord.equalsIgnoreCase(CharUtils.standardized(tipsWord))) {
  491 + resultTerms.remove(tipsWord);
  492 + break;
  493 + }
  494 + }
  495 +
  496 + // 构建结果 放入缓存
  497 + jsonObject = new JSONObject();
  498 + jsonObject.put("terms_suggestion", resultTerms);
  499 + searchCacheService.addJSONObjectToCache(indexName, productIndexSearchParam, jsonObject);
  500 + logger.info("[func=suggestTipsByProductListResult][query={}][cost={}]", queryWord, System.currentTimeMillis() - /**/begin);
  501 + return jsonObject;
  502 + }
  503 +
406 private JSONObject suggestTipsByConversionIndex(Map<String, String> paramMap) { 504 private JSONObject suggestTipsByConversionIndex(Map<String, String> paramMap) {
407 String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase(); 505 String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
408 long begin = System.currentTimeMillis(); 506 long begin = System.currentTimeMillis();
@@ -417,7 +515,8 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl @@ -417,7 +515,8 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
417 searchKeyWordService.recordSuggestTip(queryWord); 515 searchKeyWordService.recordSuggestTip(queryWord);
418 516
419 // 3) 从conversion索引中获取转换后的关键词列表 517 // 3) 从conversion索引中获取转换后的关键词列表
420 - String dest = getSuggestConversionDestBySource(queryWord); 518 + String dest = CUSTOM_SUGGEST_CONVERSIONS.get(queryWord.trim().toLowerCase());
  519 + dest = dest != null ? null : getSuggestConversionDestBySource(queryWord);
421 if (StringUtils.isEmpty(dest)) { 520 if (StringUtils.isEmpty(dest)) {
422 return null; 521 return null;
423 } 522 }