yoho-search / yoho-search-service · Commits

Go to a project

Authored by Gino Zhang
Commit e9351d13fc92845a87635c2f363e29d6c1568a4c 1 parent dd4caf45

整理SuggestTips接口的相关代码 增加注释

Inline Side-by-side
Showing 1 changed file with 59 additions and 37 deletions
... ... @@ -269,17 +269,16 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
if (suggestTipResult == null) {
// 2.1) 可能是ES发生异常 如果搜索不到相关是应该是集合为空而不是对象为null
return defaultSuggestTips();
}
List<String> resultTerms = (List<String>) suggestTipResult.get("terms_suggestion");
if (CollectionUtils.isNotEmpty(resultTerms)) {
} else if (CollectionUtils.isNotEmpty((List<String>) suggestTipResult.get("terms_suggestion"))) {
// 2.2) 如果能直接从suggest获取到则直接返回
return suggestTipResult;
}
// 3) 根据搜索结果提取关键词
suggestTipResult = suggestTipsByProductListResult(searchResult, paramMap);
if (suggestTipResult != null) {
if (suggestTipResult == null) {
return defaultSuggestTips();
} else if (CollectionUtils.isNotEmpty((List<String>) suggestTipResult.get("terms_suggestion"))) {
return suggestTipResult;
}
... ... @@ -291,11 +290,18 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
return defaultSuggestTips();
} catch (Exception e) {
logger.error("[func=suggestTips]Get suggestion by keyword [" + queryWord + "] failed!", e);
logger.error("[func=suggestTips]Get suggest tips by keyword [" + queryWord + "] failed!", e);
return defaultSuggestTips();
}
}
/**
* 根据suggest索引获取搜索提示词
*
* @param paramMap 搜索参数
* @param hasChangedKeyword 是否已拼写纠错
* @return 提示词结果 如果为null时说明为ES失败需要快速返回，如果terms_suggestion对应的集合为空说明匹配不到，继续往下匹配
*/
private JSONObject suggestTipsBySuggestIndex(final Map<String, String> paramMap, boolean hasChangedKeyword) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
long begin = System.currentTimeMillis();
... ... @@ -303,27 +309,23 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
// 1) 如果是查询SKN 则返回空结果去走聚合
if (searchCommonHelper.isQuerySkn(queryWord)) {
JSONObject emptySuggestResult = new JSONObject();
emptySuggestResult.put("terms_suggestion", new ArrayList<>());
return emptySuggestResult;
return defaultEmptySuggestTips();
}
// 1) 先对query进行分词
// 2) 先对query进行分词
List<String> terms = searchKeyWordService.getAnalyzeTerms(queryWord, "ik_smart", true);
if (CollectionUtils.isEmpty(terms) || isQuerySknWithoutStopWord(terms)) {
JSONObject emptySuggestResult = new JSONObject();
emptySuggestResult.put("terms_suggestion", new ArrayList<>());
return emptySuggestResult;
return defaultEmptySuggestTips();
}
Set<String> termSet = terms.stream().collect(Collectors.toSet());
logger.info("[func=suggestTipsBySuggestIndex][termSet={}]", termSet);
// 2) 根据terms搜索构造搜索请求
// 3) 根据terms搜索构造搜索请求
final String countField = getCountField(paramMap);
SearchParam searchParam = new SearchParam();
// 2.1) 对于suggest的multi-fields至少要有一个字段匹配到20% 匹配打分为常量1
// 3.1) 对于suggest的multi-fields至少要有一个字段匹配到20% 匹配打分为常量1
MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(queryWord.trim().toLowerCase(),
"keyword", "keyword.keyword_ik", "keyword.keyword_pinyin", "keyword.keyword_jianpin", "keyword.keyword_lowercase")
.analyzer("ik_smart")
... ... @@ -334,11 +336,11 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.constantScoreQuery(queryBuilder));
for (String term : termSet) {
// 2.2) 对于完全匹配Term的加1分
// 3.2) 对于完全匹配Term的加1分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("standardKeyword", CharUtils.standardized(term)),
ScoreFunctionBuilders.weightFactorFunction(1));
// 2.3) 对于匹配到一个Term的加2分
// 3.3) 对于匹配到一个Term的加2分
functionScoreQueryBuilder.add(QueryBuilders.termQuery("keyword.keyword_ik", term),
ScoreFunctionBuilders.weightFactorFunction(2));
}
... ... @@ -357,36 +359,36 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
searchParam.setFiter(boolFilter);
// 2.4) 增加拼写纠错 为了增加缓存命中率 此次都增加拼写检查处理
// 3.4) 增加拼写纠错 为了增加缓存命中率 此次都增加拼写检查处理
searchParam.setSuggestionBuilder(SuggestBuilders.termSuggestion("keyword_suggestion")
.text(queryWord.trim().toLowerCase()).field("keyword.keyword_lowercase").size(1));
// 2.5) 按照得分、权重、数量的规则降序排序
// 3.5) 按照得分、权重、数量的规则降序排序
List<SortBuilder> sortBuilders = new ArrayList<>(3);
sortBuilders.add(SortBuilders.fieldSort("_score").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort("weight").order(SortOrder.DESC));
sortBuilders.add(SortBuilders.fieldSort(countField).order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
// 3) 先从缓存中获取
// 4) 先从缓存中获取
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject suggestResult = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggestResult != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion, keyword = {}.", queryWord);
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion by suggest, keyword = {}.", queryWord);
return suggestResult;
}
// 4) 调用ES执行搜索
// 5) 调用ES执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_SUGGEST, searchParam);
if (searchResult == null) {
return null;
}
// 5) 构建结果加入缓存
// 6) 构建结果加入缓存
List<String> correntSpellingSuggestResultTerms = null;
String correnctSpellingKeyword = getCorrectKeywordFromResult(searchResult);
if (StringUtils.isNotEmpty(correnctSpellingKeyword) && !hasChangedKeyword) {
// 6) 执行拼写纠错处理 为了避免无限次递归 只允许拼写检查一次
// 7) 执行拼写纠错处理 为了避免无限次递归 只允许拼写检查一次
logger.info("[func=suggestTipsBySuggestIndex]Switch the suggest keyword from [{}] to [{}].", queryWord, correnctSpellingKeyword);
Map<String, String> newParamMap = new HashMap<>(paramMap.size());
newParamMap.putAll(paramMap);
... ... @@ -395,15 +397,15 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
correntSpellingSuggestResultTerms = correntSpellingSuggestResult != null ? (List<String>) correntSpellingSuggestResult.get("terms_suggestion") : null;
}
// 7) 构建结果加入缓存 将拼写纠错的词放在第一个
// 8) 构建结果加入缓存 将拼写纠错的词放在第一个
suggestResult = new JSONObject();
List<String> resultTerms = searchResult.getResultList().stream().map(map -> (String) map.get("keyword")).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(correntSpellingSuggestResultTerms)) {
if (CollectionUtils.isEmpty(resultTerms)) {
// 7.1） 如果拼写检查返回的结果不为空而原先的结果为空 则使用拼写检查的
resultTerms.addAll(correntSpellingSuggestResultTerms);
// 8.1） 如果拼写检查返回的结果不为空而原先的结果为空 则使用拼写检查的
resultTerms = correntSpellingSuggestResultTerms;
} else if (correntSpellingSuggestResultTerms.contains(correnctSpellingKeyword)) {
// 7.2） 如果拼写检查返回的结果不为空而原先的结果也不为空 则将拼写纠错词放在第一位 这里用contains是确保纠错词的count不会小于20个商品
// 8.2） 如果拼写检查返回的结果不为空而原先的结果也不为空 则将拼写纠错词放在第一位 这里用contains是确保纠错词的count不会小于20个商品
resultTerms = addCorrectSpellingKeywordToFirst(queryWord, correnctSpellingKeyword, resultTerms);
}
}
... ... @@ -439,15 +441,22 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
return resultTerms;
}
/**
* 根据商品列表结果聚合获取搜索提示词
*
* @param searchResult 商品列表搜索结果
* @param paramMap 搜索参数
* @return 提示词结果 如果为null时说明为ES失败需要快速返回，如果terms_suggestion对应的集合为空说明匹配不到，继续往下匹配
*/
private JSONObject suggestTipsByProductListResult(SearchApiResult searchResult, Map<String, String> paramMap) {
if (searchResult == null || searchResult.getData() == null) {
return null;
return defaultEmptySuggestTips();
}
JSONObject dataMap = (JSONObject) searchResult.getData();
List<Map<String, Object>> productList = (List<Map<String, Object>>) dataMap.get("product_list");
if (CollectionUtils.isEmpty(productList)) {
return null;
return defaultEmptySuggestTips();
}
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
... ... @@ -457,7 +466,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
// 结果里只有品牌的信息 所以需要到ES里获取品类和风格的信息
List<Integer> sknList = productList.stream().map(map -> (Integer) map.get("product_skn")).filter(skn -> skn != null).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sknList)) {
return null;
return defaultEmptySuggestTips();
}
SearchParam productIndexSearchParam = new SearchParam();
... ... @@ -508,18 +517,24 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
resultTerms = resultTerms.size() <= SMART_SUGGESTION_TERM_COUNT ? resultTerms : resultTerms.subList(0, SMART_SUGGESTION_TERM_COUNT);
jsonObject.put("terms_suggestion", resultTerms);
searchCacheService.addJSONObjectToCache(indexName, productIndexSearchParam, jsonObject);
logger.info("[func=suggestTipsByProductListResult][query={}][cost={}]", queryWord, System.currentTimeMillis() - /**/begin);
logger.info("[func=suggestTipsByProductListResult][query={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return jsonObject;
}
/**
* 根据爬虫和品牌关键词的转换结果获取搜索提示词
*
* @param paramMap 搜索参数
* @return 提示词结果 如果为null时说明为ES失败需要快速返回，如果terms_suggestion对应的集合为空说明匹配不到，继续往下匹配
*/
private JSONObject suggestTipsByConversionIndex(Map<String, String> paramMap) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
long begin = System.currentTimeMillis();
logger.info("[func=suggestTipsByConversionIndex][query={}][begin={}]", queryWord, begin);
// 1) 判断是否支持从conversion获取推荐词
if (!searchDynamicConfigService.isSearchSuggestionFromConversionOpen()) {
return null;
if (searchCommonHelper.isQuerySkn(queryWord) || !searchDynamicConfigService.isSearchSuggestionFromConversionOpen()) {
return defaultEmptySuggestTips();
}
// 2) 异步上报query关键词 用于统计和分析
... ... @@ -566,7 +581,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
final String indexName = ISearchConstants.INDEX_NAME_SUGGEST;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (jsonObject != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion conversion, keyword = {}.", queryWord);
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion by conversion&suggest, keyword = {}.", queryWord);
return jsonObject;
}
... ... @@ -581,7 +596,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
List<String> resultTerms = searchResult.getResultList().stream().map(map -> (String) map.get("keyword")).collect(Collectors.toList());
jsonObject.put("terms_suggestion", resultTerms);
searchCacheService.addJSONObjectToCache(indexName, searchParam, jsonObject);
logger.info("[func=suggestTipsByConversionIndex][query={}][cost={}]", queryWord, System.currentTimeMillis() - /**/begin);
logger.info("[func=suggestTipsByConversionIndex][query={}][cost={}]", queryWord, System.currentTimeMillis() - begin);
return jsonObject;
}
... ... @@ -607,7 +622,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
final String indexName = ISearchConstants.INDEX_NAME_CONVERSION;
JSONObject suggestResult = searchCacheService.getJSONObjectFromCache(indexName, searchParam);
if (suggestResult != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms conversion, keyword = {}.", queryWord);
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion by conversion, keyword = {}.", queryWord);
return suggestResult.getString("dest");
}
... ... @@ -638,6 +653,13 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
return defSuggestResult;
}
private JSONObject defaultEmptySuggestTips() {
JSONObject emptySuggestResult = new JSONObject();
emptySuggestResult.put("terms_suggestion", new ArrayList<>());
return emptySuggestResult;
}
/**
* 根据关键词和时间查询关键词转换关系
*
... ...