Authored by Gino Zhang

搜索提示支持:1.根据搜索结果聚合出提示词; 2.支持代码自定义的转换关系,如it

package com.yoho.search.service.aggregations.impls;
import com.yoho.search.core.es.agg.AbstractAggregation;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* 单个字段进行聚合的抽象类
*/
public abstract class AbstractSingleFieldAggregation extends AbstractAggregation {
private int count;
public AbstractSingleFieldAggregation(int count) {
this.count = count;
}
@Override
public AbstractAggregationBuilder getBuilder() {
return AggregationBuilders.terms(aggName()).field(getField()).size(count);
}
@Override
public Object getAggregationResponseMap(Map<String, Aggregation> aggMaps) {
MultiBucketsAggregation aggregation = this.getAggregation(aggMaps);
if (aggregation == null) {
return null;
}
List<String> valueList = new ArrayList<String>();
Iterator<? extends Bucket> itSizeAgg = aggregation.getBuckets().iterator();
while (itSizeAgg.hasNext()) {
Bucket ltSize = itSizeAgg.next();
valueList.add(ltSize.getKeyAsString());
}
return valueList;
}
abstract protected String getField();
}
... ...
package com.yoho.search.service.aggregations.impls;
public class BrandNameAggregation extends AbstractSingleFieldAggregation {
public BrandNameAggregation(int count) {
super(count);
}
@Override
public String aggName() {
return "brandNameAgg";
}
@Override
protected String getField() {
return "brandName.brandName_keyword";
}
}
... ...
package com.yoho.search.service.aggregations.impls;
public class SmallSortNameAggregation extends AbstractSingleFieldAggregation {
public SmallSortNameAggregation(int count) {
super(count);
}
@Override
public String aggName() {
return "smallSortNameAgg";
}
@Override
protected String getField() {
return "smallSortName";
}
}
... ...
package com.yoho.search.service.aggregations.impls;
public class StyleNameAggregation extends AbstractSingleFieldAggregation {
public StyleNameAggregation(int count) {
super(count);
}
@Override
public String aggName() {
return "styleNameAgg";
}
@Override
protected String getField() {
return "style";
}
}
... ...
... ... @@ -19,10 +19,11 @@ public interface ISuggestService {
/**
* 根据query词获取term建议和phrase建议。
* 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
* @param searchResult 搜索结果
* @param paramMap 搜索参数
* @return 包括term建议和phrase建议。
*/
JSONObject suggestTips(Map<String, String> paramMap);
JSONObject suggestTips(SearchApiResult searchResult, Map<String, String> paramMap);
/**
* 根据关键词和时间查询关键词转换关系
... ...
... ... @@ -19,8 +19,6 @@ import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.utils.SearchApiResultUtils;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SuggestApiResult;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
... ... @@ -65,7 +63,7 @@ public class ProductListServiceImpl implements IProductListService {
if (needTermSuggestion(searchResult, paramMap)) {
// 当商品数量太少或无结果时 支持智能推荐Term搜索
JSONObject dataMap = ((JSONObject) searchResult.getData());
dataMap.put("suggestion", suggestService.suggestTips(paramMap));
dataMap.put("suggestion", suggestService.suggestTips(searchResult, paramMap));
}
return searchResult;
... ...
... ... @@ -5,9 +5,13 @@ import com.yoho.error.event.SearchEvent;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.agg.IAggregation;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.es.utils.IgnoreSomeException;
import com.yoho.search.service.aggregations.impls.BrandNameAggregation;
import com.yoho.search.service.aggregations.impls.SmallSortNameAggregation;
import com.yoho.search.service.aggregations.impls.StyleNameAggregation;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.service.SearchDynamicConfigService;
... ... @@ -23,6 +27,7 @@ import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
... ... @@ -57,6 +62,8 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
private static final List<String> DEFAULT_SUGGEST_TIPS = Arrays.asList("潮流", "时尚", "休闲");
private static final Map<String, String> CUSTOM_SUGGEST_CONVERSIONS = new HashMap<>();
private static final int VALID_STATUS = 1;
@Autowired
... ... @@ -73,6 +80,9 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.publisher = applicationEventPublisher;
// it是一个停用词 所以直接配置转换关系
CUSTOM_SUGGEST_CONVERSIONS.put("it", ":CHOCOOLATE,izzue,5cm");
}
@Override
... ... @@ -237,10 +247,11 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
* 根据query词获取term建议和phrase建议。
* 用于搜索结果数量太少或者无结果的时候给予用户的搜索建议。
*
* @param paramMap 商品列表搜索参数
* @param searchResult 搜索结果
* @param paramMap 商品列表搜索参数
* @return 包括term建议和phrase建议。
*/
public JSONObject suggestTips(Map<String, String> paramMap) {
public JSONObject suggestTips(SearchApiResult searchResult, Map<String, String> paramMap) {
// 1) 对query进行判断 为空时不处理
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
if (StringUtils.isEmpty(queryWord)) {
... ... @@ -248,7 +259,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
}
try {
// 2) 先调用suggest获取搜索提示词 只要能匹配到20%的term即可
// 2) 先调用suggest获取搜索提示词 支持拼写纠错
JSONObject suggestTipResult = suggestTipsBySuggestIndex(paramMap, false);
if (suggestTipResult == null) {
// 2.1) 可能是ES发生异常 如果搜索不到相关是应该是集合为空而不是对象为null
... ... @@ -261,6 +272,12 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
return suggestTipResult;
}
// 3) 根据搜索结果提取关键词
suggestTipResult = suggestTipsByProductListResult(searchResult, paramMap);
if (suggestTipResult != null) {
return suggestTipResult;
}
// 3) 通过conversion转换后再到suggest获取提示词
suggestTipResult = suggestTipsByConversionIndex(paramMap);
if (suggestTipResult != null && CollectionUtils.isNotEmpty((List<String>) suggestTipResult.get("terms_suggestion"))) {
... ... @@ -274,14 +291,16 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
}
}
public JSONObject suggestTipsBySuggestIndex(final Map<String, String> paramMap, boolean hasChangedKeyword) {
private JSONObject suggestTipsBySuggestIndex(final Map<String, String> paramMap, boolean hasChangedKeyword) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD);
long begin = System.currentTimeMillis();
logger.info("[func=suggestTipsBySuggestIndex][query={}][begin={}]", queryWord, begin);
// 1) 先对query进行分词
List<String> terms = searchKeyWordService.getAnalyzeTerms(queryWord, "ik_smart", true);
if (CollectionUtils.isEmpty(terms)) {
return null;
JSONObject emptySuggestResult = new JSONObject();
emptySuggestResult.put("terms_suggestion", new ArrayList<>());
return emptySuggestResult;
}
Set<String> termSet = terms.stream().collect(Collectors.toSet());
... ... @@ -392,7 +411,7 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
break;
}
if(!correnctSpellingKeyword.equalsIgnoreCase(item)){
if (!correnctSpellingKeyword.equalsIgnoreCase(item)) {
newResultTerms.add(item);
}
}
... ... @@ -403,6 +422,85 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
return resultTerms;
}
private JSONObject suggestTipsByProductListResult(SearchApiResult searchResult, Map<String, String> paramMap) {
if (searchResult == null || searchResult.getData() == null) {
return null;
}
JSONObject dataMap = (JSONObject) searchResult.getData();
List<Map<String, Object>> productList = (List<Map<String, Object>>) dataMap.get("product_list");
if (CollectionUtils.isEmpty(productList)) {
return null;
}
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
long begin = System.currentTimeMillis();
logger.info("[func=suggestTipsByProductListResult][query={}][size={}][begin={}]", queryWord, productList.size(), begin);
// 结果里只有品牌的信息 所以需要到ES里获取品类和风格的信息
List<Integer> sknList = productList.stream().map(map -> (Integer) map.get("product_skn")).filter(skn -> skn != null).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sknList)) {
return null;
}
SearchParam productIndexSearchParam = new SearchParam();
productIndexSearchParam.setSize(0);
productIndexSearchParam.setFiter(QueryBuilders.termsQuery("productSkn", sknList));
List<AbstractAggregationBuilder> aggregationBuilderList = new ArrayList<>();
IAggregation brandNameAgg = new BrandNameAggregation(1);
IAggregation smallSortNameAgg = new SmallSortNameAggregation(1);
IAggregation styleNameAgg = new StyleNameAggregation(1);
aggregationBuilderList.add(brandNameAgg.getBuilder());
aggregationBuilderList.add(smallSortNameAgg.getBuilder());
aggregationBuilderList.add(styleNameAgg.getBuilder());
productIndexSearchParam.setAggregationBuilders(aggregationBuilderList);
final String indexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(indexName, productIndexSearchParam);
if (jsonObject != null) {
CACHE_MATCH_REQUEST.info("match cache for product list terms suggestion by aggregation, keyword = {}.", queryWord);
return jsonObject;
}
SearchResult productIndexSearchResult = searchCommonService.doSearch(indexName, productIndexSearchParam);
if (productIndexSearchResult == null) {
return null;
}
List<String> resultTerms = new ArrayList<>();
List<String> list = (List<String>) smallSortNameAgg.getAggregationResponseMap(productIndexSearchResult.getAggMaps());
if (CollectionUtils.isNotEmpty(list)) {
resultTerms.addAll(list);
}
list = (List<String>) brandNameAgg.getAggregationResponseMap(productIndexSearchResult.getAggMaps());
if (CollectionUtils.isNotEmpty(list)) {
resultTerms.addAll(list);
}
list = (List<String>) styleNameAgg.getAggregationResponseMap(productIndexSearchResult.getAggMaps());
if (CollectionUtils.isNotEmpty(list)) {
resultTerms.addAll(list);
}
// 移除queryWord 避免重复
String standardQueryWord = CharUtils.standardized(queryWord);
for (String tipsWord : resultTerms) {
if (standardQueryWord.equalsIgnoreCase(CharUtils.standardized(tipsWord))) {
resultTerms.remove(tipsWord);
break;
}
}
// 构建结果 放入缓存
jsonObject = new JSONObject();
jsonObject.put("terms_suggestion", resultTerms);
searchCacheService.addJSONObjectToCache(indexName, productIndexSearchParam, jsonObject);
logger.info("[func=suggestTipsByProductListResult][query={}][cost={}]", queryWord, System.currentTimeMillis() - /**/begin);
return jsonObject;
}
private JSONObject suggestTipsByConversionIndex(Map<String, String> paramMap) {
String queryWord = paramMap.get(SearchRequestParams.PARAM_SEARCH_KEYWORD).toLowerCase();
long begin = System.currentTimeMillis();
... ... @@ -417,7 +515,8 @@ public class SuggestServiceImpl implements ISuggestService, ApplicationEventPubl
searchKeyWordService.recordSuggestTip(queryWord);
// 3) 从conversion索引中获取转换后的关键词列表
String dest = getSuggestConversionDestBySource(queryWord);
String dest = CUSTOM_SUGGEST_CONVERSIONS.get(queryWord.trim().toLowerCase());
dest = dest != null ? null : getSuggestConversionDestBySource(queryWord);
if (StringUtils.isEmpty(dest)) {
return null;
}
... ...