|
|
|
package com.yoho.search.service.searchexplainer;
|
|
|
|
|
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
|
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
|
|
|
|
import com.google.common.cache.CacheBuilder;
|
|
|
|
import com.google.common.cache.CacheLoader;
|
|
|
|
import com.yoho.search.base.utils.ISearchConstants;
|
|
|
|
import com.yoho.search.core.es.IElasticsearchClient;
|
|
|
|
import com.yoho.search.core.es.model.SearchParam;
|
|
|
|
import com.yoho.search.core.es.model.SearchResult;
|
|
|
|
import com.yoho.search.service.service.ESClientMgr;
|
|
|
|
import org.apache.commons.collections.CollectionUtils;
|
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
|
|
|
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
|
|
|
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
|
|
|
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
|
|
|
import org.elasticsearch.index.query.QueryBuilders;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
import org.springframework.util.Assert;
|
|
|
|
|
|
|
|
import java.util.*;
|
|
|
|
import java.util.concurrent.ExecutionException;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Created by ginozhang on 2016/11/16.
|
|
|
|
*/
|
|
|
|
@Service
|
|
|
|
public class SearchExplainerService {
|
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(SearchExplainerService.class);
|
|
|
|
|
|
|
|
private static final String DEFAULT_ANALYZER = "keyword";
|
|
|
|
|
|
|
|
private static final String MATCH_TYPE_STRING = "String Match";
|
|
|
|
private static final String MATCH_TYPE_RANGE = "Range Match";
|
|
|
|
|
|
|
|
private static final String RETURN_TYPE_ALL = "all";
|
|
|
|
private static final String RETURN_TYPE_MATCH_ONLY = "match_only";
|
|
|
|
|
|
|
|
// 拷贝search-service的search.default.field配置参数
|
|
|
|
// private static final String fieldsToSearch = Configuration.getString("search.default.field");
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private ESClientMgr esClientMgr;
|
|
|
|
|
|
|
|
private volatile Map<String, FieldDesc> localFieldDescMap = null;
|
|
|
|
|
|
|
|
private volatile Map<String, List<String>> localCopiedFieldMap = null;
|
|
|
|
|
|
|
|
private volatile List<String> localMutilFields = null;
|
|
|
|
|
|
|
|
//入参和索引中字段名的映射
|
|
|
|
public final static Map<String, EsField> fieldMap = new HashMap<String, EsField>() {{
|
|
|
|
//字符类
|
|
|
|
put("brand", new EsField("brandId", MATCH_TYPE_STRING));
|
|
|
|
put("shop", new EsField("shopId", MATCH_TYPE_STRING));
|
|
|
|
put("msort", new EsField("maxSortId", MATCH_TYPE_STRING));
|
|
|
|
put("misort", new EsField("middleSortId", MATCH_TYPE_STRING));
|
|
|
|
put("sort", new EsField("smallSortId", MATCH_TYPE_STRING));
|
|
|
|
put("color", new EsField("colorIds", MATCH_TYPE_STRING));
|
|
|
|
put("size", new EsField("sizeIds", MATCH_TYPE_STRING));
|
|
|
|
put("price", new EsField("salesPrice", MATCH_TYPE_STRING));
|
|
|
|
put("gender", new EsField("gender", MATCH_TYPE_STRING));
|
|
|
|
put("specialoffer", new EsField("specialoffer", MATCH_TYPE_STRING));
|
|
|
|
put("isdiscount", new EsField("isDiscount", MATCH_TYPE_STRING));
|
|
|
|
put("promotion", new EsField("ispromotion", MATCH_TYPE_STRING));
|
|
|
|
put("vdt", new EsField("vipDiscountType", MATCH_TYPE_STRING));
|
|
|
|
put("attribute", new EsField("attribute", MATCH_TYPE_STRING));
|
|
|
|
put("limited", new EsField("islimited", MATCH_TYPE_STRING));
|
|
|
|
put("new", new EsField("isnew", MATCH_TYPE_STRING));
|
|
|
|
put("outlets", new EsField("isOutlets", MATCH_TYPE_STRING));
|
|
|
|
put("status", new EsField("status", MATCH_TYPE_STRING));
|
|
|
|
put("style", new EsField("styleIds", MATCH_TYPE_STRING));
|
|
|
|
put("sell_channels", new EsField("sellChannels", MATCH_TYPE_STRING));
|
|
|
|
put("folder_id", new EsField("folder_id", MATCH_TYPE_STRING));
|
|
|
|
put("series_id", new EsField("series_id", MATCH_TYPE_STRING));
|
|
|
|
put("day", new EsField("shelveDay", MATCH_TYPE_STRING));
|
|
|
|
put("brand", new EsField("salesPrice", MATCH_TYPE_STRING));
|
|
|
|
put("shop", new EsField("salesPrice", MATCH_TYPE_STRING));
|
|
|
|
put("brand", new EsField("brandId", MATCH_TYPE_STRING));
|
|
|
|
put("shop", new EsField("shop", MATCH_TYPE_STRING));
|
|
|
|
put("brand", new EsField("brandId", MATCH_TYPE_STRING));
|
|
|
|
put("breaking", new EsField("breakingRate", MATCH_TYPE_STRING));
|
|
|
|
put("ageLevel", new EsField("ageLevel", MATCH_TYPE_STRING));
|
|
|
|
put("product_skn", new EsField("productSkn", MATCH_TYPE_STRING));
|
|
|
|
//范围类
|
|
|
|
put("stocknumber", new EsField("storageNum", MATCH_TYPE_RANGE));
|
|
|
|
put("p_d", new EsField("promotionDiscountInt", MATCH_TYPE_RANGE));
|
|
|
|
put("p_d_int", new EsField("promotionDiscountInt", MATCH_TYPE_RANGE));
|
|
|
|
put("first_shelve_time", new EsField("firstShelveTime", MATCH_TYPE_RANGE));
|
|
|
|
put("shelve_time", new EsField("shelveTime", MATCH_TYPE_RANGE));
|
|
|
|
}};
|
|
|
|
|
|
|
|
public String fieldConvert(String paraField) {
|
|
|
|
EsField esField = fieldMap.get(paraField);
|
|
|
|
if (esField == null) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
return esField.getFieldName();
|
|
|
|
}
|
|
|
|
|
|
|
|
public Map<String, Object> explain(Map<String, String> paramMap) throws Exception {
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
|
logger.info("Begin to explain search. start: {}, paramMap: {}.", start, paramMap);
|
|
|
|
String skn = paramMap.get("skn");
|
|
|
|
if (StringUtils.isEmpty(skn)) {
|
|
|
|
Map<String, Object> map = new LinkedHashMap<>();
|
|
|
|
map.put("code", "400");
|
|
|
|
map.put("message", "skn is null");
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
String returnType = paramMap.get("returnType");
|
|
|
|
if (returnType == null) {
|
|
|
|
returnType = RETURN_TYPE_MATCH_ONLY;
|
|
|
|
}
|
|
|
|
|
|
|
|
String multiSearchType = paramMap.get("multiSearchType");
|
|
|
|
if (multiSearchType == null) {
|
|
|
|
multiSearchType = ISearchConstants.SEARCH_MULTIMATCHQUERY_TYPE;
|
|
|
|
}
|
|
|
|
|
|
|
|
String keyword = paramMap.get("query");
|
|
|
|
logger.info("Begin to explain search keyword [{}] for product skn {}.", keyword, skn);
|
|
|
|
|
|
|
|
// 1. 根据skn获取es中的数据
|
|
|
|
Map<String, Object> document = getDocumentBySkn(skn);
|
|
|
|
logger.info("Get the document for the product. document: \n{}", document);
|
|
|
|
|
|
|
|
// 2.分析过滤的参数
|
|
|
|
Map<String, Object> explainResult = explainFilter(paramMap, document);
|
|
|
|
|
|
|
|
if ((Boolean) explainResult.get("filter_match") && StringUtils.isNotEmpty(keyword)) {
|
|
|
|
List<SearchMatch> queryMatchList = explainMultiMatchQuery(document, keyword, returnType, multiSearchType);
|
|
|
|
explainResult.put("query_match", queryMatchList.stream().anyMatch(SearchMatch::isMatch));
|
|
|
|
explainResult.put("query_list", queryMatchList);
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("End to explain search keyword [{}] for product skn {}. cost: {}", keyword, skn, System.currentTimeMillis() - start);
|
|
|
|
return explainResult;
|
|
|
|
}
|
|
|
|
|
|
|
|
private Map<String, Object> explainFilter(Map<String, String> paramMap, Map<String, Object> document) {
|
|
|
|
Map<String, Object> map = new LinkedHashMap<>();
|
|
|
|
List<String> warns = new ArrayList<>();
|
|
|
|
boolean isFilterMatch = true;
|
|
|
|
List<FilterMatch> filterMatchList = new ArrayList<>();
|
|
|
|
for (Map.Entry<String, String> entry : paramMap.entrySet()) {
|
|
|
|
String key = entry.getKey();
|
|
|
|
if (key.equals("query") || key.equals("skn") || key.equals("returnType")) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!fieldMap.containsKey(key)) {
|
|
|
|
warns.add("parameter " + key + " is invalid\n");
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
Object value = document.get(fieldConvert(key));
|
|
|
|
if (value != null) {
|
|
|
|
String paraValue = value.toString();
|
|
|
|
EsField esField = fieldMap.get(key);
|
|
|
|
//将入参和字段值都转化为字符数组
|
|
|
|
String[] paraArray = entry.getValue().split(",");
|
|
|
|
String[] indexArray = paraValue.split(",");
|
|
|
|
Set<String> indexSet = new HashSet<String>();
|
|
|
|
for (String i : indexArray) {
|
|
|
|
indexSet.add(i);
|
|
|
|
}
|
|
|
|
//字符匹配比较
|
|
|
|
if (esField.getFieldType().equals(MATCH_TYPE_STRING)) {
|
|
|
|
for (String p : paraArray) {
|
|
|
|
if (!indexSet.contains(p)) {
|
|
|
|
filterMatchList.add(new FilterMatch(key, entry.getValue(), paraValue, esField.getFieldType(), false));
|
|
|
|
isFilterMatch = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
//取值范围比较
|
|
|
|
if (esField.getFieldType().equals(MATCH_TYPE_RANGE)) {
|
|
|
|
if (key.equals("stocknumber")) {
|
|
|
|
if (Integer.valueOf(value.toString()) < 1) {
|
|
|
|
filterMatchList.add(new FilterMatch(key, entry.getValue(), paraValue, esField.getFieldType(), false));
|
|
|
|
isFilterMatch = false;
|
|
|
|
}
|
|
|
|
} else if (paraArray.length == 2 && indexArray.length == 1) {
|
|
|
|
Integer paraRangeStart = Integer.valueOf(paraArray[0]);
|
|
|
|
Integer paraRangeEnd = Integer.valueOf(paraArray[1]);
|
|
|
|
Integer indexValue = Integer.valueOf(indexArray[0]);
|
|
|
|
if (indexValue < paraRangeStart || indexValue > paraRangeEnd) {
|
|
|
|
filterMatchList.add(new FilterMatch(key, entry.getValue(), paraValue, esField.getFieldType(), false));
|
|
|
|
isFilterMatch = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
map.put("code", "200");
|
|
|
|
map.put("message", "search explainer");
|
|
|
|
if (CollectionUtils.isNotEmpty(warns)) {
|
|
|
|
map.put("warn", warns);
|
|
|
|
}
|
|
|
|
map.put("filter_match", isFilterMatch);
|
|
|
|
map.put("filter_list", filterMatchList);
|
|
|
|
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<SearchMatch> explainMultiMatchQuery(Map<String, Object> document, String keyword, String returnType, String multiSearchType) throws Exception {
|
|
|
|
// 3. 解析mapping文件获取字段元数据
|
|
|
|
Map<String, FieldDesc> fieldDescMap = parseMapping();
|
|
|
|
logger.info("Get the field description by mapping. fieldDescMap: \n{}", fieldDescMap);
|
|
|
|
|
|
|
|
// 4. 处理多字段和copy_to的字段并赋值
|
|
|
|
processMultiFields(fieldDescMap, document);
|
|
|
|
processCopiedFields(fieldDescMap, document);
|
|
|
|
logger.info("Set the value of copied field succeeded.");
|
|
|
|
|
|
|
|
// 5. 分析每个multi-match的每个字段 根据分词结果判定是否匹配
|
|
|
|
List<String> sortedSearchFields = getSortedSearchFields();
|
|
|
|
logger.info("Get the search fields. fields: {}", sortedSearchFields);
|
|
|
|
|
|
|
|
String searchType = ISearchConstants.SEARCH_MULTIMATCHQUERY_TYPE;
|
|
|
|
if ("cross_fields".equalsIgnoreCase(multiSearchType)) {
|
|
|
|
return explainCrossFieldsQuery(document, keyword, returnType, fieldDescMap, sortedSearchFields);
|
|
|
|
} else {
|
|
|
|
return explainBestFieldsQuery(document, keyword, returnType, fieldDescMap, sortedSearchFields);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private List explainBestFieldsQuery(Map<String, Object> document, String keyword, String returnType, Map<String, FieldDesc> fieldDescMap, List<String> sortedSearchFields) throws Exception {
|
|
|
|
boolean isQueryMatch = false;
|
|
|
|
boolean isFieldQueryMatch = false;
|
|
|
|
List<FieldMatch> queryMatchList = new ArrayList<>();
|
|
|
|
for (String field : sortedSearchFields) {
|
|
|
|
String value = document.get(field) != null ? document.get(field).toString() : null;
|
|
|
|
FieldDesc fieldDesc = fieldDescMap.get(field);
|
|
|
|
Assert.notNull(fieldDesc, "no field " + field + " defined.");
|
|
|
|
List<String> analyzerResult = getAnalyzerResult(fieldDesc, value);
|
|
|
|
List<String> keywordAnalyzerResult = getKeyWordAnalyzerResult(fieldDesc, keyword);
|
|
|
|
isFieldQueryMatch = judgeMatch(analyzerResult, keywordAnalyzerResult);
|
|
|
|
isQueryMatch = isFieldQueryMatch || isQueryMatch;
|
|
|
|
//如果是匹配模式则只返回匹配的字段，如果是ALL模式则返回索引字段
|
|
|
|
if (returnType.equals(RETURN_TYPE_ALL) || isFieldQueryMatch) {
|
|
|
|
queryMatchList.add(new FieldMatch(field, value, fieldDesc.analyzer, analyzerResult, fieldDesc.search_analyzer, keywordAnalyzerResult, isFieldQueryMatch));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (returnType.equals(RETURN_TYPE_MATCH_ONLY) && isFieldQueryMatch) {
|
|
|
|
// 找到匹配的就返回
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return queryMatchList;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List explainCrossFieldsQuery(Map<String, Object> document, String keyword, String returnType, Map<String, FieldDesc> fieldDescMap, List<String> sortedSearchFields) throws Exception {
|
|
|
|
List<GroupMatch> groupMatchList = new ArrayList<>();
|
|
|
|
List<FieldDesc> searchfieldDescList = new ArrayList<>();
|
|
|
|
fieldDescMap.forEach((field, fieldDesc) -> {
|
|
|
|
if (sortedSearchFields.contains(field)) {
|
|
|
|
if (StringUtils.isEmpty(fieldDesc.analyzer)) {
|
|
|
|
fieldDesc.analyzer = DEFAULT_ANALYZER;
|
|
|
|
}
|
|
|
|
if (StringUtils.isEmpty(fieldDesc.search_analyzer)) {
|
|
|
|
fieldDesc.search_analyzer = fieldDesc.analyzer;
|
|
|
|
}
|
|
|
|
|
|
|
|
searchfieldDescList.add(fieldDesc);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
// 根据search_analyzer进行分组
|
|
|
|
Map<String, List<FieldDesc>> groupMap = searchfieldDescList.stream().collect(Collectors.groupingBy(fieldDesc -> fieldDesc.search_analyzer));
|
|
|
|
for (Map.Entry<String, List<FieldDesc>> groupEntry : groupMap.entrySet()) {
|
|
|
|
// 对于cross_field来说 必须所有输入的token在同一组内
|
|
|
|
String searchAnalyzer = groupEntry.getValue().get(0).search_analyzer;
|
|
|
|
List<String> keywordAnalyzerResult = getAnalyzerTokens(keyword, searchAnalyzer);
|
|
|
|
List<String> sortedFieldDescNames = groupEntry.getValue().stream().map(fieldDesc -> fieldDesc.field).collect(Collectors.toList());
|
|
|
|
boolean isMatch = true;
|
|
|
|
List<FieldMatch> fieldMatchList = new ArrayList<>();
|
|
|
|
for (String token : keywordAnalyzerResult) {
|
|
|
|
boolean isTokenMatch = false;
|
|
|
|
for (String field : sortedSearchFields) {
|
|
|
|
// 检查field是否匹配token
|
|
|
|
if (!sortedFieldDescNames.contains(field)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
String value = document.get(field) != null ? document.get(field).toString() : null;
|
|
|
|
FieldDesc fieldDesc = fieldDescMap.get(field);
|
|
|
|
Assert.notNull(fieldDesc, "for " + field);
|
|
|
|
List<String> analyzerResult = getAnalyzerResult(fieldDesc, value);
|
|
|
|
isTokenMatch = analyzerResult.contains(token);
|
|
|
|
if (returnType.equals(RETURN_TYPE_ALL) || isTokenMatch) {
|
|
|
|
fieldMatchList.add(new FieldMatch(field, value, fieldDesc.analyzer, analyzerResult, fieldDesc.search_analyzer, Arrays.asList(token), isTokenMatch));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (returnType.equals(RETURN_TYPE_MATCH_ONLY) && isTokenMatch) {
|
|
|
|
// 找到匹配的就返回
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
isMatch = isMatch && isTokenMatch;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (returnType.equals(RETURN_TYPE_ALL) || isMatch) {
|
|
|
|
groupMatchList.add(new GroupMatch("", searchAnalyzer, keywordAnalyzerResult, fieldMatchList, isMatch));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return groupMatchList;
|
|
|
|
}
|
|
|
|
|
|
|
|
public Map<String, Object> getTokens(String skn) throws Exception {
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
|
logger.info("Begin to get tokens for product skn {}.", skn);
|
|
|
|
|
|
|
|
// 1. 根据skn获取es中的数据
|
|
|
|
Map<String, Object> document = getDocumentBySkn(skn);
|
|
|
|
logger.info("Get the document for the product. document: \n{}", document);
|
|
|
|
|
|
|
|
// 2. 解析mapping文件获取字段元数据
|
|
|
|
Map<String, FieldDesc> fieldDescMap = parseMapping();
|
|
|
|
logger.info("Get the field description by mapping. fieldDescMap: \n{}", fieldDescMap);
|
|
|
|
|
|
|
|
// 3. 处理多字段和copy_to的字段并赋值
|
|
|
|
processMultiFields(fieldDescMap, document);
|
|
|
|
processCopiedFields(fieldDescMap, document);
|
|
|
|
logger.info("Set the value of copied field succeeded.");
|
|
|
|
|
|
|
|
// 4. 分析每个multi-match的每个字段 根据权重优先级展示token列表
|
|
|
|
List<String> sortedSearchFields = getSortedSearchFields();
|
|
|
|
logger.info("Get the search fields. fields: {}", sortedSearchFields);
|
|
|
|
boolean isMatch = false;
|
|
|
|
List<String> tokens = new ArrayList<>();
|
|
|
|
for (String field : sortedSearchFields) {
|
|
|
|
String value = document.get(field) != null ? document.get(field).toString() : null;
|
|
|
|
FieldDesc fieldDesc = fieldDescMap.get(field);
|
|
|
|
Assert.notNull(fieldDesc, "no field " + field + " defined.");
|
|
|
|
List<String> analyzerResult = getAnalyzerResult(fieldDesc, value);
|
|
|
|
if (CollectionUtils.isNotEmpty(analyzerResult)) {
|
|
|
|
analyzerResult.removeAll(tokens);
|
|
|
|
tokens.addAll(analyzerResult);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Map<String, Object> map = new HashMap<>();
|
|
|
|
map.put("code", "200");
|
|
|
|
map.put("message", "product tokens.");
|
|
|
|
map.put("data", tokens);
|
|
|
|
logger.info("End to get tokens for product skn {}. cost: {}", skn, System.currentTimeMillis() - start);
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
private boolean judgeMatch(List<String> analyzerResult, List<String> keywordAnalyzerResult) {
|
|
|
|
// 对于best_fields必须包含所有token
|
|
|
|
return analyzerResult.containsAll(keywordAnalyzerResult);
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<String> getKeyWordAnalyzerResult(FieldDesc fieldDesc, String keyword) throws ExecutionException {
|
|
|
|
if (fieldDesc.noNeedAnalyzer()) {
|
|
|
|
return Arrays.asList(keyword);
|
|
|
|
}
|
|
|
|
|
|
|
|
String searchAnalyzer = fieldDesc.search_analyzer;
|
|
|
|
if (StringUtils.isEmpty(searchAnalyzer)) {
|
|
|
|
searchAnalyzer = fieldDesc.analyzer;
|
|
|
|
}
|
|
|
|
if (StringUtils.isEmpty(searchAnalyzer)) {
|
|
|
|
searchAnalyzer = DEFAULT_ANALYZER;
|
|
|
|
}
|
|
|
|
return getAnalyzerTokens(keyword, searchAnalyzer);
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<String> getAnalyzerResult(FieldDesc fieldDesc, String value) throws ExecutionException {
|
|
|
|
if (StringUtils.isEmpty(value)) {
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fieldDesc.noNeedAnalyzer()) {
|
|
|
|
return Arrays.asList(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
String analyzer = fieldDesc.analyzer;
|
|
|
|
if (StringUtils.isEmpty(analyzer)) {
|
|
|
|
analyzer = DEFAULT_ANALYZER;
|
|
|
|
}
|
|
|
|
|
|
|
|
return getAnalyzerTokens(value, analyzer);
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<String> getAnalyzerTokens(String text, String analyzer) throws ExecutionException {
|
|
|
|
// 使用guava的本地缓存
|
|
|
|
return CacheBuilder.newBuilder()
|
|
|
|
.maximumSize(1000)
|
|
|
|
.expireAfterWrite(3, TimeUnit.MINUTES)
|
|
|
|
.build(
|
|
|
|
new CacheLoader<String, List<String>>() {
|
|
|
|
public List<String> load(String key) {
|
|
|
|
String tempAnalyzer = key.split("@")[0];
|
|
|
|
String tempText = key.split("@")[1];
|
|
|
|
|
|
|
|
if ("keyword".equals(analyzer)) {
|
|
|
|
// keyword 分词就不去调用ES
|
|
|
|
return Arrays.asList(tempText);
|
|
|
|
}
|
|
|
|
|
|
|
|
IElasticsearchClient client = esClientMgr.getClient(ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
|
|
|
|
List<AnalyzeResponse.AnalyzeToken> list = client.getAnalyzeResponse(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, text, analyzer).getTokens();
|
|
|
|
return list.stream().map(item -> item.getTerm()).collect(Collectors.toList());
|
|
|
|
}
|
|
|
|
}).get(analyzer + "@" + text);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private Map<String, Object> getDocumentBySkn(String skn) {
|
|
|
|
IElasticsearchClient client = esClientMgr.getClient(ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
|
|
|
|
if (!client.indexExists(ISearchConstants.INDEX_NAME_PRODUCT_INDEX)) {
|
|
|
|
throw new RuntimeException("index not exist");
|
|
|
|
}
|
|
|
|
|
|
|
|
SearchParam searchParam = new SearchParam();
|
|
|
|
searchParam.setQuery(QueryBuilders.termQuery("productSkn", skn));
|
|
|
|
SearchResult searchResult = client.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
|
|
|
|
if (searchResult == null || searchResult.getResultList() == null || searchResult.getResultList().isEmpty()) {
|
|
|
|
throw new RuntimeException("product skn not exist");
|
|
|
|
}
|
|
|
|
|
|
|
|
return searchResult.getResultList().get(0);
|
|
|
|
}
|
|
|
|
|
|
|
|
private Map<String, FieldDesc> parseMapping() {
|
|
|
|
if (localFieldDescMap != null) {
|
|
|
|
return localFieldDescMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
synchronized (this) {
|
|
|
|
if (localFieldDescMap != null) {
|
|
|
|
// double check
|
|
|
|
return localFieldDescMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
// 从ES获取mapping
|
|
|
|
IElasticsearchClient client = esClientMgr.getClient(ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
|
|
|
|
GetMappingsResponse response = client.getMapping(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
|
|
|
|
Assert.notNull(response, ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
|
|
|
|
ImmutableOpenMap<String, MappingMetaData> mapping = response.getMappings().get(ISearchConstants.INDEX_NAME_PRODUCT_INDEX);
|
|
|
|
Iterator<ObjectObjectCursor<String, MappingMetaData>> iterator = mapping.iterator();
|
|
|
|
while(iterator.hasNext())
|
|
|
|
{
|
|
|
|
ObjectObjectCursor<String, MappingMetaData> object = iterator.next();
|
|
|
|
//object.value.getSourceAsMap();
|
|
|
|
}
|
|
|
|
|
|
|
|
// String mappingContent = yohoIndexService.getIndex(ISearchConstants.INDEX_NAME_PRODUCT_INDEX).getMappingContent();
|
|
|
|
// JSONObject jsonObject = JSON.parseObject(mappingContent);
|
|
|
|
// JSONObject mapping = jsonObject.getJSONObject(ISearchConstants.INDEX_NAME_PRODUCT_INDEX).getJSONObject("properties");
|
|
|
|
// Map<String, FieldDesc> fieldDescMap = new HashMap<>();
|
|
|
|
// for (String field : mapping.keySet()) {
|
|
|
|
// parseToFieldDesc(fieldDescMap, field, mapping.getJSONObject(field));
|
|
|
|
// }
|
|
|
|
|
|
|
|
// localFieldDescMap = fieldDescMap;
|
|
|
|
return localFieldDescMap;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void parseToFieldDesc(Map<String, FieldDesc> fieldDescMap, String field, JSONObject jsonObject) {
|
|
|
|
if (jsonObject.containsKey("fields")) {
|
|
|
|
// 多字段类型
|
|
|
|
JSONObject innerJsonObject = jsonObject.getJSONObject("fields");
|
|
|
|
for (String innerField : innerJsonObject.keySet()) {
|
|
|
|
String realFieldName = innerField;
|
|
|
|
if (!innerField.equals(field)) {
|
|
|
|
realFieldName = field + "." + innerField;
|
|
|
|
}
|
|
|
|
|
|
|
|
parseSingle(fieldDescMap, realFieldName, innerJsonObject.getJSONObject(innerField));
|
|
|
|
}
|
|
|
|
} else if (jsonObject.containsKey("properties")) {
|
|
|
|
// nested object类型 搜索暂不支持查询内嵌对象的字段
|
|
|
|
} else {
|
|
|
|
parseSingle(fieldDescMap, field, jsonObject);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void parseSingle(Map<String, FieldDesc> fieldDescMap, String field, JSONObject jsonObject) {
|
|
|
|
FieldDesc fieldDesc = new FieldDesc();
|
|
|
|
fieldDesc.field = field;
|
|
|
|
fieldDesc.type = jsonObject.getString("type");
|
|
|
|
fieldDesc.index = jsonObject.getString("index");
|
|
|
|
fieldDesc.analyzer = jsonObject.getString("analyzer");
|
|
|
|
fieldDesc.search_analyzer = jsonObject.getString("search_analyzer");
|
|
|
|
Object copyTo = jsonObject.get("copy_to");
|
|
|
|
if (copyTo != null && copyTo instanceof JSONArray) {
|
|
|
|
JSONArray jsonArray = (JSONArray) copyTo;
|
|
|
|
for (int i = 0; i < jsonArray.size(); i++) {
|
|
|
|
fieldDesc.addCopyTo(jsonArray.getString(i));
|
|
|
|
}
|
|
|
|
} else if (copyTo != null && copyTo instanceof JSONObject) {
|
|
|
|
fieldDesc.addCopyTo((String) copyTo);
|
|
|
|
}
|
|
|
|
|
|
|
|
fieldDescMap.put(field, fieldDesc);
|
|
|
|
}
|
|
|
|
|
|
|
|
private void processMultiFields(Map<String, FieldDesc> fieldDescMap, Map<String, Object> document) {
|
|
|
|
for (String field : fieldDescMap.keySet()) {
|
|
|
|
String[] parts = field.split("\\.");
|
|
|
|
if (parts != null && parts.length == 2) {
|
|
|
|
document.put(field, document.get(parts[0]));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void processCopiedFields(Map<String, FieldDesc> fieldDescMap, Map<String, Object> document) {
|
|
|
|
Map<String, List<String>> copiedFieldMap = getCopiedFieldMap(fieldDescMap);
|
|
|
|
for (Map.Entry<String, List<String>> entry : copiedFieldMap.entrySet()) {
|
|
|
|
String copiedField = entry.getKey();
|
|
|
|
StringBuffer sb = new StringBuffer();
|
|
|
|
for (String field : entry.getValue()) {
|
|
|
|
if (document.get(field) != null) {
|
|
|
|
sb.append(document.get(field)).append(" ");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
document.put(copiedField, sb.toString());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private Map<String, List<String>> getCopiedFieldMap(Map<String, FieldDesc> fieldDescMap) {
|
|
|
|
if (localCopiedFieldMap != null) {
|
|
|
|
return localCopiedFieldMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
synchronized (this) {
|
|
|
|
if (localCopiedFieldMap != null) {
|
|
|
|
// double check
|
|
|
|
return localCopiedFieldMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
Map<String, List<String>> copiedFieldMap = new HashMap<>();
|
|
|
|
for (Map.Entry<String, FieldDesc> entry : fieldDescMap.entrySet()) {
|
|
|
|
for (String dest : entry.getValue().copy_to) {
|
|
|
|
List<String> list = copiedFieldMap.get(dest);
|
|
|
|
if (list == null) {
|
|
|
|
list = new ArrayList<>();
|
|
|
|
copiedFieldMap.put(dest, list);
|
|
|
|
}
|
|
|
|
|
|
|
|
list.add(entry.getKey());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
localCopiedFieldMap = copiedFieldMap;
|
|
|
|
return localCopiedFieldMap;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<String> getSortedSearchFields() {
|
|
|
|
if (localMutilFields != null) {
|
|
|
|
return localMutilFields;
|
|
|
|
}
|
|
|
|
|
|
|
|
synchronized (this) {
|
|
|
|
if (localMutilFields != null) {
|
|
|
|
// double check
|
|
|
|
return localMutilFields;
|
|
|
|
}
|
|
|
|
|
|
|
|
List<FieldWithBoost> list = new ArrayList<>();
|
|
|
|
List<String> fields = ISearchConstants.SEARCH_DEFAULT_FIELD;
|
|
|
|
for (String field : fields) {
|
|
|
|
String[] fieldBoost = field.split("\\^");
|
|
|
|
if (fieldBoost.length == 2) {
|
|
|
|
list.add(new FieldWithBoost(fieldBoost[0], Integer.valueOf(fieldBoost[1])));
|
|
|
|
} else if (fieldBoost.length == 1) {
|
|
|
|
list.add(new FieldWithBoost(fieldBoost[0], 1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Collections.sort(list);
|
|
|
|
List<String> result = new ArrayList<>();
|
|
|
|
for (FieldWithBoost item : list) {
|
|
|
|
result.add(item.fieldName);
|
|
|
|
}
|
|
|
|
|
|
|
|
localMutilFields = result;
|
|
|
|
return localMutilFields;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static class FieldWithBoost implements Comparable {
|
|
|
|
int boost;
|
|
|
|
String fieldName;
|
|
|
|
|
|
|
|
public FieldWithBoost(String fieldName, int boost) {
|
|
|
|
this.boost = boost;
|
|
|
|
this.fieldName = fieldName;
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public int compareTo(Object o) {
|
|
|
|
return ((FieldWithBoost) o).boost - boost;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static class FieldDesc {
|
|
|
|
String field;
|
|
|
|
|
|
|
|
String type;
|
|
|
|
|
|
|
|
String index;
|
|
|
|
|
|
|
|
String analyzer;
|
|
|
|
|
|
|
|
String search_analyzer;
|
|
|
|
|
|
|
|
Set<String> copy_to = new HashSet<>();
|
|
|
|
|
|
|
|
public void addCopyTo(String item) {
|
|
|
|
copy_to.add(item);
|
|
|
|
}
|
|
|
|
|
|
|
|
public boolean noNeedAnalyzer() {
|
|
|
|
// 声明不分词的类型 或者 其他非string类型
|
|
|
|
return "not_analyzed".equals(index) || (type != null && !"string".equals(type));
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public String toString() {
|
|
|
|
return "FieldDesc{" +
|
|
|
|
"type='" + type + '\'' +
|
|
|
|
", index='" + index + '\'' +
|
|
|
|
", analyzer='" + analyzer + '\'' +
|
|
|
|
", search_analyzer='" + search_analyzer + '\'' +
|
|
|
|
'}';
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} |
...
|
...
|
|
