yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit e7bd923b2dc0ae99caf6e423a2c5c960f3a5aa42 1 parent be9d28bb

使用SearchFieldFactory重构模糊搜索相关字段

Inline Side-by-side
Showing 7 changed files with 39 additions and 63 deletions
... ... @@ -20,10 +20,6 @@ public class SearchServiceConfiger {
return configurer.getString("search.es.servers","");
}
public List<String> getSearchDefaultField(){
return configurer.getList("search.default.field");
}
public String getSearchMultiMatchQueryType(){
return configurer.getString("search.multiMatchQuery.type", "BEST_FIELDS");
}
... ...
package com.yoho.search.recall.sort.helper;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.JsonUtil;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.personalized.BigDataRedisService;
import com.yoho.search.core.personalized.version.PersonalVersionManager;
import com.yoho.search.recall.sort.model.UserLikeBrands;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.SearchDynamicConfigService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.service.helper.SearchCommonHelper;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
... ... @@ -22,19 +30,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.JsonUtil;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.personalized.BigDataRedisService;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.SearchDynamicConfigService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.recall.sort.model.UserLikeBrands;
import java.util.*;
@Component
public class RecallPersionalHelper {
... ... @@ -44,6 +40,8 @@ public class RecallPersionalHelper {
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
@Autowired
private PersonalVersionManager personalVersionManager;
@Autowired
private BigDataRedisService bigDataRedisService;
@Autowired
private SearchCommonService searchCommonService;
... ... @@ -89,7 +87,7 @@ public class RecallPersionalHelper {
private List<Integer> getUserGlobalBrandIds(Map<String, String> paramMap) {
try {
String uid = MapUtils.getString(paramMap, "uid", "0");
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
String vectorFeatureVersion = personalVersionManager.getCurrentVersionInZk();
String userGlobalBrandIds = bigDataRedisService.getUserGlobalFaveriteBrand(uid, vectorFeatureVersion);
JSONArray brandJsonArray = JSON.parseArray(userGlobalBrandIds);
return JsonUtil.jsonArrayToList(brandJsonArray, Integer.class);
... ... @@ -108,7 +106,7 @@ public class RecallPersionalHelper {
private List<Integer> getUserLikeBrandIds(Map<String, String> paramMap) {
try {
String uid = MapUtils.getString(paramMap, "uid", "0");
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
String vectorFeatureVersion = personalVersionManager.getCurrentVersionInZk();
String userVectorFeature = bigDataRedisService.getUserBrandVectorFeature(uid, vectorFeatureVersion);
if (StringUtils.isBlank(vectorFeatureVersion) || StringUtils.isBlank(userVectorFeature)) {
return new ArrayList<Integer>();
... ...
... ... @@ -8,6 +8,8 @@ import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.SearchServiceConfiger;
import com.yoho.search.common.utils.SearchKeyWordUtils;
import com.yoho.search.core.es.model.SearchField;
import com.yoho.search.core.es.utils.SearchFieldFactory;
import com.yoho.search.service.base.SearchRequestParams;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
... ... @@ -94,14 +96,9 @@ public class SearchServiceHelper {
* @param queryBuilder
*/
private void setDefaultSearchField(MultiMatchQueryBuilder queryBuilder) {
List<String> fields = configurer.getSearchDefaultField();
for (String field : fields) {
String[] fieldBoost = field.split("\\^");
if (fieldBoost.length == 2) {
queryBuilder.field(fieldBoost[0], Float.parseFloat(fieldBoost[1]));
} else if (fieldBoost.length == 1) {
queryBuilder.field(fieldBoost[0]);
}
List<SearchField> fuzzySearchFields = SearchFieldFactory.getFuzzySearchFields();
for (SearchField searchField : fuzzySearchFields) {
queryBuilder.field(searchField.getEsField(),searchField.getBoost());
}
}
... ...
... ... @@ -7,8 +7,10 @@ import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.SearchServiceConfiger;
import com.yoho.search.core.es.IElasticsearchClient;
import com.yoho.search.core.es.model.SearchField;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.es.utils.SearchFieldFactory;
import com.yoho.search.service.base.ESClientMgr;
import com.yoho.search.service.searchexplainer.models.*;
import org.apache.commons.collections.CollectionUtils;
... ... @@ -686,14 +688,9 @@ public class SearchExplainerService {
return localMutilFields;
}
List<FieldWithBoost> list = new ArrayList<>();
List<String> fields = searchServiceConfiger.getSearchDefaultField();
for (String field : fields) {
String[] fieldBoost = field.split("\\^");
if (fieldBoost.length == 2) {
list.add(new FieldWithBoost(fieldBoost[0], Integer.valueOf(fieldBoost[1])));
} else if (fieldBoost.length == 1) {
list.add(new FieldWithBoost(fieldBoost[0], 1));
}
List<SearchField> searchFields = SearchFieldFactory.getFuzzySearchFields();
for (SearchField searchField : searchFields) {
list.add(new FieldWithBoost(searchField.getEsField(), (int)searchField.getBoost()));
}
Collections.sort(list);
localMutilFieldWithBoostList = list;
... ... @@ -701,7 +698,6 @@ public class SearchExplainerService {
for (FieldWithBoost item : localMutilFieldWithBoostList) {
result.add(item.fieldName);
}
localMutilFields = result;
return localMutilFields;
}
... ...
... ... @@ -5,8 +5,10 @@ import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.common.utils.SearchKeyWordUtils;
import com.yoho.search.core.es.model.SearchField;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.es.utils.SearchFieldFactory;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.models.SearchSort;
import com.yoho.search.service.base.SearchCommonService;
... ... @@ -38,6 +40,7 @@ import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.*;
@Service
... ... @@ -66,19 +69,9 @@ public class ShopListServiceImpl implements IShopListService {
// 2、构建query
MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keyword);
queryBuilder.operator(Operator.OR);
StringBuilder searchField = new StringBuilder();
searchField.append("brandName.brandName_lowercase^4000,brandName^900").append(',');
searchField.append("shopName.shopName_lowercase^4000,shopName^900").append(',');
searchField.append("brandNameCn^850,brandNameCn.brandNameCn_pinyin^850").append(',');
searchField.append("brandNameEn^800,brandDomain^700");
String[] fields = searchField.toString().split(",");
for (String field : fields) {
String[] fieldBoost = field.split("\\^");
if (fieldBoost.length == 2) {
queryBuilder.field(fieldBoost[0], Float.parseFloat(fieldBoost[1]));
} else if (fieldBoost.length == 1) {
queryBuilder.field(fieldBoost[0]);
}
List<SearchField> shopSearchFields = SearchFieldFactory.getShopNameSearchFields();
for (SearchField searchField: shopSearchFields) {
queryBuilder.field(searchField.getEsField(), searchField.getBoost());
}
queryBuilder.minimumShouldMatch("100%");
// 2.1 全球购得分减半
... ... @@ -255,9 +248,9 @@ public class ShopListServiceImpl implements IShopListService {
*/
private Map<String, Map<String, Object>> queryYohoValidShopMap(List<Integer> yohoShopIds) {
if (yohoShopIds == null || yohoShopIds.isEmpty()) {
return new HashMap<String, Map<String, Object>>();
return new HashMap<>();
}
Map<String, String> shopParam = new HashMap<String, String>();
Map<String, String> shopParam = new HashMap<>();
shopParam.put(SearchRequestParams.PARAM_SEARCH_SHOP, StringUtils.join(yohoShopIds, ","));
shopParam.put(SearchRequestParams.SHOPS_PARAM_STATUS, "1");
return shopsIndexBaseService.queryShopMapByParam(shopParam);
... ... @@ -265,8 +258,8 @@ public class ShopListServiceImpl implements IShopListService {
/**
* 获取全球购开店的品牌
*
* @param yohoShopIds
*
* @param globalBrandIds
* @return
*/
private Map<String, Map<String, Object>> queryGlobalValidMap(List<Integer> globalBrandIds) {
... ...
... ... @@ -30,8 +30,6 @@ search.index.translog.flush_threshold_ops=10000
#search
search.minimum.should.match=3<90%
search.operator=or
search.default.field=smallSort^1000,smallSort.smallSort_pinyin^1000,middleSort^950,middleSort.middleSort_pinyin^950,maxSort^900,maxSort.maxSort_pinyin^900,brandName^900,brandName.brandName_lowercase^4000,brandNameCn^850,brandNameCn.brandNameCn_pinyin^850,brandNameEn^800,brandDomain^800,brandKeyword^30,shopName^900,shopName.shopName_lowercase^4000,specialSearchField^700,specialSearchFieldPrice^700,specialSearchFieldVideo^700,productName.productName_ansj^300,standardOnlyNames.standardOnlyNames_pinyin^250,standardOnlyNames.standardOnlyNames_ansj^250,productKeyword^50,salesPhrase^50,marketPhrase^50,genderS^20,searchField_ansj^10,promotionTitles^5,searchField,productSkn.productSkn_ansj,productSkn.productSkn_ik
search.script.score=_score+doc['sortWeight'].value*0.003+(100-doc['breakingRate'].value)/100 * doc['salesWithDateDiff'].value/pow((now-doc['shelveTime'].value)/3600+2,1.8)
search.script.lang=groovy
search.use.cache=true
... ...
... ... @@ -30,8 +30,6 @@ search.index.translog.flush_threshold_ops=${search.index.translog.flush_threshol
#search
search.minimum.should.match=3<90%
search.operator=or
search.default.field=smallSort^1000,smallSort.smallSort_pinyin^1000,middleSort^950,middleSort.middleSort_pinyin^950,maxSort^900,maxSort.maxSort_pinyin^900,brandName^900,brandName.brandName_lowercase^4000,brandNameCn^850,brandNameCn.brandNameCn_pinyin^850,brandNameEn^800,brandDomain^800,brandKeyword^30,shopName^900,shopName.shopName_lowercase^4000,specialSearchField^700,specialSearchFieldPrice^700,specialSearchFieldVideo^700,productName.productName_ansj^300,standardOnlyNames.standardOnlyNames_pinyin^250,standardOnlyNames.standardOnlyNames_ansj^250,productKeyword^50,salesPhrase^50,marketPhrase^50,genderS^20,searchField_ansj^10,promotionTitles^5,searchField,productSkn.productSkn_ansj,productSkn.productSkn_ik
search.script.score=_score+doc['sortWeight'].value*0.003+(100-doc['breakingRate'].value)/100 * doc['salesWithDateDiff'].value/pow((now-doc['shelveTime'].value)/3600+2,1.8)
search.script.lang=groovy
search.use.cache=true
... ...