Authored by 胡古飞

找相似功能优化

package com.yoho.search.service.personalized;
import com.yoho.search.base.utils.HttpServletRequestUtils;
import com.yoho.search.service.service.SearchDynamicConfigService;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
... ... @@ -13,141 +13,163 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.Map;
import com.yoho.search.base.utils.HttpServletRequestUtils;
import com.yoho.search.service.service.SearchDynamicConfigService;
@Service
public class PersonalVectorFeatureSearch {
private static final Logger PERSONALIZED = LoggerFactory.getLogger("PERSONALIZED");
private static final Double BASE_CONSTANT = 1.0D;
private static final Double FACTOR_CONSTANT = 1.0D;
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
@Autowired
private PersonalizedRedisService personalizedRedisService;
public void addPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String,String> paramMap) {
// 1. 获取特征向量版本(即生成时间,该时间需要与skn的生成时间一致才有意义)
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return;
}
String uid = paramMap.get("uid");
// 2. 获取用户的特征向量
String userVectorFeature = personalizedRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
if (StringUtils.isEmpty(userVectorFeature)) {
return;
}
PERSONALIZED.info("do personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
// 3. 传入参数调用脚本
// field -> productindex索引种保存skn特征向量的字段名
// userFeatureFactors -> 用户特征向量值,多个值之间用逗号分隔
// vectorFeatureVersion -> 用户特征向量版本,当与skn的版本一致才计算相关性
// baseConstant,factorConstant -> 相关性常量和系数,计算规则为 baseConstant + factorConstant * cos(Vuser, Vskn)
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
scriptParams.put("userFeatureFactors", userVectorFeature);
scriptParams.put("vectorFeatureVersion", vectorFeatureVersion);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script("feature_factor_vector_score", ScriptService.ScriptType.INLINE, "native", scriptParams);
functionScoreQueryBuilder.add(ScoreFunctionBuilders.scriptFunction(script));
}
/**
* 根据UID和SKN列表获取特征相关性评分,该方法用于测试和问题定位
*
* @param uid 用户标识
* @param productVectorFeatureMap 商品特征map
* @return 相关性计算结果
*/
public Map<String, Object> calVectorFeature(String uid, String version, Map<String, String> productVectorFeatureMap) {
Map<String, Object> scoreMap = new HashMap<>();
if (productVectorFeatureMap == null || productVectorFeatureMap.isEmpty()) {
scoreMap.put("productVectorFeatureMap", "empty");
return scoreMap;
}
String vectorFeatureVersion = StringUtils.isNotEmpty(version) ? version : searchDynamicConfigService.personalizedSearchVersion();
scoreMap.put("vectorFeatureVersion", vectorFeatureVersion);
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return scoreMap;
}
// 2. 获取用户的特征向量
String userVectorFeature = personalizedRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
scoreMap.put("userVectorFeature", userVectorFeature);
if (StringUtils.isEmpty(userVectorFeature)) {
return scoreMap;
}
String[] userFeatureFactorArr = userVectorFeature.split(",");
double tempUserFeatureVectorNorm = 0.0D;
int dimensionOfFactors = userFeatureFactorArr.length;
double[] userFeatureFactors = new double[dimensionOfFactors];
double temp;
for (int index = 0; index < dimensionOfFactors; index++) {
temp = Double.parseDouble(userFeatureFactorArr[index].trim());
userFeatureFactors[index] = temp;
tempUserFeatureVectorNorm += temp * temp;
}
final double userFeatureVectorNorm = tempUserFeatureVectorNorm;
scoreMap.put("userFeatureVectorNorm", userFeatureVectorNorm);
scoreMap.put("dimensionOfFactors", dimensionOfFactors);
// 3. 计算相关性得分
productVectorFeatureMap.forEach((skn, vector) -> {
Map<String, Object> content = new HashMap<String, Object>();
content.put("vector", vector);
content.put("score", calculateScore(vector, vectorFeatureVersion, userFeatureFactors, userFeatureVectorNorm));
scoreMap.put(skn, content);
});
return scoreMap;
}
public double calculateScore(String productFeatureFactor, String vectorFeatureVersion, double[] userFeatureFactors, double userFeatureVectorNorm) {
if (productFeatureFactor == null || productFeatureFactor.trim().isEmpty()) {
return BASE_CONSTANT;
}
String versionPrefix = vectorFeatureVersion + "|";
if (!productFeatureFactor.trim().startsWith(versionPrefix)) {
return BASE_CONSTANT;
}
String[] productFeatureFactorArr = productFeatureFactor.trim().substring(versionPrefix.length()).split(",");
if (productFeatureFactorArr == null || productFeatureFactorArr.length != userFeatureFactors.length) {
return BASE_CONSTANT;
}
double prodFeatureVectorNorm = 0.0D;
double productiveSum = 0.0D;
double tempProdFactor;
for (int i = 0; i < userFeatureFactors.length; i++) {
tempProdFactor = Double.parseDouble(productFeatureFactorArr[i].trim());
productiveSum += tempProdFactor * userFeatureFactors[i];
prodFeatureVectorNorm += tempProdFactor * tempProdFactor;
}
if (prodFeatureVectorNorm == 0) {
return BASE_CONSTANT;
}
double cosScore = productiveSum / (Math.sqrt(prodFeatureVectorNorm) * Math.sqrt(userFeatureVectorNorm));
double finalScore = BASE_CONSTANT + FACTOR_CONSTANT * cosScore;
return finalScore;
}
private static final Logger PERSONALIZED = LoggerFactory.getLogger("PERSONALIZED");
private static final Double BASE_CONSTANT = 1.0D;
private static final Double FACTOR_CONSTANT = 1.0D;
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
@Autowired
private PersonalizedRedisService personalizedRedisService;
public void addPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
// 1. 获取特征向量版本(即生成时间,该时间需要与skn的生成时间一致才有意义)
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return;
}
String uid = paramMap.get("uid");
// 2. 获取用户的特征向量
String userVectorFeature = personalizedRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
if (StringUtils.isEmpty(userVectorFeature)) {
return;
}
PERSONALIZED.info("do personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
// 3. 传入参数调用脚本
// field -> productindex索引种保存skn特征向量的字段名
// userFeatureFactors -> 用户特征向量值,多个值之间用逗号分隔
// vectorFeatureVersion -> 用户特征向量版本,当与skn的版本一致才计算相关性
// baseConstant,factorConstant -> 相关性常量和系数,计算规则为 baseConstant +
// factorConstant * cos(Vuser, Vskn)
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
scriptParams.put("userFeatureFactors", userVectorFeature);
scriptParams.put("vectorFeatureVersion", vectorFeatureVersion);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script("feature_factor_vector_score", ScriptService.ScriptType.INLINE, "native", scriptParams);
functionScoreQueryBuilder.add(ScoreFunctionBuilders.scriptFunction(script));
}
public void addPersonalizedScriptScoreUserProductFeature(FunctionScoreQueryBuilder functionScoreQueryBuilder, String productVectorFeature) {
// 1. 获取特征向量版本(即生成时间,该时间需要与skn的生成时间一致才有意义)
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return;
}
// 2. 获取商品特征向量
if(StringUtils.isBlank(productVectorFeature)){
return;
}
// 3. 传入参数调用脚本,以商品特征代替用户特征
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
scriptParams.put("userFeatureFactors", productVectorFeature);
scriptParams.put("vectorFeatureVersion", vectorFeatureVersion);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script("feature_factor_vector_score", ScriptService.ScriptType.INLINE, "native", scriptParams);
functionScoreQueryBuilder.add(ScoreFunctionBuilders.scriptFunction(script));
}
/**
* 根据UID和SKN列表获取特征相关性评分,该方法用于测试和问题定位
*
* @param uid
* 用户标识
* @param productVectorFeatureMap
* 商品特征map
* @return 相关性计算结果
*/
public Map<String, Object> calVectorFeature(String uid, String version, Map<String, String> productVectorFeatureMap) {
Map<String, Object> scoreMap = new HashMap<>();
if (productVectorFeatureMap == null || productVectorFeatureMap.isEmpty()) {
scoreMap.put("productVectorFeatureMap", "empty");
return scoreMap;
}
String vectorFeatureVersion = StringUtils.isNotEmpty(version) ? version : searchDynamicConfigService.personalizedSearchVersion();
scoreMap.put("vectorFeatureVersion", vectorFeatureVersion);
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return scoreMap;
}
// 2. 获取用户的特征向量
String userVectorFeature = personalizedRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
scoreMap.put("userVectorFeature", userVectorFeature);
if (StringUtils.isEmpty(userVectorFeature)) {
return scoreMap;
}
String[] userFeatureFactorArr = userVectorFeature.split(",");
double tempUserFeatureVectorNorm = 0.0D;
int dimensionOfFactors = userFeatureFactorArr.length;
double[] userFeatureFactors = new double[dimensionOfFactors];
double temp;
for (int index = 0; index < dimensionOfFactors; index++) {
temp = Double.parseDouble(userFeatureFactorArr[index].trim());
userFeatureFactors[index] = temp;
tempUserFeatureVectorNorm += temp * temp;
}
final double userFeatureVectorNorm = tempUserFeatureVectorNorm;
scoreMap.put("userFeatureVectorNorm", userFeatureVectorNorm);
scoreMap.put("dimensionOfFactors", dimensionOfFactors);
// 3. 计算相关性得分
productVectorFeatureMap.forEach((skn, vector) -> {
Map<String, Object> content = new HashMap<String, Object>();
content.put("vector", vector);
content.put("score", calculateScore(vector, vectorFeatureVersion, userFeatureFactors, userFeatureVectorNorm));
scoreMap.put(skn, content);
});
return scoreMap;
}
public double calculateScore(String productFeatureFactor, String vectorFeatureVersion, double[] userFeatureFactors, double userFeatureVectorNorm) {
if (productFeatureFactor == null || productFeatureFactor.trim().isEmpty()) {
return BASE_CONSTANT;
}
String versionPrefix = vectorFeatureVersion + "|";
if (!productFeatureFactor.trim().startsWith(versionPrefix)) {
return BASE_CONSTANT;
}
String[] productFeatureFactorArr = productFeatureFactor.trim().substring(versionPrefix.length()).split(",");
if (productFeatureFactorArr == null || productFeatureFactorArr.length != userFeatureFactors.length) {
return BASE_CONSTANT;
}
double prodFeatureVectorNorm = 0.0D;
double productiveSum = 0.0D;
double tempProdFactor;
for (int i = 0; i < userFeatureFactors.length; i++) {
tempProdFactor = Double.parseDouble(productFeatureFactorArr[i].trim());
productiveSum += tempProdFactor * userFeatureFactors[i];
prodFeatureVectorNorm += tempProdFactor * tempProdFactor;
}
if (prodFeatureVectorNorm == 0) {
return BASE_CONSTANT;
}
double cosScore = productiveSum / (Math.sqrt(prodFeatureVectorNorm) * Math.sqrt(userFeatureVectorNorm));
double finalScore = BASE_CONSTANT + FACTOR_CONSTANT * cosScore;
return finalScore;
}
}
... ...
... ... @@ -11,27 +11,39 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import com.yoho.search.service.downgrade.aop.DownGradeAble;
import com.yoho.search.service.servicenew.ISearchLikeInShopService;
import com.yoho.search.service.servicenew.ISearchLikeService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.vo.SearchApiResult;
/**
* 找相似功能
*
* @author hugufei
*/
@Controller
public class SearchLikeController {
@Autowired
private ISearchLikeService searchLikeService;
@DownGradeAble(key="searchLike")
@Autowired
private ISearchLikeInShopService searchLikeInShopService;
@DownGradeAble(key = "searchLike")
@RequestMapping(method = RequestMethod.GET, value = "/searchLike")
@ResponseBody
public SearchApiResult searchLike(HttpServletRequest request){
public SearchApiResult searchLike(HttpServletRequest request) {
Map<String, String> paramMap = HttpServletRequestUtils.transParamType(request);
return searchLikeService.searchLike(paramMap);
}
@DownGradeAble(key = "searchLikeInShop")
@RequestMapping(method = RequestMethod.GET, value = "/searchLikeInShop")
@ResponseBody
public SearchApiResult searchLikeInShop(HttpServletRequest request) {
Map<String, String> paramMap = HttpServletRequestUtils.transParamType(request);
return searchLikeInShopService.searchLikeInShop(paramMap);
}
}
... ...
... ... @@ -29,7 +29,7 @@ public class FunctionScoreSearchHelper {
private SearchDynamicConfigService dynamicConfig;
static float globalWeight = 0.50f;
private WeightBuilder genWeightFactorBuilder(float factor) {
return ScoreFunctionBuilders.weightFactorFunction(factor);
}
... ... @@ -62,6 +62,19 @@ public class FunctionScoreSearchHelper {
return functionScoreQueryBuilder;
}
/**
* 直接使用商品特征的向量用来做个性化打分
*
* @param queryBuilder
* @param productFeature
* @return
*/
public QueryBuilder buildFunctionScoreQueryBuildWithProductFeature(QueryBuilder queryBuilder, String productFeature) {
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder);
personalVectorFeatureSearch.addPersonalizedScriptScoreUserProductFeature(functionScoreQueryBuilder, productFeature);
return functionScoreQueryBuilder;
}
private QueryBuilder getPhysicalChannelQueryBuilder(Map<String, String> paramMap) {
String physicalChannel = paramMap.get(SearchRequestParams.PHYSICAL_CHANNEL);
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
... ...
package com.yoho.search.service.servicenew;
import java.util.Map;
import com.yoho.search.service.vo.SearchApiResult;
public interface ISearchLikeInShopService {
/**
* 店铺内找相似功能
*
* @param paramMap
* @return
*/
public SearchApiResult searchLikeInShop(Map<String, String> paramMap);
}
... ...
package com.yoho.search.service.servicenew.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.service.helper.AggProductListHelper;
import com.yoho.search.service.service.helper.FunctionScoreSearchHelper;
import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.service.helper.SearchSortHelper;
import com.yoho.search.service.vo.SearchFieldBoost;
@Component
public class SearchLikeCommonService {
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchServiceHelper searchServiceHelper;
@Autowired
private SearchSortHelper searchSortHelper;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private FunctionScoreSearchHelper functionScoreSearchHelper;
@Autowired
private AggProductListHelper aggProductListHelper;
public List<SearchFieldBoost> searchFieldBoosts = new ArrayList<SearchFieldBoost>();
private int sortMaxBoost = 1000;
private int brandMaxBoost = 700;
private int productNameMaxBoost = 600;
private int standardOnlyNamesBoost = 500;
private int otherBoost = 150;
private void addSearchFieldBoost(List<String> searchFields, float maxBoost) {
float boost = maxBoost;
for (String searchField : searchFields) {
searchFieldBoosts.add(new SearchFieldBoost(searchField, boost));
boost = boost - 20;
}
}
@PostConstruct
void init() {
// 品类权重
List<String> searchFields = Arrays.asList("smallSort", "smallSort.smallSort_pinyin", "middleSort", "middleSort.middleSort_pinyin", "maxSort", "maxSort.maxSort_pinyin");
this.addSearchFieldBoost(searchFields, sortMaxBoost);
// 商品名称权重
searchFields = Arrays.asList("productName.productName_ansj");
this.addSearchFieldBoost(searchFields, productNameMaxBoost);
// 品牌权重
searchFields = Arrays.asList("brandName.brandName_lowercase", "brandName", "brandNameCn", "brandNameCn.brandNameCn_pinyin", "brandDomain", "brandNameEn");
this.addSearchFieldBoost(searchFields, brandMaxBoost);
// 规则权重
searchFields = Arrays.asList("standardOnlyNames.standardOnlyNames_ansj", "standardOnlyNames.standardOnlyNames_pinyin");
this.addSearchFieldBoost(searchFields, standardOnlyNamesBoost);
// 其他字段权重
searchFields = Arrays.asList("specialSearchField", "productKeyword", "brandKeyword", "genderS", "searchField_ansj", "searchField");
this.addSearchFieldBoost(searchFields, otherBoost);
}
/**
* 获取SKN在ES中的原生信息
*
* @param productSkn
* @return
*/
public JSONObject getProductInfoInEs(String productSkn) {
SearchParam searchParam = new SearchParam();
searchParam.setQuery(QueryBuilders.matchAllQuery());
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termsQuery("productSkn", productSkn));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
searchParam.setFiter(boolFilter);
searchParam.setAggregationBuilders(null);
searchParam.setPage(0);
searchParam.setOffset(0);
searchParam.setSize(1);
String productIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(productIndexName, searchParam);
if (jsonObject != null) {
return jsonObject;
}
SearchResult searchResult = searchCommonService.doSearch(productIndexName, searchParam);
if (searchResult == null) {
return null;
}
List<Map<String, Object>> productLists = searchResult.getResultList();
if (productLists == null || productLists.isEmpty()) {
return null;
}
jsonObject = new JSONObject();
jsonObject.putAll(productLists.get(0));
searchCacheService.addJSONObjectToCache(productIndexName, searchParam, jsonObject);
return jsonObject;
}
/**
* 根据ES的原生信息生成返回的结果
*
* @param productInfoInEs
* @return
*/
public Map<String, Object> genProductInfoResult(JSONObject productInfoInEs) {
List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
resultList.add(productInfoInEs);
List<Map<String, Object>> results = searchServiceHelper.getProductMapList(resultList);
if (results == null || results.isEmpty()) {
return null;
}
return results.get(0);
}
/**
* 找相似相关的匹配打分策略
*
* @param query
* @param minimumShouldMatch
* @param productFeatureFactor
* @return
*/
public QueryBuilder genSearchLikeQueryBuilder(String query, String minimumShouldMatch, String productFeatureFactor) {
// 1、设置字段权重
MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(query);
multiMatchQueryBuilder.type(MultiMatchQueryBuilder.Type.CROSS_FIELDS);
for (SearchFieldBoost searchFieldBoost : searchFieldBoosts) {
multiMatchQueryBuilder.field(searchFieldBoost.getFieldName(), searchFieldBoost.getBoost());
}
// 2、设置匹配度
multiMatchQueryBuilder.minimumShouldMatch(minimumShouldMatch);
// 3、添加基于商品特征的个性化打分
QueryBuilder queryBuilder = functionScoreSearchHelper.buildFunctionScoreQueryBuildWithProductFeature(multiMatchQueryBuilder, productFeatureFactor);
return queryBuilder;
}
/**
* 找相似默认的条件
*
* @param productSkn
* @return
*/
public BoolQueryBuilder genDefaultQueryBuilder(List<String> notProductSkns) {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y"));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
boolFilter.must(QueryBuilders.termQuery("status", 1));
boolFilter.must(QueryBuilders.termQuery("isOutlets", 2));
boolFilter.must(QueryBuilders.termQuery("attribute", 1));
boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(1));
if (notProductSkns != null && !notProductSkns.isEmpty()) {
boolFilter.mustNot(QueryBuilders.termsQuery("productSkn", notProductSkns));
}
return boolFilter;
}
}
... ...
package com.yoho.search.service.servicenew.impl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.cache.CacheEnum;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.service.helper.AggProductListHelper;
import com.yoho.search.service.service.helper.FunctionScoreSearchHelper;
import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.service.helper.SearchSortHelper;
import com.yoho.search.service.servicenew.ISearchLikeInShopService;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
@Service
public class SearchLikeInShopServiceImpl implements ISearchLikeInShopService {
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchServiceHelper searchServiceHelper;
@Autowired
private SearchSortHelper searchSortHelper;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private FunctionScoreSearchHelper functionScoreSearchHelper;
@Autowired
private AggProductListHelper aggProductListHelper;
@Autowired
private SearchLikeCommonService searchLikeCommonService;
public boolean isLegalInteger(Integer id) {
if (id != null && id > 0) {
return true;
}
return false;
}
@Override
public SearchApiResult searchLikeInShop(Map<String, String> paramMap) {
// 1、获取参数
String productSkn = paramMap.get(SearchRequestParams.PARAM_SYNC_SKN);
if (StringUtils.isBlank(productSkn)) {
return new SearchApiResult().setCode(400).setMessage("请输入SKN");
}
// 2、获取当前查询的SKN的基本信息
JSONObject productInfoInEs = searchLikeCommonService.getProductInfoInEs(productSkn);
if (productInfoInEs == null) {
return new SearchApiResult().setCode(400).setMessage("SKN不存在");
}
// 3、检测分页参数
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 60 : Integer.parseInt(paramMap.get("viewNum"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1 || pageSize < 0) {
return new SearchApiResult().setCode(400).setMessage("分页参数不合法");
}
if (pageSize > 60 || pageSize < 10) {
pageSize = 60;
}
// 4、设置SearchParam
SearchParam searchParam = new SearchParam();
// 4.1)设置query
String productName = productInfoInEs.getString("productName");
String standardOnlyNames = productInfoInEs.getString("standardOnlyNames");
String style = productInfoInEs.getString("style");
String brandName = productInfoInEs.getString("brandName");
String productFeatureFactor = productInfoInEs.getString("productFeatureFactor");
String query = brandName + "," + productName + "," + standardOnlyNames + "," + style;
QueryBuilder queryBuilder = searchLikeCommonService.genSearchLikeQueryBuilder(query, "40%", productFeatureFactor);
searchParam.setQuery(queryBuilder);
// 5、设置filter
BoolQueryBuilder boolFilter = searchLikeCommonService.genDefaultQueryBuilder(Arrays.asList(productSkn));
Integer brandId = productInfoInEs.getInteger("brandId");
Integer shopId = productInfoInEs.getInteger("shopId");
if (isLegalInteger(shopId)) {
boolFilter.must(QueryBuilders.termQuery("shopId", shopId));
} else if (isLegalInteger(brandId)) {
boolFilter.must(QueryBuilders.termQuery("brandId", brandId));
}
// 6、设置排序规则[按打分排序]
List<SortBuilder> sortBuilders = new ArrayList<SortBuilder>();
sortBuilders.add(SortBuilders.fieldSort("_score").order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
// 7、设置分页参数
searchParam.setPage(1);
searchParam.setOffset(0);
searchParam.setSize(pageSize);
// 8、从缓存中获取数据,有则直接返回
String productIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
CacheEnum cacheEnum = CacheEnum.EHCACHE;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(cacheEnum, productIndexName, searchParam);
if (jsonObject != null) {
return new SearchApiResult().setData(jsonObject);
}
// 9、执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult == null) {
return new SearchApiResult().setData(null);
}
// 10、构造返回结果并加入缓存
List<Map<String, Object>> productList = searchServiceHelper.getProductMapList(searchResult.getResultList());
jsonObject = new JSONObject();
jsonObject.put("page", 1);
jsonObject.put("page_total", 1);
jsonObject.put("page_size", pageSize);
jsonObject.put("total", productList.size());
jsonObject.put("product_info", searchLikeCommonService.genProductInfoResult(productInfoInEs));
jsonObject.put("product_list", productList);
searchCacheService.addJSONObjectToCache(cacheEnum, productIndexName, searchParam, jsonObject);
return new SearchApiResult().setData(jsonObject);
}
}
... ...
... ... @@ -6,11 +6,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
... ... @@ -27,7 +24,6 @@ import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
... ... @@ -42,7 +38,6 @@ import com.yoho.search.service.service.helper.SearchSortHelper;
import com.yoho.search.service.servicenew.ISearchLikeService;
import com.yoho.search.service.utils.SearchRequestParams;
import com.yoho.search.service.vo.SearchApiResult;
import com.yoho.search.service.vo.SearchFieldBoost;
@Service
public class SearchLikeServiceImpl implements ISearchLikeService {
... ... @@ -61,45 +56,8 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
private FunctionScoreSearchHelper functionScoreSearchHelper;
@Autowired
private AggProductListHelper aggProductListHelper;
private List<SearchFieldBoost> searchFieldBoosts = new ArrayList<SearchFieldBoost>();
private int sortMaxBoost = 1000;
private int brandMaxBoost = 700;
private int productNameMaxBoost = 600;
private int standardOnlyNamesBoost = 500;
private int otherBoost = 150;
private void addSearchFieldBoost(List<String> searchFields, float maxBoost) {
float boost = maxBoost;
for (String searchField : searchFields) {
searchFieldBoosts.add(new SearchFieldBoost(searchField, boost));
boost = boost - 20;
}
}
@PostConstruct
void init() {
// 品类权重
List<String> searchFields = Arrays.asList("smallSort", "smallSort.smallSort_pinyin", "middleSort", "middleSort.middleSort_pinyin", "maxSort", "maxSort.maxSort_pinyin");
this.addSearchFieldBoost(searchFields, sortMaxBoost);
// 商品名称权重
searchFields = Arrays.asList("productName.productName_ansj");
this.addSearchFieldBoost(searchFields, productNameMaxBoost);
// 品牌权重
searchFields = Arrays.asList("brandName.brandName_lowercase", "brandName", "brandNameCn", "brandNameCn.brandNameCn_pinyin", "brandDomain", "brandNameEn");
this.addSearchFieldBoost(searchFields, brandMaxBoost);
// 规则权重
searchFields = Arrays.asList("standardOnlyNames.standardOnlyNames_ansj", "standardOnlyNames.standardOnlyNames_pinyin");
this.addSearchFieldBoost(searchFields, standardOnlyNamesBoost);
// 其他字段权重
searchFields = Arrays.asList("specialSearchField", "productKeyword", "brandKeyword", "genderS", "searchField_ansj", "searchField");
this.addSearchFieldBoost(searchFields, otherBoost);
}
@Autowired
private SearchLikeCommonService searchLikeCommonService;
@Override
public SearchApiResult searchLike(Map<String, String> paramMap) {
... ... @@ -109,7 +67,7 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
return new SearchApiResult().setCode(400).setMessage("请输入SKN");
}
// 2、获取当前查询的SKN的基本信息
JSONObject productInfoInEs = this.getProductInfoInEs(productSkn);
JSONObject productInfoInEs = searchLikeCommonService.getProductInfoInEs(productSkn);
if (productInfoInEs == null) {
return new SearchApiResult().setCode(400).setMessage("SKN不存在");
}
... ... @@ -123,8 +81,8 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
pageSize = 60;
}
// 4、获取同一个品牌下一半商品
JSONArray inBrandProductList = this.getProductListInBrand(productInfoInEs, paramMap, pageSize / 2);
// 4、获取同一个品牌下最多5个商品
JSONArray inBrandProductList = this.getProductListInBrand(productInfoInEs, paramMap, 5);
// 5、获取不同品牌下的余下数量的商品[通过聚合的方式来查找]
List<String> notInProductSkns = new ArrayList<String>();
... ... @@ -145,58 +103,11 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
dataMap.put("page_total", 1);
dataMap.put("page_size", pageSize);
dataMap.put("total", productList.size());
dataMap.put("product_info", this.genProductInfoResult(productInfoInEs));
dataMap.put("product_info", searchLikeCommonService.genProductInfoResult(productInfoInEs));
dataMap.put("product_list", productList);
return new SearchApiResult().setData(dataMap);
}
private Map<String, Object> genProductInfoResult(JSONObject productInfoInEs) {
List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
resultList.add(productInfoInEs);
List<Map<String, Object>> results = searchServiceHelper.getProductMapList(resultList);
if (results == null || results.isEmpty()) {
return null;
}
return results.get(0);
}
/**
* 获取SKN在ES中的原生信息
*
* @param productSkn
* @return
*/
private JSONObject getProductInfoInEs(String productSkn) {
SearchParam searchParam = new SearchParam();
searchParam.setQuery(QueryBuilders.matchAllQuery());
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termsQuery("productSkn", productSkn));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
searchParam.setFiter(boolFilter);
searchParam.setAggregationBuilders(null);
searchParam.setPage(0);
searchParam.setOffset(0);
searchParam.setSize(1);
String productIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
JSONObject jsonObject = searchCacheService.getJSONObjectFromCache(productIndexName, searchParam);
if (jsonObject != null) {
return jsonObject;
}
SearchResult searchResult = searchCommonService.doSearch(productIndexName, searchParam);
if (searchResult == null) {
return null;
}
List<Map<String, Object>> productLists = searchResult.getResultList();
if (productLists == null || productLists.isEmpty()) {
return null;
}
jsonObject = new JSONObject();
jsonObject.putAll(productLists.get(0));
searchCacheService.addJSONObjectToCache(productIndexName, searchParam, jsonObject);
return jsonObject;
}
/**
* 找出该品牌下的最多10个相似商品
*
... ... @@ -310,27 +221,19 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
String standardOnlyNames = productInfoInEs.getString("standardOnlyNames");
String style = productInfoInEs.getString("style");
String brandName = productInfoInEs.getString("brandName");
String productFeatureFactor = productInfoInEs.getString("productFeatureFactor");
String query = standardOnlyNames + "," + style;
if (isInBrand) {
query = brandName + "," + productName + "," + query;
} else {
query = productName.replaceAll(brandName, "") + "," + query;
}
// 2、设置字段权重
MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(query);
multiMatchQueryBuilder.type(MultiMatchQueryBuilder.Type.CROSS_FIELDS);
for (SearchFieldBoost searchFieldBoost : searchFieldBoosts) {
multiMatchQueryBuilder.field(searchFieldBoost.getFieldName(), searchFieldBoost.getBoost());
}
// 3、设置匹配度
if (isInBrand) {
multiMatchQueryBuilder.minimumShouldMatch("30%");
} else {
multiMatchQueryBuilder.minimumShouldMatch("30%");
// 2、设置匹配度
String minimumShouldMatch = "30%";
if (!isInBrand) {
minimumShouldMatch = "40%";
}
// 4、添加个性化打分
QueryBuilder queryBuilder = functionScoreSearchHelper.buildFunctionScoreQueryBuild(multiMatchQueryBuilder, paramMap);
return queryBuilder;
return searchLikeCommonService.genSearchLikeQueryBuilder(query, minimumShouldMatch, productFeatureFactor);
}
private List<String> getGenderInfo(String gender) {
... ... @@ -344,7 +247,7 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
}
private BoolQueryBuilder genBoolQueryBuilder(JSONObject productInfoInEs, Map<String, String> paramMap, List<String> notProductSkns, boolean isInBrand) {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
BoolQueryBuilder boolFilter = searchLikeCommonService.genDefaultQueryBuilder(notProductSkns);
// 1)设置此SKN相关的过滤条件
String gender = productInfoInEs.getString("gender");
List<String> genderList = this.getGenderInfo(gender);
... ... @@ -358,29 +261,13 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
sortFilter.should(QueryBuilders.termQuery("middleSortId", middle_sort_id));
sortFilter.should(QueryBuilders.termQuery("smallSortId", small_sort_id));
boolFilter.must(sortFilter);
// 3)设置默认的过滤条件
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y"));
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y"));
boolFilter.must(QueryBuilders.termQuery("status", 1));
boolFilter.must(QueryBuilders.termQuery("isOutlets", 2));
boolFilter.must(QueryBuilders.termQuery("attribute", 1));
boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(1));
if (notProductSkns != null && notProductSkns.size() > 0) {
boolFilter.mustNot(QueryBuilders.termsQuery("productSkn", notProductSkns.toArray(new String[notProductSkns.size()])));
}
// 4)设置品牌
// 3)设置品牌
Integer brandId = productInfoInEs.getInteger("brandId");
if (isInBrand) {
boolFilter.must(QueryBuilders.termQuery("brandId", brandId));
} else {
boolFilter.mustNot(QueryBuilders.termQuery("brandId", brandId));
}
// 5)设置店铺
if (!isInBrand && paramMap.containsKey(SearchRequestParams.PARAM_SEARCH_SHOP) && StringUtils.isNotBlank(paramMap.get(SearchRequestParams.PARAM_SEARCH_SHOP))) {
int[] shopids = ConvertUtils.stringToIntArray(paramMap.get(SearchRequestParams.PARAM_SEARCH_SHOP), ",");
boolFilter.must(QueryBuilders.termsQuery("shopId", shopids));
}
return boolFilter;
}
... ... @@ -391,5 +278,4 @@ public class SearchLikeServiceImpl implements ISearchLikeService {
}
return jsonArray;
}
}
... ...