Authored by 胡古飞

优化个性化相关的代码

... ... @@ -3,79 +3,162 @@ package com.yoho.search.service.personalized;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.personalized.PConsts;
import com.yoho.search.service.personalized.model.SearchFeature;
import com.yoho.search.service.service.SearchDynamicConfigService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
@Service
public final class PersonalizedSearch {
private static final float MaxUserFeatureBoost = 50;
@Autowired
private UserFeaturesRedis userFeatures;
@Autowired
private SearchDynamicConfigService dynamicConfig;
private static final float MaxUserFeatureBoost = 50;
private static final Logger PERSONALIZED = LoggerFactory.getLogger("PERSONALIZED");
// public QueryBuilder builder(QueryBuilder queryBuilder, String uid, String
// pageId) {
// QueryBuilder qBuilder = null;
// List<SearchFeature> sfRedis = userFeatures.getUserFeaturesFromRedis(uid,
// pageId);
// if (sfRedis == null || sfRedis.isEmpty()) {
// qBuilder = queryBuilder;
// } else {
// BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// boolQueryBuilder.must(queryBuilder);
//
// List<SearchFeature> searchFeatures = null;
// // should个数超过一定数量, 按权重值desc排序, 截取数据
// if (sfRedis.size() > ISearchConstants.PERSONALIZED_SEARCH_SHOULD_MAX) {
// // 按照boost进行排序
// Collections.sort(sfRedis, new Comparator<SearchFeature>() {
// @Override
// public int compare(SearchFeature left, SearchFeature right) {
// Float fleft = Float.valueOf(left.getBoost());
// Float fright = Float.valueOf(right.getBoost());
// return fright.compareTo(fleft);
// }
// });
// searchFeatures = sfRedis.subList(0,
// ISearchConstants.PERSONALIZED_SEARCH_SHOULD_MAX);
// } else {
// searchFeatures = sfRedis;
// }
// // 获取用户最大的boost
// float maxBoost = this.getMaxBoost(searchFeatures);
// boolean isFuzzySearch = this.isFuzzySearch(pageId);
// for (SearchFeature searchFeature : searchFeatures) {
// float boost = searchFeature.getBoost();
// if (isFuzzySearch) {
// boost = getAdaptoredBoost(maxBoost, boost);
// }
// boolQueryBuilder.should(QueryBuilders.termQuery(searchFeature.getTargetParam(),
// searchFeature.getParamValues()).boost(boost));
// }
// qBuilder = boolQueryBuilder;
// }
//
// float factor = getFunctionScoreFactor(pageId);
// // new_score = old_score * log(2 + factor * page_boosts)
// String fieldName = getFuncScoreField(pageId);
// FunctionScoreQueryBuilder fsQueryBuilder = new
// FunctionScoreQueryBuilder(qBuilder);
// fsQueryBuilder.add(
// ScoreFunctionBuilders.fieldValueFactorFunction(fieldName).factor(factor).modifier(FieldValueFactorFunction.Modifier.LOG2P)
// .missing(PConsts.PRODUCT_FUNCTION_MISSING_VALUE)).boostMode(CombineFunction.MULT);
//
// return fsQueryBuilder;
// }
private String getUidFromParamMap(Map<String, String> paramMap) {
return paramMap.get("uid");
}
public QueryBuilder builder(QueryBuilder queryBuilder, String uid, String pageId) {
QueryBuilder qBuilder = null;
private String getPageIdFromParamMap(Map<String, String> paramMap) {
String pageId = paramMap.get("pageId");
if (StringUtils.isBlank(pageId)) {
pageId = PConsts.PAGE_ID_NEW;
}
return pageId;
}
/**
* 针对用户特征,对用户进行加分
*
* @param queryBuilder
* @param uid
* @param pageId
* @return
*/
public QueryBuilder buildPersonalizedQueryBuilder(QueryBuilder queryBuilder, Map<String, String> paramMap) {
PERSONALIZED.info("do personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
// 2、获取用户信息和页面信息
String uid = this.getUidFromParamMap(paramMap);
String pageId = this.getPageIdFromParamMap(paramMap);
List<SearchFeature> sfRedis = userFeatures.getUserFeaturesFromRedis(uid, pageId);
if (sfRedis == null || sfRedis.isEmpty()) {
qBuilder = queryBuilder;
} else {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(queryBuilder);
List<SearchFeature> searchFeatures = null;
// should个数超过一定数量, 按权重值desc排序, 截取数据
if (sfRedis.size() > ISearchConstants.PERSONALIZED_SEARCH_SHOULD_MAX) {
// 按照boost进行排序
Collections.sort(sfRedis, new Comparator<SearchFeature>() {
@Override
public int compare(SearchFeature left, SearchFeature right) {
Float fleft = Float.valueOf(left.getBoost());
Float fright = Float.valueOf(right.getBoost());
return fright.compareTo(fleft);
}
});
searchFeatures = sfRedis.subList(0, ISearchConstants.PERSONALIZED_SEARCH_SHOULD_MAX);
} else {
searchFeatures = sfRedis;
}
// 获取用户最大的boost
float maxBoost = this.getMaxBoost(searchFeatures);
boolean isFuzzySearch = this.isFuzzySearch(pageId);
for (SearchFeature searchFeature : searchFeatures) {
float boost = searchFeature.getBoost();
if (isFuzzySearch) {
boost = getAdaptoredBoost(maxBoost, boost);
return queryBuilder;
}
// 3、对 用户权重进行排序,should个数超过一定数量, 按权重值desc排序, 截取数据
List<SearchFeature> searchFeatures = null;
if (sfRedis.size() > ISearchConstants.PERSONALIZED_SEARCH_SHOULD_MAX) {
// 按照boost进行排序
Collections.sort(sfRedis, new Comparator<SearchFeature>() {
@Override
public int compare(SearchFeature left, SearchFeature right) {
Float fleft = Float.valueOf(left.getBoost());
Float fright = Float.valueOf(right.getBoost());
return fright.compareTo(fleft);
}
boolQueryBuilder.should(QueryBuilders.termQuery(searchFeature.getTargetParam(), searchFeature.getParamValues()).boost(boost));
});
searchFeatures = sfRedis.subList(0, ISearchConstants.PERSONALIZED_SEARCH_SHOULD_MAX);
} else {
searchFeatures = sfRedis;
}
// 4、对用户权重 进行加分
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(queryBuilder);
float maxBoost = this.getMaxBoost(searchFeatures);// 获取用户最大的boost,已调整boost值
boolean isFuzzySearch = this.isFuzzySearch(pageId);
for (SearchFeature searchFeature : searchFeatures) {
float boost = searchFeature.getBoost();
if (isFuzzySearch) {
boost = getAdaptoredBoost(maxBoost, boost);
}
qBuilder = boolQueryBuilder;
boolQueryBuilder.should(QueryBuilders.termQuery(searchFeature.getTargetParam(), searchFeature.getParamValues()).boost(boost));
}
return boolQueryBuilder;
}
public ScoreFunctionBuilder getPersonalizedScoreFunctionBuilder(Map<String, String> paramMap) {
String pageId = this.getPageIdFromParamMap(paramMap);
float factor = getFunctionScoreFactor(pageId);
// new_score = old_score * log(2 + factor * page_boosts)
String fieldName = getFuncScoreField(pageId);
FunctionScoreQueryBuilder fsQueryBuilder = new FunctionScoreQueryBuilder(qBuilder);
fsQueryBuilder.add(
ScoreFunctionBuilders.fieldValueFactorFunction(fieldName).factor(factor).modifier(FieldValueFactorFunction.Modifier.LOG2P)
.missing(PConsts.PRODUCT_FUNCTION_MISSING_VALUE)).boostMode(CombineFunction.MULT);
return fsQueryBuilder;
return ScoreFunctionBuilders.fieldValueFactorFunction(fieldName).factor(factor).modifier(FieldValueFactorFunction.Modifier.LOG2P)
.missing(PConsts.PRODUCT_FUNCTION_MISSING_VALUE);
}
private float getMaxBoost(List<SearchFeature> searchFeatures) {
float boost = 0f;
for (SearchFeature searchFeature : searchFeatures) {
... ... @@ -114,7 +197,7 @@ public final class PersonalizedSearch {
public static void main(String[] args) {
System.out.println(getAdaptoredBoost(459.90198f, 109.67697f));
}
private float getFunctionScoreFactor(String pageId) {
float factor = 0.0f;
// 对于模糊搜索, 需要把商品特征的权重得分降低, 其余的按其排序
... ...
package com.yoho.search.service.service.helper;
import java.util.Map;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.service.personalized.PersonalizedSearch;
import com.yoho.search.service.service.SearchDynamicConfigService;
@Component
public class FunctionScoreSearchHelper {
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private PersonalizedSearch personalizedSearch;
@Autowired
private SearchDynamicConfigService dynamicConfig;
private static float globalWeightFactor = 0.50f;
public QueryBuilder buildFunctionScoreQueryBuild(QueryBuilder queryBuilder, Map<String, String> paramMap) {
//个性化时先对用户特征加分
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
queryBuilder = personalizedSearch.buildPersonalizedQueryBuilder(queryBuilder, paramMap);
}
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder);
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
functionScoreQueryBuilder.add(personalizedSearch.getPersonalizedScoreFunctionBuilder(paramMap));
}
if (searchCommonHelper.containGlobal(paramMap)) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("isGlobal", "Y"), ScoreFunctionBuilders.weightFactorFunction(globalWeightFactor));
}
if (searchCommonHelper.isNeedDeScoreBrandSearch(paramMap)) {
functionScoreQueryBuilder.add(QueryBuilders.termQuery("isForbiddenSortBrand", "1"), ScoreFunctionBuilders.weightFactorFunction(0.000001f));
}
functionScoreQueryBuilder.boostMode(CombineFunction.MULT);
return functionScoreQueryBuilder;
}
}
... ...
... ... @@ -6,15 +6,12 @@ import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.service.service.SearchDynamicConfigService;
import com.yoho.search.service.utils.HttpServletRequestUtils;
import com.yoho.search.service.utils.SearchRequestParams;
@Component
... ... @@ -22,8 +19,29 @@ public class SearchCommonHelper {
@Autowired
private SearchDynamicConfigService dynamicConfig;
private static final Logger PERSONALIZED = LoggerFactory.getLogger("PERSONALIZED");
/**
* 是否需要开启个性化
*
* @param paramMap
* @return
*/
public boolean isNeedPersonalSearch(Map<String, String> paramMap) {
// 从 zk中动态获取是否使用个性化
boolean openPersonalized = dynamicConfig.openPersonalized();
if (!openPersonalized) {
return false;
}
String uid = paramMap.get("uid");
if (StringUtils.isBlank(uid) || uid.equals("0")) {
return false;
}
String sortFields = paramMap.get("order");
if (!StringUtils.isBlank(sortFields)) {
return false;
}
return true;
}
/**
* 是否需要对品牌降分
... ... @@ -56,25 +74,6 @@ public class SearchCommonHelper {
}
/**
* 是否需要开启个性化
*
* @param paramMap
* @return
*/
public boolean isNeedPersonalSearch(Map<String, String> paramMap) {
// 从 zk中动态获取是否使用个性化
boolean openPersonalized = dynamicConfig.openPersonalized();
if (openPersonalized) { // 启用个性化搜索
String uid = paramMap.get("uid");
if (!StringUtils.isBlank(uid) && !uid.equals("0")) {
PERSONALIZED.info("do personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
return true;
}
}
return false;
}
/**
* 判断搜索是否需要包含全球购
*
* @param paramMap
... ...
... ... @@ -54,6 +54,8 @@ public class SearchServiceHelper {
private SearchCommonService searchCommonService;
@Autowired
private DynamicSearchRuleHelper dynamicSearchRuleHelper;
@Autowired
private FunctionScoreSearchHelper functionScoreSearchHelper;
/**
* 构造关键字查询的query
... ... @@ -154,10 +156,13 @@ public class SearchServiceHelper {
*/
public QueryBuilder constructQueryBuilderForProductList(Map<String, String> paramMap) {
QueryBuilder queryBuilder = this.constructQueryBuilder(paramMap);
queryBuilder = this.buildPersonalSearch(queryBuilder, paramMap);
queryBuilder = this.buildFunctionScoreQueryBuild(queryBuilder, paramMap);
queryBuilder = functionScoreSearchHelper.buildFunctionScoreQueryBuild(queryBuilder, paramMap);
return queryBuilder;
// String dynamicRuleValue = dynamicSearchRuleHelper.getDynamicRuleValue(paramMap);
// queryBuilder = this.buildPersonalSearch(queryBuilder, paramMap);
// queryBuilder = this.buildFunctionScoreQueryBuild(queryBuilder, paramMap);
// return queryBuilder;
// String dynamicRuleValue = dynamicSearchRuleHelper.getDynamicRuleValue(paramMap);
// if (StringUtils.isEmpty(dynamicRuleValue) || "-1".equals(dynamicRuleValue)) {
// queryBuilder = this.buildGlobalSearch(queryBuilder, paramMap);
// queryBuilder = this.buildDeScoreBrandSearch(queryBuilder, paramMap);
... ... @@ -169,8 +174,9 @@ public class SearchServiceHelper {
public QueryBuilder constructOrQueryBuilderForProductList(Map<String, String> paramMap) {
QueryBuilder queryBuilder = this.constructOrQueryBuilder(paramMap);
queryBuilder = this.buildPersonalSearch(queryBuilder, paramMap);
queryBuilder = this.buildFunctionScoreQueryBuild(queryBuilder, paramMap);
queryBuilder = functionScoreSearchHelper.buildFunctionScoreQueryBuild(queryBuilder, paramMap);
// queryBuilder = this.buildPersonalSearch(queryBuilder, paramMap);
// queryBuilder = this.buildFunctionScoreQueryBuild(queryBuilder, paramMap);
return queryBuilder;
}
... ... @@ -180,15 +186,15 @@ public class SearchServiceHelper {
* @param paramMap
* @return
*/
private QueryBuilder buildPersonalSearch(QueryBuilder queryBuilder, Map<String, String> paramMap) {
if (!searchCommonHelper.isNeedPersonalSearch(paramMap)) {
return queryBuilder;
}
String uid = paramMap.get("uid");
String pageId = paramMap.get("pageId");
QueryBuilder builder = personalizedSearch.builder(queryBuilder, uid, pageId);
return builder;
}
// private QueryBuilder buildPersonalSearch(QueryBuilder queryBuilder, Map<String, String> paramMap) {
// if (!searchCommonHelper.isNeedPersonalSearch(paramMap)) {
// return queryBuilder;
// }
// String uid = paramMap.get("uid");
// String pageId = paramMap.get("pageId");
// QueryBuilder builder = personalizedSearch.builder(queryBuilder, uid, pageId);
// return builder;
// }
private static float globalWeightFactor = 0.50f;
... ...
... ... @@ -20,10 +20,10 @@ import com.yoho.search.base.utils.ISearchConstants;
@Component
public class SearchSortHelper {
@Autowired
private SearchCommonHelper searchCommonHelper;
private static final List<String> orderValues = new ArrayList<String>();
@PostConstruct
... ... @@ -94,7 +94,7 @@ public class SearchSortHelper {
}
return realOrder.toString().replaceFirst(",", "");
}
/**
*
* @param sortBuilders
... ... @@ -109,6 +109,29 @@ public class SearchSortHelper {
filteredFieldNames.add(fieldName);
sortBuilders.add(SortBuilders.fieldSort(fieldName).order(sortOrder));
}
/**
* 是否需要优先按分数排序
*
* @param paramMap
* @return
*/
private boolean isNeedScoreOrderFirst(Map<String, String> paramMap) {
// 1、模糊搜索且不传order时
String order = this.getLegalOrder(paramMap);
if (paramMap.containsKey("query") && StringUtils.isBlank(order)) {
return true;
}
// 2、开了个性化搜索时
if (searchCommonHelper.isNeedPersonalSearch(paramMap)){
return true;
}
// 3、需要做降分处理时
if (searchCommonHelper.isNeedDeScoreBrandSearch(paramMap)) {
return true;
}
return false;
}
/**
* 构造排序方式
... ... @@ -119,22 +142,16 @@ public class SearchSortHelper {
public List<SortBuilder> buildSortList(Map<String, String> paramMap) {
List<SortBuilder> sortBuilders = new ArrayList<SortBuilder>();
List<String> filteredFieldNames = new ArrayList<String>();
// 1、如果需要降分处理,则按_score排序
if (searchCommonHelper.isNeedDeScoreBrandSearch(paramMap)) {
// 1、判断是否需要分数优先
if (isNeedScoreOrderFirst(paramMap)) {
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "_score", SortOrder.DESC);
}
// 2、处理order
String order = this.getLegalOrder(paramMap);
// 2、order为空时
if (StringUtils.isBlank(order)) {
// 2.1模糊搜索或个性化搜索,则按_score排序
if (paramMap.containsKey("query") || searchCommonHelper.isNeedPersonalSearch(paramMap)) {
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "_score", SortOrder.DESC);
}
// 2.2添加次要排序条件
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "id", SortOrder.DESC);
return sortBuilders;
order = "";
}
// 3、根据order参数构造
String[] sortTypes = order.split(",");
for (String sortType : sortTypes) {
String[] sortParts = sortType.split(ISearchConstants.SPLIT_CHAR_COLON);
... ... @@ -153,6 +170,8 @@ public class SearchSortHelper {
this.addSortBuildSorts(sortBuilders, filteredFieldNames, fieldName, sortOrder);
}
}
// 3、任意条件下,都要添加次要排序条件
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "shelveTime", SortOrder.DESC);
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "id", SortOrder.DESC);
return sortBuilders;
}
... ...