yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 7c4b5c70954227d223691b349824eb0e784214fe 1 parent 52dd3259

向量代码兼容

Inline Side-by-side
Showing 9 changed files with 317 additions and 311 deletions
... ... @@ -28,7 +28,7 @@ public class RecommendBrandAggregation extends AbstractAggregation {
this.simpleFieldAggs = new ArrayList<SimpleFieldAgg>();
simpleFieldAggs.add(new SimpleFieldAgg(aggName(), ProductIndexEsField.brandId, 300));
// 构造TopHitOrder
if (personalVectorFeatureSearch.getPersonalizedSearch(paramMap) != null) {
if (personalVectorFeatureSearch.queryPersonalizedSearch(paramMap) != null) {
topHitOrder = "_score:desc";
}
}
... ...
... ... @@ -36,7 +36,7 @@ public class RecommendShopAggregation extends AbstractAggregation {
this.simpleFieldAggs = new ArrayList<SimpleFieldAgg>();
simpleFieldAggs.add(new SimpleFieldAgg(aggName(), ProductIndexEsField.shopId, shopCount));
// 构造topHit排序规则
if (personalVectorFeatureSearch.getPersonalizedSearch(paramMap) != null) {
if (personalVectorFeatureSearch.queryPersonalizedSearch(paramMap) != null) {
topHitOrder = "_score:desc";
}
}
... ...
... ... @@ -6,7 +6,9 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.service.recall.config.SpecialShopConstants;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
... ... @@ -34,252 +36,257 @@ import com.yoho.search.service.scene.general.ProductCountService;
@Component
public class FunctionScoreSearchHelper {
private static final Logger logger = LoggerFactory.getLogger(FunctionScoreSearchHelper.class);
private static final Logger logger = LoggerFactory.getLogger(FunctionScoreSearchHelper.class);
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private PersonalVectorFeatureSearch personalVectorFeatureSearch;
@Autowired
private SearchDynamicConfigService dynamicConfig;
@Autowired
private SearchScorerFactory searchScorerFactory;
@Autowired
private ProductCountService productCountService;
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private PersonalVectorFeatureSearch personalVectorFeatureSearch;
@Autowired
private SearchDynamicConfigService dynamicConfig;
@Autowired
private SearchScorerFactory searchScorerFactory;
@Autowired
private ProductCountService productCountService;
// 普通个性化的时间维度
private static final FirstShelveTimeScore COMMON_FIRST_SHELVE_SCORE = new FirstShelveTimeScore(30, 7, 23);
// 新品到着的个性化时间维度
private static final FirstShelveTimeScore NEW_REC_FIRST_SHELVE_SCORE = new FirstShelveTimeScore(4, 1, 3);
// 模糊搜索的个性化时间维度
private static final FirstShelveTimeScore FUZZY_FIRST_SHELVE_SCORE = new FirstShelveTimeScore(90, 60, 30);
// 普通个性化的时间维度
private static final FirstShelveTimeScore COMMON_FIRST_SHELVE_SCORE = new FirstShelveTimeScore(30, 7, 23);
// 新品到着的个性化时间维度
private static final FirstShelveTimeScore NEW_REC_FIRST_SHELVE_SCORE = new FirstShelveTimeScore(4, 1, 3);
// 模糊搜索的个性化时间维度
private static final FirstShelveTimeScore FUZZY_FIRST_SHELVE_SCORE = new FirstShelveTimeScore(90, 60, 30);
/**
* 添加打分规则
*
* @param queryBuilder
* @param boolQueryBuilder
* @param paramMap
* @return
*/
public QueryBuilder buildFunctionScoreQueryBuild(QueryBuilder queryBuilder, BoolQueryBuilder boolQueryBuilder, Map<String, String> paramMap) {
// 1、构造persionalFilter
BoolQueryBuilder persionalFilter = this.genPersionalFilter(queryBuilder, boolQueryBuilder);
// 2、获取打分器
String pageId = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_PAGEID, "0");
List<IScorer> scorers = this.getScorers(pageId, persionalFilter, paramMap);
if (scorers == null || scorers.isEmpty()) {
return new FunctionScoreQueryBuilder(queryBuilder);
}
// 3、构造functionScoreQueryBuilder
YohoFilterFunctionBuilders yohoFilterFunctionBuilders = new YohoFilterFunctionBuilders();
for (IScorer iScorer : scorers) {
if(iScorer!=null){
iScorer.addScorer(yohoFilterFunctionBuilders);
}
}
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder, yohoFilterFunctionBuilders.getFilterFunctionBuilders());
// 4、设置打分模式
functionScoreQueryBuilder.boostMode(CombineFunction.MULTIPLY);
return functionScoreQueryBuilder;
}
/**
* 添加打分规则
*
* @param queryBuilder
* @param boolQueryBuilder
* @param paramMap
* @return
*/
public QueryBuilder buildFunctionScoreQueryBuild(QueryBuilder queryBuilder, BoolQueryBuilder boolQueryBuilder, Map<String, String> paramMap) {
// 1、构造persionalFilter
BoolQueryBuilder persionalFilter = this.genPersionalFilter(queryBuilder, boolQueryBuilder);
// 2、获取打分器
String pageId = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_PAGEID, "0");
List<IScorer> scorers = this.getScorers(pageId, persionalFilter, paramMap);
if (scorers == null || scorers.isEmpty()) {
return new FunctionScoreQueryBuilder(queryBuilder);
}
// 3、构造functionScoreQueryBuilder
YohoFilterFunctionBuilders yohoFilterFunctionBuilders = new YohoFilterFunctionBuilders();
for (IScorer iScorer : scorers) {
if (iScorer != null) {
iScorer.addScorer(yohoFilterFunctionBuilders);
}
}
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder, yohoFilterFunctionBuilders.getFilterFunctionBuilders());
// 4、设置打分模式
functionScoreQueryBuilder.boostMode(CombineFunction.MULTIPLY);
return functionScoreQueryBuilder;
}
private BoolQueryBuilder genPersionalFilter(QueryBuilder queryBuilder, BoolQueryBuilder boolQueryBuilder) {
BoolQueryBuilder persionalFilter = QueryBuilders.boolQuery();
if (queryBuilder != null) {
persionalFilter.must(queryBuilder);
}
if (boolQueryBuilder != null) {
persionalFilter.must(boolQueryBuilder);
}
return persionalFilter;
}
private BoolQueryBuilder genPersionalFilter(QueryBuilder queryBuilder, BoolQueryBuilder boolQueryBuilder) {
BoolQueryBuilder persionalFilter = QueryBuilders.boolQuery();
if (queryBuilder != null) {
persionalFilter.must(queryBuilder);
}
if (boolQueryBuilder != null) {
persionalFilter.must(boolQueryBuilder);
}
return persionalFilter;
}
private List<IScorer> getScorers(String pageId, BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = null;
switch (pageId) {
case SearchPageIdDefine.PAGE_ID_SORT:
scorers = this.getSortPageScorers(persionalFilter, paramMap);
break;
case SearchPageIdDefine.PAGE_ID_SEARCH:
scorers = this.getFuzzyPageScorers(persionalFilter, paramMap);
break;
case SearchPageIdDefine.PAGE_ID_NEW:
scorers = this.getNewArrivePageScorers(persionalFilter, paramMap);
break;
default:
scorers = this.getOtherPageScorers(persionalFilter, paramMap);
break;
}
return scorers;
}
private List<IScorer> getScorers(String pageId, BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = null;
switch (pageId) {
case SearchPageIdDefine.PAGE_ID_SORT:
scorers = this.getSortPageScorers(persionalFilter, paramMap);
break;
case SearchPageIdDefine.PAGE_ID_SEARCH:
scorers = this.getFuzzyPageScorers(persionalFilter, paramMap);
break;
case SearchPageIdDefine.PAGE_ID_NEW:
scorers = this.getNewArrivePageScorers(persionalFilter, paramMap);
break;
default:
scorers = this.getOtherPageScorers(persionalFilter, paramMap);
break;
}
return scorers;
}
// 品类页的打分器
private List<IScorer> getSortPageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<IScorer>();
// 1、不是默认品类页，返回空列表
if (!searchCommonHelper.isSortPageDefault(paramMap)) {
return scorers;
}
// 2、获取通用打分器
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
// 3、添加首次上架时间的打分器
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(getOtherPageShelveTimeScore()));
// 4、品类页添加加分SKN的打分器
scorers.add(searchScorerFactory.getAddScoreSknsScorer());
return scorers;
}
// 品类页的打分器
private List<IScorer> getSortPageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<IScorer>();
// 1、不是默认品类页，返回空列表
if (!searchCommonHelper.isSortPageDefault(paramMap)) {
return scorers;
}
// 2、获取通用打分器
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
// 3、添加首次上架时间的打分器
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(getOtherPageShelveTimeScore()));
// 4、品类页添加加分SKN的打分器
scorers.add(searchScorerFactory.getAddScoreSknsScorer());
return scorers;
}
// 模糊搜索页的打分器
private List<IScorer> getFuzzyPageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<>();
// 1、不是默认搜索类页，返回空列表
if (!searchCommonHelper.isFuzzySearchPageDefault(paramMap)) {
return scorers;
}
// 2、获取通用打分器
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
// 3、添加首次上架时间的打分器
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(FUZZY_FIRST_SHELVE_SCORE));
// 4、添加频道搜索
scorers.add(searchScorerFactory.getChannelSearchScorer(paramMap));
// 5、添加线下可售商品打分器
scorers.add(searchScorerFactory.getOfflineSaleOnlyScorer());
// 6、添加关键词完全匹配的打分器
String query = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_QUERY);
if(StringUtils.isNotBlank(query)){
scorers.add(searchScorerFactory.getCsBrandKeyWordScorer(query));
}
// 7、添加特殊店铺的打分器
scorers.add(searchScorerFactory.getSpecialShopScorer(SpecialShopConstants.DOWNGRADE_SHOPIDS, SpecialShopConstants.DOWNGRADE_SHOP_WEIGHT));
return scorers;
}
// 新品到着页的打分器
private List<IScorer> getNewArrivePageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<IScorer>();
if (searchCommonHelper.isNewRecPageDefault(paramMap)) {
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(getNewArrivalPageShelveTimeScore()));
} else if (searchCommonHelper.isNewRecHeatValueDescSearch(paramMap)) {
scorers.add(searchScorerFactory.getNewArriveHeatDescScorer());// 新品到着页的人气值排序
}
return scorers;
}
// 模糊搜索页的打分器
private List<IScorer> getFuzzyPageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<>();
// 1、不是默认搜索类页，返回空列表
if (!searchCommonHelper.isFuzzySearchPageDefault(paramMap)) {
return scorers;
}
// 2、获取通用打分器
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
// 3、添加首次上架时间的打分器
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(FUZZY_FIRST_SHELVE_SCORE));
// 4、添加频道搜索
scorers.add(searchScorerFactory.getChannelSearchScorer(paramMap));
// 5、添加线下可售商品打分器
scorers.add(searchScorerFactory.getOfflineSaleOnlyScorer());
// 6、添加关键词完全匹配的打分器
String query = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_QUERY);
if (StringUtils.isNotBlank(query)) {
scorers.add(searchScorerFactory.getCsBrandKeyWordScorer(query));
}
// 7、添加特殊店铺的打分器
scorers.add(searchScorerFactory.getSpecialShopScorer(SpecialShopConstants.DOWNGRADE_SHOPIDS, SpecialShopConstants.DOWNGRADE_SHOP_WEIGHT));
return scorers;
}
/**
* 其他页面的打分规则
*
* @param persionalFilter
* @param paramMap
*/
private List<IScorer> getOtherPageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<IScorer>();
if (!searchCommonHelper.isOrderEmpty(paramMap)) {
return scorers;
}
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(getOtherPageShelveTimeScore()));
return scorers;
}
// 新品到着页的打分器
private List<IScorer> getNewArrivePageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<IScorer>();
if (searchCommonHelper.isNewRecPageDefault(paramMap)) {
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(getNewArrivalPageShelveTimeScore()));
} else if (searchCommonHelper.isNewRecHeatValueDescSearch(paramMap)) {
scorers.add(searchScorerFactory.getNewArriveHeatDescScorer());// 新品到着页的人气值排序
}
return scorers;
}
// 公用的打分器
private List<IScorer> getCommonScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
// 1、传了order，则都不生效
if (!searchCommonHelper.isOrderEmpty(paramMap)) {
return new ArrayList<IScorer>();
}
List<IScorer> scorers = new ArrayList<IScorer>();
// 2、全球购商品
scorers.add(searchScorerFactory.getGlobalProductSearch(paramMap));
// 3、一高三低商品降分[非新品并且零交际的商品]
scorers.add(searchScorerFactory.getProblemProductScorer());
// 4、first_product_skn
scorers.add(searchScorerFactory.getFirstProductSknScorer(paramMap));
// 5、断码商品
//scorers.add(searchScorerFactory.getBreakSizeProductScorer(paramMap));
// 6、基于向量的个性化打分
scorers.add(this.getPersonalVectorFeatureScorer(persionalFilter, paramMap));
return scorers;
}
/**
* 其他页面的打分规则
*
* @param persionalFilter
* @param paramMap
*/
private List<IScorer> getOtherPageScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
List<IScorer> scorers = new ArrayList<IScorer>();
if (!searchCommonHelper.isOrderEmpty(paramMap)) {
return scorers;
}
scorers.addAll(this.getCommonScorers(persionalFilter, paramMap));
scorers.add(searchScorerFactory.getFirstShelveTimeScorer(getOtherPageShelveTimeScore()));
return scorers;
}
/**
* 个性化的打分器
*
* @param persionalFilter
* @param paramMap
* @return
*/
private IScorer getPersonalVectorFeatureScorer(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
// 1、判断是否开启了个性化搜索
if (!searchCommonHelper.isNeedPersonalSearch(paramMap)) {
return null;
}
// 2、获取PersonalizedSearch
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.getPersonalizedSearch(paramMap);
if (personalizedSearch == null) {
return null;
}
// 3、为个性化打分添加filter条件
BoolQueryBuilder scoreFilter = productCountService.genScoreFilter(persionalFilter);
return searchScorerFactory.getFeatureFactorScorer(scoreFilter, personalizedSearch.getUserVectorFeature(), personalizedSearch.getVectorFeatureVersion());
}
// 公用的打分器
private List<IScorer> getCommonScorers(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
// 1、传了order，则都不生效
if (!searchCommonHelper.isOrderEmpty(paramMap)) {
return new ArrayList<IScorer>();
}
List<IScorer> scorers = new ArrayList<IScorer>();
// 2、全球购商品
scorers.add(searchScorerFactory.getGlobalProductSearch(paramMap));
// 3、一高三低商品降分[非新品并且零交际的商品]
scorers.add(searchScorerFactory.getProblemProductScorer());
// 4、first_product_skn
scorers.add(searchScorerFactory.getFirstProductSknScorer(paramMap));
// 5、断码商品
//scorers.add(searchScorerFactory.getBreakSizeProductScorer(paramMap));
// 6、基于向量的个性化打分
scorers.add(this.getPersonalVectorFeatureScorer(persionalFilter, paramMap));
return scorers;
}
/**
* 直接使用商品特征的向量用来做个性化打分
*
* @param queryBuilder
* @param productVectorFeature
* @return
*/
public QueryBuilder buildFunctionScoreQueryBuildWithProductFeature(QueryBuilder queryBuilder, String productVectorFeature) {
// 1. 获取商品特征向量
if (StringUtils.isBlank(productVectorFeature)) {
return queryBuilder;
}
// 2. 传入参数调用脚本，以商品特征代替用户特征
String[] productVectorFeatures = productVectorFeature.split("\\|", 2);
if (productVectorFeatures.length != 2) {
return queryBuilder;
}
IScorer scorer = searchScorerFactory.getFeatureFactorScorer(queryBuilder, productVectorFeatures[1], productVectorFeatures[0]);
YohoFilterFunctionBuilders yohoFilterFunctionBuilders = new YohoFilterFunctionBuilders();
if(scorer!=null){
scorer.addScorer(yohoFilterFunctionBuilders);
}
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,yohoFilterFunctionBuilders.getFilterFunctionBuilders());
return functionScoreQueryBuilder;
}
/**
* 个性化的打分器
*
* @param persionalFilter
* @param paramMap
* @return
*/
private IScorer getPersonalVectorFeatureScorer(BoolQueryBuilder persionalFilter, Map<String, String> paramMap) {
// 1、判断是否开启了个性化搜索
if (!searchCommonHelper.isNeedPersonalSearch(paramMap)) {
return null;
}
// 2、获取PersonalizedSearch
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.queryPersonalizedSearch(paramMap);
if (personalizedSearch == null) {
return null;
}
// 3、为个性化打分添加filter条件
BoolQueryBuilder scoreFilter = productCountService.genScoreFilter(persionalFilter);
return searchScorerFactory.getFeatureFactorScorer(scoreFilter, personalizedSearch.getVectorVersion(), personalizedSearch.getVectorFeatures());
}
// 新品到着页的根据首次上架时间衰减函数规则值
private FirstShelveTimeScore getNewArrivalPageShelveTimeScore() {
try {
String values = dynamicConfig.getNewArrivalPageDecayRuleValue();
List<Integer> valueList = Arrays.asList(values.split(",")).stream().map(Integer::valueOf).collect(Collectors.toList());
Assert.isTrue(valueList.size() == 2, "For " + values);
Integer scale = valueList.get(0) - valueList.get(1);
Assert.isTrue(scale.intValue() > 0, "For " + values);
return new FirstShelveTimeScore(valueList.get(0), valueList.get(1), scale);
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return NEW_REC_FIRST_SHELVE_SCORE;
}
}
/**
* 直接使用商品特征的向量用来做个性化打分
*
* @param queryBuilder
* @param productVectorFeature
* @return
*/
public QueryBuilder buildFunctionScoreQueryBuildWithProductFeature(QueryBuilder queryBuilder, String productVectorFeature) {
// 1. 获取商品特征向量
if (StringUtils.isBlank(productVectorFeature)) {
return queryBuilder;
}
// 2. 传入参数调用脚本，以商品特征代替用户特征
String[] productVectorFeatures = productVectorFeature.split("\\|", 2);
if (productVectorFeatures.length != 2) {
return queryBuilder;
}
// 3、参数检测
String featureVersion = productVectorFeatures[0];
List<Double> featureFactors = ConvertUtils.stringToDoubleList(productVectorFeatures[1], ",");
if (StringUtils.isBlank(featureVersion) || CollectionUtils.isEmpty(featureFactors)) {
return queryBuilder;
}
//4、构造scorer
IScorer scorer = searchScorerFactory.getFeatureFactorScorer(queryBuilder, featureVersion, featureFactors);
YohoFilterFunctionBuilders yohoFilterFunctionBuilders = new YohoFilterFunctionBuilders();
scorer.addScorer(yohoFilterFunctionBuilders);
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder, yohoFilterFunctionBuilders.getFilterFunctionBuilders());
return functionScoreQueryBuilder;
}
// 非新品到着页的根据首次上架时间衰减函数规则值
private FirstShelveTimeScore getOtherPageShelveTimeScore() {
try {
String values = dynamicConfig.getOtherPageDecayRuleValue();
List<Integer> valueList = Arrays.asList(values.split(",")).stream().map(Integer::valueOf).collect(Collectors.toList());
Assert.isTrue(valueList.size() == 2, "For " + values);
Integer scale = valueList.get(0) - valueList.get(1);
Assert.isTrue(scale.intValue() > 0, "For " + values);
return new FirstShelveTimeScore(valueList.get(0), valueList.get(1), scale);
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return COMMON_FIRST_SHELVE_SCORE;
}
}
// 新品到着页的根据首次上架时间衰减函数规则值
private FirstShelveTimeScore getNewArrivalPageShelveTimeScore() {
try {
String values = dynamicConfig.getNewArrivalPageDecayRuleValue();
List<Integer> valueList = Arrays.asList(values.split(",")).stream().map(Integer::valueOf).collect(Collectors.toList());
Assert.isTrue(valueList.size() == 2, "For " + values);
Integer scale = valueList.get(0) - valueList.get(1);
Assert.isTrue(scale.intValue() > 0, "For " + values);
return new FirstShelveTimeScore(valueList.get(0), valueList.get(1), scale);
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return NEW_REC_FIRST_SHELVE_SCORE;
}
}
// 非新品到着页的根据首次上架时间衰减函数规则值
private FirstShelveTimeScore getOtherPageShelveTimeScore() {
try {
String values = dynamicConfig.getOtherPageDecayRuleValue();
List<Integer> valueList = Arrays.asList(values.split(",")).stream().map(Integer::valueOf).collect(Collectors.toList());
Assert.isTrue(valueList.size() == 2, "For " + values);
Integer scale = valueList.get(0) - valueList.get(1);
Assert.isTrue(scale.intValue() > 0, "For " + values);
return new FirstShelveTimeScore(valueList.get(0), valueList.get(1), scale);
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return COMMON_FIRST_SHELVE_SCORE;
}
}
}
... ...
... ... @@ -182,7 +182,7 @@ public class UserRecallResponseBuilder {
//1、获取用户向量
Map<String, String> paramMap = new HashMap<>();
paramMap.put("uid", "" + uid);
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.getPersonalizedSearch(paramMap);
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.queryPersonalizedSearch(paramMap);
UserFeatureFactor userFeatureFactor = new UserFeatureFactor(personalizedSearch);
//2、计算相关性
int recommendSknIndex = 10000;
... ...
package com.yoho.search.service.recall.beans.persional;
import com.yoho.search.base.helper.Word2VectorCalculator;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.core.personalized.models.PersonalizedSearch;
import com.yoho.search.service.recall.models.personal.UserFeatureFactor;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
@Component
... ... @@ -21,23 +22,28 @@ public class ProductFeatureFactorComponent {
public double calProductFeatureFactor(UserFeatureFactor userFeatureFactor, String productFeatureFactor) {
try {
//商品向量不存在，则返回
if (StringUtils.isBlank(productFeatureFactor)) {
return 0;
}
//用户向量不存在，则返回0
if (userFeatureFactor == null || StringUtils.isBlank(userFeatureFactor.getVectorFeatureVersion())) {
if (userFeatureFactor == null || StringUtils.isBlank(userFeatureFactor.getVectorVersion())) {
return 0;
}
if (StringUtils.isBlank(productFeatureFactor)) {
if (CollectionUtils.isEmpty(userFeatureFactor.getVectorFeatureList())) {
return 0;
}
String versionPrefix = userFeatureFactor.getVectorFeatureVersion() + "|";
//版本不匹配，则返回0
String versionPrefix = userFeatureFactor.getVectorVersion() + "|";
if (!productFeatureFactor.trim().startsWith(versionPrefix)) {
return 0;
}
String[] productFeatureFactorArr = productFeatureFactor.trim().substring(versionPrefix.length()).split(",");
List<Double> productFeatureVectors = this.arrayToList(productFeatureFactorArr);
if (productFeatureVectors == null || productFeatureVectors.size() != userFeatureFactor.getUserFeatureVectorList().size()) {
//执行计算
List<Double> productFeatureVectors = ConvertUtils.stringToDoubleList(productFeatureFactor.trim().substring(versionPrefix.length()), ",");
if (productFeatureVectors == null || productFeatureVectors.size() != userFeatureFactor.getVectorFeatureList().size()) {
return 0;
}
double score = Word2VectorCalculator.calScore(userFeatureFactor.getUserFeatureVectorList(), userFeatureFactor.getUserFeatureVectorNorm(), productFeatureVectors);
double score = Word2VectorCalculator.calScore(userFeatureFactor.getVectorFeatureList(), userFeatureFactor.getUserFeatureVectorNorm(), productFeatureVectors);
double finalScore = baseConstant + factorConstant * score;
return finalScore;
} catch (Exception e) {
... ... @@ -46,22 +52,12 @@ public class ProductFeatureFactorComponent {
}
}
private List<Double> arrayToList(String[] productFeatureFactorStrArr) {
List<Double> results = new ArrayList<>();
if (productFeatureFactorStrArr == null) {
return results;
}
for (String productFeatureFactor : productFeatureFactorStrArr) {
results.add(Double.parseDouble(productFeatureFactor));
}
return results;
}
public static void main(String[] args) {
PersonalizedSearch personalizedSearch = new PersonalizedSearch("1", "20180408", "0.342045,-0.547933,0.291732,-0.056515,-0.182701,0.31113,0.151578,0.087678,-0.045536,-0.525699,-0.394715,-0.103153,-0.05575,-0.540641,0.028046,-0.193109,-0.003591,0.180923,0.290261,0.532309,-0.202463,-0.047271,-0.246197,0.324561,0.188814,0.36475,0.079007,0.455753,-0.11848,-0.135874,-0.187155,-0.055342,-0.12525,0.210669,-0.388331,-0.197123,0.132309,-0.4231,0.217752,-0.203266,0.190836,0.373428,-0.0102,-0.038654,0.2379,0.044424,0.071826,-0.201054,0.257434,0.141901,-0.390064,0.437099,0.559701,-0.040162,-0.193089,0.442338,-0.141678,-0.049696,0.315545,-0.028972,0.278694,-0.064345,-0.327943,0.103025,-0.40344,-0.34269,-0.237931,0.287046,0.139693,-0.38454,0.019959,-0.156907,0.374996,-0.074558,-0.019391,0.050522,0.315171,0.211605,-0.15418,0.502362,0.10184,0.153274,0.592659,-0.010284,0.28029,0.319741,-0.164559,0.286884,0.420483,-0.628866,-0.172259,0.027954,-0.411674,0.376585,0.322832,0.352039,0.078705,0.045152,0.139083,-0.164182");
List<Double> array = ConvertUtils.stringToDoubleList("0.342045,-0.547933,0.291732,-0.056515,-0.182701,0.31113,0.151578,0.087678,-0.045536,-0.525699,-0.394715,-0.103153,-0.05575,-0.540641,0.028046,-0.193109,-0.003591,0.180923,0.290261,0.532309,-0.202463,-0.047271,-0.246197,0.324561,0.188814,0.36475,0.079007,0.455753,-0.11848,-0.135874,-0.187155,-0.055342,-0.12525,0.210669,-0.388331,-0.197123,0.132309,-0.4231,0.217752,-0.203266,0.190836,0.373428,-0.0102,-0.038654,0.2379,0.044424,0.071826,-0.201054,0.257434,0.141901,-0.390064,0.437099,0.559701,-0.040162,-0.193089,0.442338,-0.141678,-0.049696,0.315545,-0.028972,0.278694,-0.064345,-0.327943,0.103025,-0.40344,-0.34269,-0.237931,0.287046,0.139693,-0.38454,0.019959,-0.156907,0.374996,-0.074558,-0.019391,0.050522,0.315171,0.211605,-0.15418,0.502362,0.10184,0.153274,0.592659,-0.010284,0.28029,0.319741,-0.164559,0.286884,0.420483,-0.628866,-0.172259,0.027954,-0.411674,0.376585,0.322832,0.352039,0.078705,0.045152,0.139083,-0.164182", ",");
PersonalizedSearch personalizedSearch = new PersonalizedSearch("1", "20180408", array);
UserFeatureFactor userFeatureFactor = new UserFeatureFactor(personalizedSearch);
String productFeatureFactor = "20180408|0.342045,-0.547933,0.291732,-0.056515,-0.182701,0.31113,0.151578,0.087678,-0.045536,-0.525699,-0.394715,-0.103153,-0.05575,-0.540641,0.028046,-0.193109,-0.003591,0.180923,0.290261,0.532309,-0.202463,-0.047271,-0.246197,0.324561,0.188814,0.36475,0.079007,0.455753,-0.11848,-0.135874,-0.187155,-0.055342,-0.12525,0.210669,-0.388331,-0.197123,0.132309,-0.4231,0.217752,-0.203266,0.190836,0.373428,-0.0102,-0.038654,0.2379,0.044424,0.071826,-0.201054,0.257434,0.141901,-0.390064,0.437099,0.559701,-0.040162,-0.193089,0.442338,-0.141678,-0.049696,0.315545,-0.028972,0.278694,-0.064345,-0.327943,0.103025,-0.40344,-0.34269,-0.237931,0.287046,0.139693,-0.38454,0.019959,-0.156907,0.374996,-0.074558,-0.019391,0.050522,0.315171,0.211605,-0.15418,0.502362,0.10184,0.153274,0.592659,-0.010284,0.28029,0.319741,-0.164559,0.286884,0.420483,-0.628866,-0.172259,0.027954,-0.411674,0.376585,0.322832,0.352039,0.078705,0.045152,0.139083,-0.164182";
System.out.println(new ProductFeatureFactorComponent().calProductFeatureFactor(userFeatureFactor,productFeatureFactor));
String productFeatureFactor = "20180408|1.342045,-0.547933,0.291732,-0.056515,-0.182701,0.31113,0.151578,0.087678,-0.045536,-0.525699,-0.394715,-0.103153,-0.05575,-0.540641,0.028046,-0.193109,-0.003591,0.180923,0.290261,0.532309,-0.202463,-0.047271,-0.246197,0.324561,0.188814,0.36475,0.079007,0.455753,-0.11848,-0.135874,-0.187155,-0.055342,-0.12525,0.210669,-0.388331,-0.197123,0.132309,-0.4231,0.217752,-0.203266,0.190836,0.373428,-0.0102,-0.038654,0.2379,0.044424,0.071826,-0.201054,0.257434,0.141901,-0.390064,0.437099,0.559701,-0.040162,-0.193089,0.442338,-0.141678,-0.049696,0.315545,-0.028972,0.278694,-0.064345,-0.327943,0.103025,-0.40344,-0.34269,-0.237931,0.287046,0.139693,-0.38454,0.019959,-0.156907,0.374996,-0.074558,-0.019391,0.050522,0.315171,0.211605,-0.15418,0.502362,0.10184,0.153274,0.592659,-0.010284,0.28029,0.319741,-0.164559,0.286884,0.420483,-0.628866,-0.172259,0.027954,-0.411674,0.376585,0.322832,0.352039,0.078705,0.045152,0.139083,-0.164182";
System.out.println(new ProductFeatureFactorComponent().calProductFeatureFactor(userFeatureFactor, productFeatureFactor));
}
}
... ...
package com.yoho.search.service.recall.models.personal;
import com.yoho.search.core.personalized.models.PersonalizedSearch;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
public class UserFeatureFactor {
private static final Logger logger = LoggerFactory.getLogger(UserFeatureFactor.class);
private List<Double> userFeatureVectorList = null;
private double userFeatureVectorNorm = 0.0D;
private String vectorFeatureVersion;
public UserFeatureFactor(PersonalizedSearch personalizedSearch) {
try {
if (personalizedSearch == null) {
return;
}
this.vectorFeatureVersion = personalizedSearch.getVectorFeatureVersion();
String userVectorFeature = personalizedSearch.getUserVectorFeature();
if(StringUtils.isBlank(userVectorFeature)){
return;
}
String[] userFeatureFactorStrArr = userVectorFeature.split(",");
if(userFeatureFactorStrArr.length==0){
return;
}
userFeatureVectorList = new ArrayList<>(userFeatureFactorStrArr.length);
for (String userFeatureFactorStr : userFeatureFactorStrArr) {
double userFeatureVector = Double.parseDouble(userFeatureFactorStr.trim());
userFeatureVectorList.add(userFeatureVector);
userFeatureVectorNorm += userFeatureVector * userFeatureVector;
}
}catch (Exception e){
logger.error(e.getMessage());
}
}
public List<Double> getUserFeatureVectorList() {
return userFeatureVectorList;
}
public double getUserFeatureVectorNorm() {
return userFeatureVectorNorm;
}
public String getVectorFeatureVersion() {
return vectorFeatureVersion;
}
private static final Logger logger = LoggerFactory.getLogger(UserFeatureFactor.class);
private String vectorVersion;
private List<Double> vectorFeatureList = null;
private double userFeatureVectorNorm = 0.0D;
public UserFeatureFactor(PersonalizedSearch personalizedSearch) {
try {
if (personalizedSearch == null) {
return;
}
this.vectorVersion = personalizedSearch.getVectorVersion();
this.vectorFeatureList = personalizedSearch.getVectorFeatures();
if (CollectionUtils.isEmpty(vectorFeatureList)) {
return;
}
for (Double vectorFeature : vectorFeatureList) {
userFeatureVectorNorm += vectorFeature * vectorFeature;
}
} catch (Exception e) {
logger.error(e.getMessage());
}
}
public String getVectorVersion() {
return vectorVersion;
}
public List<Double> getVectorFeatureList() {
return vectorFeatureList;
}
public double getUserFeatureVectorNorm() {
return userFeatureVectorNorm;
}
}
... ...
... ... @@ -102,8 +102,8 @@ public class SearchScorerFactory {
}
// 获取【向量余弦夹角】的打分器
public IScorer getFeatureFactorScorer(QueryBuilder scoreFilter, String featureFactors, String featureVersion) {
return new FeatureFactorScorer(scoreFilter, featureFactors, featureVersion);
public IScorer getFeatureFactorScorer(QueryBuilder scoreFilter,String featureVersion, List<Double> featureFactors) {
return new FeatureFactorScorer(scoreFilter,featureVersion,featureFactors);
}
public IScorer getOfflineSaleOnlyScorer() {
... ...
package com.yoho.search.service.scorer.impl;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.script.Script;
... ... @@ -21,22 +24,22 @@ public class FeatureFactorScorer implements IScorer {
private static final Double FACTOR_CONSTANT = 0.4D;
private QueryBuilder scoreFilter;
private String featureFactors;
private String featureVersion;
private String featureFactorString;
public FeatureFactorScorer(QueryBuilder scoreFilter, String featureFactors, String featureVersion) {
public FeatureFactorScorer(QueryBuilder scoreFilter, String featureVersion,List<Double> featureFactors) {
super();
this.scoreFilter = scoreFilter;
this.featureFactors = featureFactors;
this.featureVersion = featureVersion;
this.featureFactorString = StringUtils.join(featureFactors,",");
}
@Override
public void addScorer(YohoFilterFunctionBuilders yohoFilterFunctionBuilders) {
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
scriptParams.put("userFeatureFactors", featureFactors);
scriptParams.put("vectorFeatureVersion", featureVersion);
scriptParams.put("userFeatureFactors", featureFactorString);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script(ScriptType.INLINE, "native", "feature_factor_vector_score", scriptParams);
... ...
package com.yoho.search.service.scorer.personal;
import com.yoho.search.aop.cache.SearchCacheAble;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.cache.CacheType;
import com.yoho.search.core.personalized.models.PersonalizedSearch;
import com.yoho.search.core.personalized.service.BidataServiceCaller;
import com.yoho.search.core.personalized.service.PersonalVersionManager;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
@Service
... ... @@ -23,8 +26,8 @@ public class PersonalVectorFeatureSearch {
@Autowired
private BidataServiceCaller bidataServiceCaller;
@SearchCacheAble(cacheInMinute = 30, cacheName = "PERSIONAL_VECTOR", returnClass = PersonalizedSearch.class, cacheType = CacheType.SEARCH_REDIS, includeParams = { "uid" })
public PersonalizedSearch getPersonalizedSearch(Map<String, String> paramMap) {
@SearchCacheAble(cacheInMinute = 30, cacheName = "USER_PERSIONAL_VECTOR", returnClass = PersonalizedSearch.class, cacheType = CacheType.SEARCH_REDIS, includeParams = { "uid" })
public PersonalizedSearch queryPersonalizedSearch(Map<String, String> paramMap) {
try {
// 1、参数校验
String uid = paramMap.get("uid");
... ... @@ -41,7 +44,11 @@ public class PersonalVectorFeatureSearch {
if (StringUtils.isEmpty(userVectorFeature)) {
return null;
}
return new PersonalizedSearch(uid, vectorFeatureVersion, userVectorFeature);
List<Double> doubleList = ConvertUtils.stringToDoubleList(userVectorFeature,",");
if(CollectionUtils.isEmpty(doubleList)){
return null;
}
return new PersonalizedSearch(uid, vectorFeatureVersion, doubleList);
} catch (Exception e) {
logger.error(e.getMessage(),e);
return null;
... ...