Authored by 胡古飞

添加基于购物车收藏夹的用户维度

... ... @@ -20,69 +20,92 @@ import com.yoho.search.service.service.SearchDynamicConfigService;
public class PersonalVectorFeatureSearch {
private static final Logger PERSONALIZED = LoggerFactory.getLogger("PERSONALIZED");
private static final Double BASE_CONSTANT = 1.0D;
private static final Double FACTOR_CONSTANT = 0.8D;
@Autowired
private SearchDynamicConfigService searchDynamicConfigService;
@Autowired
private PersonalizedRedisService personalizedRedisService;
public void addPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
private void doAddPersionalFunctionScoreQueryBuilder(FunctionScoreQueryBuilder functionScoreQueryBuilder, String userVectorFeature, String vectorFeatureVersion) {
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
scriptParams.put("userFeatureFactors", userVectorFeature);
scriptParams.put("vectorFeatureVersion", vectorFeatureVersion);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script("feature_factor_vector_score", ScriptService.ScriptType.INLINE, "native", scriptParams);
functionScoreQueryBuilder.add(ScoreFunctionBuilders.scriptFunction(script));
}
/**
* 通用的个性化打分逻辑
*
* @param functionScoreQueryBuilder
* @param paramMap
*/
public void addCommonPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
// 1. 获取特征向量版本(即生成时间,该时间需要与skn的生成时间一致才有意义)
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return;
}
String uid = paramMap.get("uid");
// 2. 获取用户的特征向量
String userVectorFeature = personalizedRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
if (StringUtils.isEmpty(userVectorFeature)) {
return;
}
PERSONALIZED.info("do common personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
// 3、添加个性化打分脚本
this.doAddPersionalFunctionScoreQueryBuilder(functionScoreQueryBuilder, userVectorFeature, vectorFeatureVersion);
}
PERSONALIZED.info("do personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
// 3. 传入参数调用脚本
// field -> productindex索引种保存skn特征向量的字段名
// userFeatureFactors -> 用户特征向量值,多个值之间用逗号分隔
// vectorFeatureVersion -> 用户特征向量版本,当与skn的版本一致才计算相关性
// baseConstant,factorConstant -> 相关性常量和系数,计算规则为 baseConstant +
// factorConstant * cos(Vuser, Vskn)
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
scriptParams.put("userFeatureFactors", userVectorFeature);
scriptParams.put("vectorFeatureVersion", vectorFeatureVersion);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script("feature_factor_vector_score", ScriptService.ScriptType.INLINE, "native", scriptParams);
functionScoreQueryBuilder.add(ScoreFunctionBuilders.scriptFunction(script));
/**
* 活动个性化打分逻辑【使用用户购物车、收藏夹的维度】
*
* @param functionScoreQueryBuilder
* @param paramMap
*/
public void addActivityPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
// 1. 获取特征向量版本(即生成时间,该时间需要与skn的生成时间一致才有意义)
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
return;
}
String uid = paramMap.get("uid");
// 2. 获取用户的特征向量
String userVectorFeature = personalizedRedisService.getUserActivityVectorFeature(uid, vectorFeatureVersion);
if (StringUtils.isEmpty(userVectorFeature)) {
userVectorFeature = personalizedRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
}
if (StringUtils.isEmpty(userVectorFeature)) {
return;
}
PERSONALIZED.info("do activity personal search , paramString is [{}]", HttpServletRequestUtils.genParamString(paramMap));
// 3、添加个性化打分脚本
this.doAddPersionalFunctionScoreQueryBuilder(functionScoreQueryBuilder, userVectorFeature, vectorFeatureVersion);
}
public void addPersonalizedScriptScoreUserProductFeature(FunctionScoreQueryBuilder functionScoreQueryBuilder, String productVectorFeature) {
/**
* 直接使用商品特征的个性化打分逻辑
*
* @param functionScoreQueryBuilder
* @param paramMap
*/
public void addProductPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, String productVectorFeature) {
// 1. 获取商品特征向量
if(StringUtils.isBlank(productVectorFeature)){
if (StringUtils.isBlank(productVectorFeature)) {
return;
}
// 2. 传入参数调用脚本,以商品特征代替用户特征
Map<String, Object> scriptParams = new HashMap<>();
scriptParams.put("field", "productFeatureFactor");
String [] productVectorFeatures = productVectorFeature.split("\\|",2);
if(productVectorFeatures.length!=2){
String[] productVectorFeatures = productVectorFeature.split("\\|", 2);
if (productVectorFeatures.length != 2) {
return;
}
scriptParams.put("userFeatureFactors", productVectorFeatures[1]);
scriptParams.put("vectorFeatureVersion", productVectorFeatures[0]);
scriptParams.put("baseConstant", BASE_CONSTANT);
scriptParams.put("factorConstant", FACTOR_CONSTANT);
Script script = new Script("feature_factor_vector_score", ScriptService.ScriptType.INLINE, "native", scriptParams);
functionScoreQueryBuilder.add(ScoreFunctionBuilders.scriptFunction(script));
// 3、添加个性化打分脚本
this.doAddPersionalFunctionScoreQueryBuilder(functionScoreQueryBuilder, productVectorFeatures[1], productVectorFeatures[0]);
}
/**
... ... @@ -172,5 +195,5 @@ public class PersonalVectorFeatureSearch {
double finalScore = BASE_CONSTANT + FACTOR_CONSTANT * cosScore;
return finalScore;
}
}
... ...
... ... @@ -20,6 +20,8 @@ public class PersonalizedRedisService {
// 保存用户特征向量的key格式,比如 “1022102:w2v:20170103” 值是一个字符串,各个值之间用逗号分隔
private static final String USER_FEATURE_KEY_TEMPLATE = "%s:w2v:%s";
private static final String USER_ACTIVITY_FEATURE_KEY_TEMPLATE = "%s:w2v_f:%s";//活动相关的向量空间【购物车、收藏夹】
private static final String USER_GENDER_KEY= "%s:gender";
@Autowired
... ... @@ -30,6 +32,11 @@ public class PersonalizedRedisService {
return bigDataRedisOper.getValue(key);
}
public String getUserActivityVectorFeature(String uid, String generateDate) {
String key = String.format(USER_ACTIVITY_FEATURE_KEY_TEMPLATE, uid, generateDate);
return bigDataRedisOper.getValue(key);
}
/**
* 获取用户性别维度
* @param uid
... ...
... ... @@ -60,7 +60,7 @@ public class FunctionScoreSearchHelper {
}
// 个性化搜索相关
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
this.addPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
this.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
}
// 针对全球购降分
if (searchCommonHelper.containGlobal(paramMap)) {
... ... @@ -87,15 +87,19 @@ public class FunctionScoreSearchHelper {
* @param functionScoreQueryBuilder
* @param paramMap
*/
public void addPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
public void addCommonPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
// 个性化搜索相关
personalVectorFeatureSearch.addPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
personalVectorFeatureSearch.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
if (searchCommonHelper.isNewRecPageDefault(paramMap)) {
this.addFirstShelveTimeScore(functionScoreQueryBuilder, newRecShelveTimeScore);
} else {
this.addFirstShelveTimeScore(functionScoreQueryBuilder, commonFirstShelveTimeScore);
}
}
public void addActivityPersonalizedScriptScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, Map<String, String> paramMap) {
personalVectorFeatureSearch.addActivityPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
}
private void addFirstShelveTimeScore(FunctionScoreQueryBuilder functionScoreQueryBuilder, FirstShelveTimeScore firstShelveTimeScore) {
int todayLastSecond = DateUtil.getLastTimeSecond(new Date());
... ... @@ -116,7 +120,7 @@ public class FunctionScoreSearchHelper {
*/
public QueryBuilder buildFunctionScoreQueryBuildWithProductFeature(QueryBuilder queryBuilder, String productFeature) {
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder);
personalVectorFeatureSearch.addPersonalizedScriptScoreUserProductFeature(functionScoreQueryBuilder, productFeature);
personalVectorFeatureSearch.addProductPersonalizedScriptScore(functionScoreQueryBuilder, productFeature);
return functionScoreQueryBuilder;
}
... ...
... ... @@ -11,6 +11,9 @@ import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
... ... @@ -40,6 +43,7 @@ import com.yoho.search.core.es.utils.IgnoreSomeException;
import com.yoho.search.service.service.AggregationService;
import com.yoho.search.service.service.SearchCacheService;
import com.yoho.search.service.service.SearchCommonService;
import com.yoho.search.service.service.helper.FunctionScoreSearchHelper;
import com.yoho.search.service.service.helper.SearchCommonHelper;
import com.yoho.search.service.service.helper.SearchServiceHelper;
import com.yoho.search.service.service.helper.SearchSortHelper;
... ... @@ -64,6 +68,8 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
private SearchCommonService searchCommonService;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private FunctionScoreSearchHelper functionScoreSearchHelper;
private ApplicationEventPublisher publisher;
... ... @@ -81,6 +87,12 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
aggTypeToEsField.put("sort", "smallSortId");
aggTypeToEsField.put("brand", "brandId");
}
private QueryBuilder builderActivityQueryBuilder(Map<String, String> paramMap) {
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(QueryBuilders.matchAllQuery());
functionScoreSearchHelper.addActivityPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
return functionScoreQueryBuilder;
}
@Override
public SearchApiResult aggProductList(Map<String, String> paramMap) {
... ... @@ -103,7 +115,7 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
// 3、构造查询条件
SearchParam searchParam = new SearchParam();
searchParam.setQuery(searchServiceHelper.constructQueryBuilderForProductList(paramMap));// 支持个性化
searchParam.setQuery(this.builderActivityQueryBuilder(paramMap));// 活动特定的个性化
searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap, null));
searchParam.setSize(0);
... ... @@ -201,7 +213,7 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
}
// 3、构造查询条件
SearchParam searchParam = new SearchParam();
searchParam.setQuery(searchServiceHelper.constructQueryBuilderForProductList(paramMap));// 支持个性化
searchParam.setQuery(this.builderActivityQueryBuilder(paramMap));// 活动特定的个性化
searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap, null));
searchParam.setSize(0);
... ... @@ -218,7 +230,7 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>();
// 4.1)构造父聚合:品牌或品类聚合【同时按子聚合的sort字段排序】
TermsBuilder brandAggregationBuilder = AggregationBuilders.terms(firstAggName).field("brandId").order(Terms.Order.aggregation("sort", sortOrder.equals(SortOrder.ASC)))
.size(200+pageSize);
.size(200 + pageSize);
// 4.2)添加子聚合:取得分最大的值
brandAggregationBuilder.subAggregation(AggregationBuilders.max("sort").field(sortField));
// 4.3)添加孙聚合:取打分最高的一个product
... ... @@ -244,7 +256,7 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
if (!aggMaps.containsKey(firstAggName)) {
return searchApiResult.setData("");
}
List<Map<String, Object>> productList = this.getProductListOrderByScore(((MultiBucketsAggregation) aggMaps.get(firstAggName)), pageSize,sortField,sortOrder);
List<Map<String, Object>> productList = this.getProductListOrderByScore(((MultiBucketsAggregation) aggMaps.get(firstAggName)), pageSize, sortField, sortOrder);
jsonObject = new JSONObject();
jsonObject.put("total", pageSize);
jsonObject.put("page", 1);
... ... @@ -302,19 +314,19 @@ public class AggProductListServiceImpl implements IAggProductListService, Applic
if (value == null) {
return 0;
}
if (value instanceof Float){
if (value instanceof Float) {
return ((Float) value).floatValue();
}
if (value instanceof Integer){
if (value instanceof Integer) {
return ((Integer) value);
}
if (value instanceof Long){
if (value instanceof Long) {
return ((Long) value);
}
if (value instanceof String){
if (value instanceof String) {
return Double.valueOf(value.toString());
}
if (value instanceof Double){
if (value instanceof Double) {
return Double.valueOf(value.toString());
}
return 0;
... ...
... ... @@ -142,7 +142,7 @@ public class GoodProductListService implements IGoodProductsService {
}
// 加上个性化打分
if (searchCommonHelper.isNeedPersonalSearch(paramMap)) {
functionScoreSearchHelper.addPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
functionScoreSearchHelper.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
}
return functionScoreQueryBuilder;
}
... ... @@ -244,7 +244,7 @@ public class GoodProductListService implements IGoodProductsService {
functionScoreQueryBuilder.add(QueryBuilders.termsQuery("productSkn", productSkns.split(",")), ScoreFunctionBuilders.weightFactorFunction(100));
}
// 强制加上个性化打分
functionScoreSearchHelper.addPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
functionScoreSearchHelper.addCommonPersonalizedScriptScore(functionScoreQueryBuilder, paramMap);
searchParam.setQuery(functionScoreQueryBuilder);
// 4、设置聚合条件
... ...