yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 2c942cd3a5ab55ae8c7cf17b3111b8f6d16177a4 1 parent 4f1ecf8c

代码优化

Inline Side-by-side
Showing 8 changed files with 78 additions and 83 deletions
package com.yoho.search.recall.scene.beans.cache;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
... ... @@ -14,9 +10,14 @@ import org.apache.commons.collections.MapUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Component
... ... @@ -25,38 +26,43 @@ public class SortBrandVectorCacheBean {
@Autowired
private SearchCommonService searchCommonService;
LoadingCache<String,Map<String,List<Double>>> SORT_BRAND_VECTOR_CACHE = CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<String, Map<String, List<Double>>>() {
@Override
public Map<String, List<Double>> load(String key) throws Exception {
return loadAllSortBrandVectors();
}
});
private ScheduledExecutorService schedule = Executors.newSingleThreadScheduledExecutor();
private Map<String, List<Double>> sortBrandVector;
@PostConstruct
void init() {
schedule.scheduleAtFixedRate(() -> sortBrandVector = loadAllSortBrandVectors(), 0, 5, TimeUnit.MINUTES);
}
public List<Double> queryBrandSortVector(SortBrand sortBrand){
public List<Double> queryBrandSortVector(SortBrand sortBrand) {
try {
Map<String,List<Double>> value = SORT_BRAND_VECTOR_CACHE.get("SORT_BRAND_VECTOR_CACHE_KEY");
return value.get(sortBrand.key());
}catch (Exception e){
return null;
if (sortBrandVector == null || sortBrandVector.isEmpty()) {
return new ArrayList<>();
}
return sortBrandVector.get(sortBrand.key());
} catch (Exception e) {
return new ArrayList<>();
}
}
private Map<String,List<Double>> loadAllSortBrandVectors(){
private Map<String, List<Double>> loadAllSortBrandVectors() {
SearchParam searchParam = new SearchParam();
searchParam.setSize(20000);
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_BIGDATASORTBRANDVECTOR,searchParam);
Map<String,List<Double>> result = new HashMap<>();
for (Map<String, Object> sortBrandVector : searchResult.getResultList()){
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_BIGDATASORTBRANDVECTOR, searchParam);
Map<String, List<Double>> result = new HashMap<>();
for (Map<String, Object> sortBrandVector : searchResult.getResultList()) {
try {
int middleSortId= MapUtils.getIntValue(sortBrandVector,"middleSortId",0);
int brandId = MapUtils.getIntValue(sortBrandVector,"brandId",0);
String vector = MapUtils.getString(sortBrandVector,"vector","[]");
List<Double> vectorArray = JSON.parseArray(vector,Double.class);
if(middleSortId==0 || brandId==0|| vectorArray.isEmpty()){
int middleSortId = MapUtils.getIntValue(sortBrandVector, "middleSortId", 0);
int brandId = MapUtils.getIntValue(sortBrandVector, "brandId", 0);
String vector = MapUtils.getString(sortBrandVector, "vector", "[]");
List<Double> vectorArray = JSON.parseArray(vector, Double.class);
if (middleSortId == 0 || brandId == 0 || vectorArray.isEmpty()) {
continue;
}
SortBrand sortBrand = new SortBrand(middleSortId,brandId);
result.put(sortBrand.key(),vectorArray);
}catch (Exception e){
SortBrand sortBrand = new SortBrand(middleSortId, brandId);
result.put(sortBrand.key(), vectorArray);
} catch (Exception e) {
}
}
... ...
... ... @@ -161,7 +161,7 @@ public class UserRecallCacheBean extends AbstractCacheBean<UserRecallRequest, Us
long begin = System.currentTimeMillis();
List<RecallRequest> commonRequests = commonRequestBuilder.buildCommonRecallRequests(userRecallRequest.getParamQueryFilter(), userRecallRequest.getFirstProductSkns());
List<RecallRequestResponse> commonRequestResponses = batchRecallCacheBean.batchRecallAndCache(commonRequests);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2]-doRecallCommon,requestCount is [{}], cost is [{}]", commonRequests.size(), System.currentTimeMillis() - begin);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2.1]-doRecallCommon,requestCount is [{}], cost is [{}]", commonRequests.size(), System.currentTimeMillis() - begin);
return commonRequestResponses;
}, recallExecutorService);
}
... ... @@ -179,7 +179,7 @@ public class UserRecallCacheBean extends AbstractCacheBean<UserRecallRequest, Us
int recommendSknCount = userPersonalFactor.getRecommendSknList() == null ? 0 : userPersonalFactor.getRecommendSknList().size();
int realTimeSimilarSknCount = userPersonalFactor.getRealTimeSimilarSknList() == null ? 0 : userPersonalFactor.getRealTimeSimilarSknList().size();
List<RecallRequestResponse> recommendSknRequestResponses = sknRecallCacheBean.batchRecallBySknList(userRecallRequest, userPersonalFactor, 8);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2]-doRecallRecommendSkn,recommendSknCount is [{}],realTimeSimilarSknCount is[{}] ,cost is [{}]", recommendSknCount, realTimeSimilarSknCount, System.currentTimeMillis() - begin);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2.2]-doRecallRecommendSkn,recommendSknCount is [{}],realTimeSimilarSknCount is[{}] ,cost is [{}]", recommendSknCount, realTimeSimilarSknCount, System.currentTimeMillis() - begin);
return recommendSknRequestResponses;
}, recallExecutorService);
}
... ... @@ -196,7 +196,7 @@ public class UserRecallCacheBean extends AbstractCacheBean<UserRecallRequest, Us
long begin = System.currentTimeMillis();
List<RecallRequest> realTimeSortBrandRequests = sortBrandRecallRequestBuilder.buildSortBrandRecallRequests(userRecallRequest.getParamQueryFilter(), userPersonalFactor.getRealTimeSortBrandList(), SortBrandType.REC_SORT_BRAND);
List<RecallRequestResponse> realTimeSortBrandRequestResponses = batchRecallCacheBean.batchRecallAndCache(realTimeSortBrandRequests);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2]-doRecallRealTimeSortBrand,requestCount is [{}], cost is [{}]", realTimeSortBrandRequests.size(), System.currentTimeMillis() - begin);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2.3]-doRecallRealTimeSortBrand,requestCount is [{}], cost is [{}]", realTimeSortBrandRequests.size(), System.currentTimeMillis() - begin);
return realTimeSortBrandRequestResponses;
}, recallExecutorService);
}
... ... @@ -211,9 +211,9 @@ public class UserRecallCacheBean extends AbstractCacheBean<UserRecallRequest, Us
private CompletableFuture<List<RecallRequestResponse>> doRecallVectorSortBrand(UserRecallRequest userRecallRequest, UserPersonalFactor userPersonalFactor) {
return CompletableFuture.supplyAsync(() -> {
long begin = System.currentTimeMillis();
List<RecallRequest> forecastSortBrandRequests = sortBrandRecallRequestBuilder.buildSortBrandRecallRequests(userRecallRequest.getParamQueryFilter(), userPersonalFactor.getVectorSortBrandList(), SortBrandType.VEC_SORT_BRAND);
List<RecallRequestResponse> vecSortBrandRequestsResponses = batchRecallCacheBean.batchRecallAndCache(forecastSortBrandRequests);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2]-doRecallVectorSortBrand,requestCount is [{}], cost is [{}]", forecastSortBrandRequests.size(), System.currentTimeMillis() - begin);
List<RecallRequest> vecortSortBrandRequests = sortBrandRecallRequestBuilder.buildSortBrandRecallRequests(userRecallRequest.getParamQueryFilter(), userPersonalFactor.getVectorSortBrandList(), SortBrandType.VEC_SORT_BRAND);
List<RecallRequestResponse> vecSortBrandRequestsResponses = batchRecallCacheBean.batchRecallAndCache(vecortSortBrandRequests);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2.4]-doRecallVectorSortBrand,requestCount is [{}], cost is [{}]", vecortSortBrandRequests.size(), System.currentTimeMillis() - begin);
return vecSortBrandRequestsResponses;
}, recallExecutorService);
}
... ... @@ -230,7 +230,7 @@ public class UserRecallCacheBean extends AbstractCacheBean<UserRecallRequest, Us
long begin = System.currentTimeMillis();
List<RecallRequest> forecastSortBrandRequests = sortBrandRecallRequestBuilder.buildSortBrandRecallRequests(userRecallRequest.getParamQueryFilter(), userPersonalFactor.getForecastSortBrandList(), SortBrandType.PRED_SORT_BRAND);
List<RecallRequestResponse> forecastSortBrandRequestsResponses = batchRecallCacheBean.batchRecallAndCache(forecastSortBrandRequests);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2]-doRecallForecastSortBrand,requestCount is [{}], cost is [{}]", forecastSortBrandRequests.size(), System.currentTimeMillis() - begin);
RECALL_NEW_LOGGER.info("UserRecallCacheBean[2.5]-doRecallForecastSortBrand,requestCount is [{}], cost is [{}]", forecastSortBrandRequests.size(), System.currentTimeMillis() - begin);
return forecastSortBrandRequestsResponses;
}, recallExecutorService);
}
... ...
... ... @@ -57,6 +57,10 @@ public class QueryUserPersionalFactorBean {
//2、获取用户的个性化因子
begin = System.currentTimeMillis();
UserPersonalFactorRspNew userFactor = userComponent.queryUserPersionalFactor(userRecallRequest.getUid(), userRecallRequest.getUdid(), pageFactor.getMisortIds());
if(userFactor.getSortBrandVector()==null || userFactor.getSortBrandVector().isEmpty()){
String text = "[-0.06842552870512009, 0.008846139535307884, 0.058502957224845886, 0.08840706944465637, -0.061707016080617905, 0.02252822555601597, -0.03671540692448616, -0.005050729028880596, -0.09626617282629013, -0.10487467795610428, 0.025082364678382874, -0.02112266607582569, 0.0006435245741158724, 0.011133095249533653, -0.010220825672149658, -0.007785319350659847, -0.004194584209471941, -0.06750216335058212, -0.05586543306708336, -0.05108240246772766]";
userFactor.setSortBrandVector(JSON.parseArray(text,Double.class));
}
cost = System.currentTimeMillis() - begin;
int forecastSortBrandSize = userFactor.getSortBrandList().size();
int realTimeSortBrandSize = userFactor.getRealTimeSortBrandList().size();
... ... @@ -69,9 +73,9 @@ public class QueryUserPersionalFactorBean {
//3、构造结果
UserPersonalFactor userPersonalFactor = this.buildUserPersonalFactor(pageFactor, userFactor);
if (!openDetailLog) {
RECALL_NEW_LOGGER.info("QueryUserPersionalFactorBean[3]:after join. uid is[{}],udid is[{}], forecastSortBrand size is[{}], realTimeSortBrand size is[{}], sortPriceAreas size is [{}] ", uid, udid, userPersonalFactor.getForecastSortBrandListSize(), userPersonalFactor.getRealTimeSortBrandListSize(), userPersonalFactor.getSortPriceAreasListSize());
RECALL_NEW_LOGGER.info("QueryUserPersionalFactorBean[3]:after join. uid is[{}],udid is[{}], forecastSortBrand size is[{}], realTimeSortBrand size is[{}], vectorSortBrand size is[{}], sortPriceAreas size is [{}] ", uid, udid, userPersonalFactor.getForecastSortBrandListSize(), userPersonalFactor.getRealTimeSortBrandListSize(),userPersonalFactor.getVectorSortBrandListSize(), userPersonalFactor.getSortPriceAreasListSize());
} else {
RECALL_NEW_LOGGER.info("QueryUserPersionalFactorBean[3]:after join. uid is[{}],udid is[{}], forecastSortBrand size is[{}], realTimeSortBrand size is[{}], sortPriceAreas size is [{}], results is [{}] ", uid, udid, userPersonalFactor.getForecastSortBrandListSize(), userPersonalFactor.getRealTimeSortBrandListSize(), userPersonalFactor.getSortPriceAreasListSize(), JSON.toJSONString(userPersonalFactor));
RECALL_NEW_LOGGER.info("QueryUserPersionalFactorBean[3]:after join. uid is[{}],udid is[{}], forecastSortBrand size is[{}], realTimeSortBrand size is[{}], vectorSortBrand size is[{}], sortPriceAreas size is [{}], results is [{}] ", uid, udid, userPersonalFactor.getForecastSortBrandListSize(), userPersonalFactor.getRealTimeSortBrandListSize(),userPersonalFactor.getVectorSortBrandListSize(), userPersonalFactor.getSortPriceAreasListSize(), JSON.toJSONString(userPersonalFactor));
}
return userPersonalFactor;
} catch (Exception e) {
... ... @@ -93,16 +97,18 @@ public class QueryUserPersionalFactorBean {
for (SortBrand existSortBrand : realTimeSortBrandList) {
filterSortBrandKeys.add(existSortBrand.key());
}
//3、构造预测的【品牌+品牌】,去除实时的品类和品牌
//3、构造基于向量的【品牌+品牌】,去除实时的品类和品牌
int maxVectorSortBrandCount = searchDynamicConfigService.maxVectorSortBrandCount();
List<SortBrand> vectorSortBrandList = sortBrandVectorComponent.queryVectorSortBrandList(pageFactor.getSortBrandList(),filterSortBrandKeys,userFactor.getSortBrandVector(), maxVectorSortBrandCount);
for (SortBrand existSortBrand : vectorSortBrandList) {
filterSortBrandKeys.add(existSortBrand.key());
}
//4、构造预测的【品牌+品牌】,去除实时的品类和品牌
int maxForecastSortBrandCount = searchDynamicConfigService.maxForecastSortBrandCount();
int maxJoinSortBrandCount = searchDynamicConfigService.maxJoinSortBrandCount();
int count = Math.max(maxForecastSortBrandCount, maxJoinSortBrandCount - realTimeSortBrandList.size());
List<SortBrand> forecastSortBrandList = this.getSortBrandListWithSort(pageSortBrandKeys,filterSortBrandKeys,userFactor.getSortBrandList(), count);
List<SortBrand> forecastSortBrandList = this.getSortBrandListWithSort(pageSortBrandKeys,filterSortBrandKeys,userFactor.getSortBrandList(), maxForecastSortBrandCount);
for (SortBrand existSortBrand : forecastSortBrandList) {
filterSortBrandKeys.add(existSortBrand.key());
}
//4、构造基于向量的【品牌+品牌】,去除实时的品类和品牌
List<SortBrand> vectorSortBrandList = sortBrandVectorComponent.queryVectorSortBrandList(pageFactor.getSortBrandList(),filterSortBrandKeys,userFactor.getSortBrandVector(), count);
//5、构造品类价格带
List<SortPriceAreas> sortPriceAreasList = this.getSortPriceAreasListWithSort(userFactor, pageFactor);
... ...
... ... @@ -23,6 +23,10 @@ public class SortBrandVectorComponent {
continue;
}
List<Double> sortBrandVactor = sortBrandVectorCacheBean.queryBrandSortVector(sortBrand);
if(sortBrandVactor==null || sortBrandVactor.isEmpty()){
continue;
}
double score = this.calScore(sortBrandVactor,userVectorList);
sortBrandVectorScores.add(new SortBrandVectorScore(sortBrand,score));
}
... ...
... ... @@ -13,15 +13,15 @@ public enum StrategyEnum {
REC_S_B_PROMOTION(43),//实时的品牌+品类的新开促销
REC_S_B_NEW(44),//实时的品牌+品类的新品
PRED_S_B_HEAT_VALUE(31),//预测的品牌+品类的人气值
PRED_S_B_REDUCE_PRICE(32),//预测的品牌+品类的最新降价
PRED_S_B_PROMOTION(33),//预测的品牌+品类的新开促销
PRED_S_B_NEW(34),//预测的品牌+品类的新品
VEC_S_B_HEAT_VALUE(31),//向量预测的品牌+品类的人气值
VEC_S_B_REDUCE_PRICE(32),//向量预测的品牌+品类的最新降价
VEC_S_B_PROMOTION(33),//向量预测的品牌+品类的新开促销
VEC_S_B_NEW(34),//向量预测的品牌+品类的新品
VEC_S_B_HEAT_VALUE(21),//向量预测的品牌+品类的人气值
VEC_S_B_REDUCE_PRICE(22),//向量预测的品牌+品类的最新降价
VEC_S_B_PROMOTION(23),//向量预测的品牌+品类的新开促销
VEC_S_B_NEW(24),//向量预测的品牌+品类的新品
PRED_S_B_HEAT_VALUE(21),//预测的品牌+品类的人气值
PRED_S_B_REDUCE_PRICE(22),//预测的品牌+品类的最新降价
PRED_S_B_PROMOTION(23),//预测的品牌+品类的新开促销
PRED_S_B_NEW(24),//预测的品牌+品类的新品
ADD_FLOW(12),//流量补偿
NEW_SHOP(11),//新开店铺
... ... @@ -39,11 +39,4 @@ public enum StrategyEnum {
return priority;
}
public static Integer getPriority(String name){
StrategyEnum strategyEnum = StrategyEnum.valueOf(name);
if(strategyEnum==null){
return 0;
}
return strategyEnum.priority;
}
}
... ...
... ... @@ -2,10 +2,10 @@ package com.yoho.search.recall.scene.constants;
public class RecallCommonConstants {
public static final int maxRealTimeSortBrandCount = 20;//截取【品牌-品类】的数量-for排序
public static final int maxRealTimeSortBrandCount = 10;//截取【品牌-品类】的数量-for召回
public static final int maxForecastSortBrandCount = 20;//截取【品牌-品类】的数量-for排序
public static final int maxVectorSortBrandCount = 10;//截取【品牌-品类】的数量-for召回
public static final int maxJoinSortBrandCount = 20;//截取【品牌-品类】的数量-for排序
public static final int maxForecastSortBrandCount = 10;//截取【品牌-品类】的数量-for召回
}
... ...
... ... @@ -10,10 +10,9 @@ public class RecallRequestResponse extends AbstractCacheRequestResponse<RecallRe
super(request);
}
public RecallRequestResponse(RecallRequest request,RecallResponse recallResponse) {
super(request);
super.setResponse(recallResponse,false);
}
public static Transfer<String,RecallResponse> toResponseTransfer = (jsonValue)-> JSON.parseObject(jsonValue, RecallResponse.class);
public static Transfer<RecallResponse,String> fromResponseTransfer =(recallResponse)-> JSON.toJSONString(recallResponse);
@Override
public Transfer<String, RecallResponse> getToResponseTransfer() {
... ... @@ -25,17 +24,4 @@ public class RecallRequestResponse extends AbstractCacheRequestResponse<RecallRe
return fromResponseTransfer;
}
public static Transfer<String,RecallResponse> toResponseTransfer = new Transfer<String, RecallResponse>() {
@Override
public RecallResponse transfer(String jsonValue) {
return JSON.parseObject(jsonValue, RecallResponse.class);
}
};
public static Transfer<RecallResponse,String> fromResponseTransfer = new Transfer<RecallResponse, String>() {
@Override
public String transfer(RecallResponse recallResponse) {
return JSON.toJSONString(recallResponse);
}
};
}
... ...
... ... @@ -167,21 +167,21 @@ public class SearchDynamicConfigService {
}
/**
* 新的的召回策略-预测的品牌品类
* 新的的召回策略-基于向量的品牌品类
*
* @return
*/
public int maxForecastSortBrandCount() {
return configReader.getInt("search.persional.newstrategy.forecast.max_sort_brand.count", RecallCommonConstants.maxForecastSortBrandCount);
public int maxVectorSortBrandCount() {
return configReader.getInt("search.persional.newstrategy.realtime.max_sort_brand.count", RecallCommonConstants.maxVectorSortBrandCount);
}
/**
* 新的的召回策略-最多预测的品牌品类
* 新的的召回策略-预测的品牌品类
*
* @return
*/
public int maxJoinSortBrandCount() {
return configReader.getInt("search.persional.newstrategy.max_join_sort_brand.count", RecallCommonConstants.maxJoinSortBrandCount);
public int maxForecastSortBrandCount() {
return configReader.getInt("search.persional.newstrategy.forecast.max_sort_brand.count", RecallCommonConstants.maxForecastSortBrandCount);
}
/**
... ...