|
|
|
package com.yoho.search.recall.beans.cache;
|
|
|
|
|
|
|
|
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
|
|
|
|
import com.yoho.search.base.beans.ZkConfigManager;
|
|
|
|
import com.yoho.search.base.constants.ZkConfigConstants;
|
|
|
|
import com.yoho.search.base.utils.CollectionUtils;
|
|
|
|
import com.yoho.search.base.utils.ISearchConstants;
|
|
|
|
import com.yoho.search.base.utils.ProductIndexEsField;
|
|
|
|
import com.yoho.search.base.utils.Transfer;
|
|
|
|
import com.yoho.search.common.cache.impls.CacheInterface;
|
|
|
|
import com.yoho.search.common.cache.impls.EhCache;
|
|
|
|
import com.yoho.search.common.cache.impls.SearchRedis;
|
|
|
|
import com.yoho.search.core.es.model.SearchParam;
|
|
|
|
import com.yoho.search.core.es.model.SearchResult;
|
|
|
|
import com.yoho.search.recall.beans.strategy.StrategyEnum;
|
|
|
|
import com.yoho.search.recall.beans.persional.PageProductIdBitSetComponent;
|
|
|
|
import com.yoho.search.recall.beans.persional.PageSknCodeBitSetComponent;
|
|
|
|
import com.yoho.search.recall.beans.strategy.impls.IRecallSknStrategy;
|
|
|
|
import com.yoho.search.recall.beans.strategy.impls.RealTimeSimilarSknStrategy;
|
|
|
|
import com.yoho.search.recall.beans.strategy.impls.RecommendSknStrategy;
|
|
|
|
import com.yoho.search.recall.config.RecallConfigService;
|
|
|
|
import com.yoho.search.recall.models.common.ParamQueryFilter;
|
|
|
|
import com.yoho.search.recall.models.personal.PageProductIdBitSet;
|
|
|
|
import com.yoho.search.recall.models.personal.PageSknCodeBitSet;
|
|
|
|
import com.yoho.search.recall.models.personal.UserPersonalFactor;
|
|
|
|
import com.yoho.search.recall.models.req.RecallRequest;
|
|
|
|
import com.yoho.search.recall.models.req.RecallRequestResponse;
|
|
...
|
...
|
@@ -22,6 +27,7 @@ import com.yoho.search.recall.models.req.RecallResponse; |
|
|
|
import com.yoho.search.recall.models.req.UserRecallRequest;
|
|
|
|
import com.yoho.search.service.base.SearchCommonService;
|
|
|
|
import org.apache.commons.collections.MapUtils;
|
|
|
|
import org.apache.commons.lang.StringUtils;
|
|
|
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
|
|
|
import org.elasticsearch.index.query.QueryBuilders;
|
|
|
|
import org.slf4j.Logger;
|
|
...
|
...
|
@@ -37,19 +43,21 @@ public class SknRecallCacheBean { |
|
|
|
private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private SearchCommonService searchCommonService;
|
|
|
|
private SknCodeCacheBean sknCodeCacheBean;
|
|
|
|
@Autowired
|
|
|
|
private ZkConfigManager zkConfigManager;
|
|
|
|
@Autowired
|
|
|
|
private PageProductIdBitSetComponent pageProductIdBitSetComponent;
|
|
|
|
private SearchCommonService searchCommonService;
|
|
|
|
@Autowired
|
|
|
|
private SknBaseInfoCacheBean sknBaseInfoCacheBean;
|
|
|
|
private PageSknCodeBitSetComponent pageSknCodeBitSetComponent;
|
|
|
|
@Autowired
|
|
|
|
private SearchRedis searchRedis;
|
|
|
|
@Autowired
|
|
|
|
private EhCache ehCache;
|
|
|
|
@Autowired
|
|
|
|
private RecallConfigService recallConfigService;
|
|
|
|
// @Autowired
|
|
|
|
// private EhCache ehCache;
|
|
|
|
|
|
|
|
private static final Boolean recallWithCache = false;
|
|
|
|
private static final Boolean recallWithCache = true;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 将推荐的skn构造成召回对象
|
|
...
|
...
|
@@ -69,6 +77,9 @@ public class SknRecallCacheBean { |
|
|
|
if (rtSimSknCount > 0 && userPersonalFactor.getRealTimeSimilarSknList() != null) {
|
|
|
|
filterSknList.addAll(userPersonalFactor.getRealTimeSimilarSknList());
|
|
|
|
}
|
|
|
|
if(filterSknList.isEmpty()){
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
//3、执行查询
|
|
|
|
List<Integer> filterSknResults;
|
|
|
|
if (recallWithCache) {
|
|
...
|
...
|
@@ -89,7 +100,7 @@ public class SknRecallCacheBean { |
|
|
|
}
|
|
|
|
|
|
|
|
private List<RecallRequestResponse> buildResults(ParamQueryFilter paramQueryFilter, List<Integer> filterSkns, Class<? extends IRecallSknStrategy> clazz, List<Integer> sknResults, int maxReturnCount) {
|
|
|
|
if (filterSkns == null || filterSkns.isEmpty() || sknResults == null || sknResults.isEmpty() || maxReturnCount<=0){
|
|
|
|
if (filterSkns == null || filterSkns.isEmpty() || sknResults == null || sknResults.isEmpty() || maxReturnCount <= 0) {
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
List<RecallRequestResponse> results = new ArrayList<>();
|
|
...
|
...
|
@@ -124,74 +135,63 @@ public class SknRecallCacheBean { |
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
private List<Integer> filterRecommendWithCache(UserRecallRequest userRecallRequest, List<Integer> recommedSknList) {
|
|
|
|
//1、skn转成productId
|
|
|
|
Map<Integer, Integer> productSknToIdMap = sknBaseInfoCacheBean.queryProductSknToProductIdMap(recommedSknList);
|
|
|
|
if (productSknToIdMap == null || productSknToIdMap.isEmpty()) {
|
|
|
|
//1、获取code版本
|
|
|
|
String codeVersion = zkConfigManager.getStringValueFromZk(ZkConfigConstants.productSknCodeVersion);
|
|
|
|
if(StringUtils.isBlank(codeVersion)){
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
//2、查询sknCodeMap
|
|
|
|
Map<Integer, Integer> sknCodeMap = sknCodeCacheBean.queryProductSknCodeMap(codeVersion,recommedSknList);
|
|
|
|
if (sknCodeMap == null || sknCodeMap.isEmpty()) {
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
List<Integer> recommedProductIds = new ArrayList<>();
|
|
|
|
recommedProductIds.addAll(productSknToIdMap.values());
|
|
|
|
|
|
|
|
//2、先从本地缓存获取
|
|
|
|
ParamQueryFilter paramQueryFilter = userRecallRequest.getParamQueryFilter();
|
|
|
|
RedisKeyBuilder redisKeyBuilder = pageProductIdBitSetComponent.genRedisKeyBuilder(paramQueryFilter);
|
|
|
|
Map<Integer, Boolean> ehcacheQueryResult = ehCache.getFromBitSet(redisKeyBuilder, recommedProductIds);
|
|
|
|
if (ehcacheQueryResult != null) {
|
|
|
|
RECALL_NEW_LOGGER.info("filterSknBy EhCache success");
|
|
|
|
return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap, ehcacheQueryResult);
|
|
|
|
RedisKeyBuilder redisKeyBuilder = pageSknCodeBitSetComponent.genRedisKeyBuilder(paramQueryFilter,codeVersion);
|
|
|
|
|
|
|
|
// //2、从本地缓存获取结果
|
|
|
|
// List<Integer> ehCacheResults = this.filterFromCache(ehCache, redisKeyBuilder, recommedSknList, sknCodeMap);
|
|
|
|
// if (ehCacheResults != null && !ehCacheResults.isEmpty()) {
|
|
|
|
// return ehCacheResults;
|
|
|
|
// }
|
|
|
|
|
|
|
|
//3、从redis获取结果
|
|
|
|
List<Integer> redisResults = this.filterFromCache(searchRedis, redisKeyBuilder, recommedSknList, sknCodeMap);
|
|
|
|
if (redisResults != null && !redisResults.isEmpty()) {
|
|
|
|
return redisResults;
|
|
|
|
}
|
|
|
|
//3、再从redis的bitset缓存结构中过滤
|
|
|
|
Map<Integer, Boolean> redisQueryResult = searchRedis.getFromBitSet(redisKeyBuilder, recommedProductIds);
|
|
|
|
if (redisQueryResult != null) {
|
|
|
|
RECALL_NEW_LOGGER.info("filterSknBy Redis success");
|
|
|
|
return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap, redisQueryResult);
|
|
|
|
|
|
|
|
//4、执行查询
|
|
|
|
PageSknCodeBitSet pageSknCodeBitSet = pageSknCodeBitSetComponent.queryPageSknCodeBitSet(userRecallRequest.getParamQueryFilter());
|
|
|
|
if (pageSknCodeBitSet == null) {
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
//4、执行查询，添加到缓存
|
|
|
|
PageProductIdBitSet pageProductIdBitSet = pageProductIdBitSetComponent.queryPageProductIdBitSet(userRecallRequest.getParamQueryFilter());
|
|
|
|
//4.1)添加到本地缓存
|
|
|
|
this.ehCache.setBitSet(redisKeyBuilder, pageProductIdBitSet.getProductIdBitSet(), pageProductIdBitSetComponent.cacheTimeInMinute());
|
|
|
|
//4.2)异步添加到redis缓存
|
|
|
|
this.searchRedis.setBitSet(redisKeyBuilder, pageProductIdBitSet.getProductIdBitSet(), pageProductIdBitSetComponent.cacheTimeInMinute());
|
|
|
|
return this.filterSknByPageProductIdBitSet(recommedSknList, productSknToIdMap, pageProductIdBitSet.getProductIdBitSet());
|
|
|
|
|
|
|
|
//5、添加到缓存
|
|
|
|
//this.ehCache.setBitSet(redisKeyBuilder, pageSknCodeBitSet.getBitSet(), pageSknCodeBitSetComponent.cacheTimeInMinute());
|
|
|
|
this.searchRedis.setBitSet(redisKeyBuilder, pageSknCodeBitSet.getBitSet(), pageSknCodeBitSetComponent.cacheTimeInMinute());
|
|
|
|
|
|
|
|
//6、构造返回结果
|
|
|
|
return this.filterResults(recommedSknList, sknCodeMap, (sknCode) -> pageSknCodeBitSet.getBitSet().get(sknCode));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 从bitset对象中构造返回对象
|
|
|
|
*
|
|
|
|
* @param recommedSknList
|
|
|
|
* @param bitSet
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
private List<Integer> filterSknByPageProductIdBitSet(List<Integer> recommedSknList, Map<Integer, Integer> productSknToIdMap, BitSet bitSet) {
|
|
|
|
List<Integer> existProductSkns = new ArrayList<>();
|
|
|
|
for (Integer recommedSkn : recommedSknList) {
|
|
|
|
Integer productId = productSknToIdMap.get(recommedSkn);
|
|
|
|
if (productId == null) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!bitSet.get(productId)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
existProductSkns.add(recommedSkn);
|
|
|
|
private List<Integer> filterFromCache(CacheInterface cacheInterface, RedisKeyBuilder redisKeyBuilder, List<Integer> recommedSknList, Map<Integer, Integer> sknCodeMap) {
|
|
|
|
Map<Integer, Boolean> cacheResult = cacheInterface.getFromBitSet(redisKeyBuilder, sknCodeMap.values());
|
|
|
|
if (cacheResult == null || cacheResult.isEmpty()) {
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
return existProductSkns;
|
|
|
|
RECALL_NEW_LOGGER.info("filterSknBy {} success", cacheInterface.getClass().getSimpleName());
|
|
|
|
return this.filterResults(recommedSknList, sknCodeMap, (sknCode) -> MapUtils.getBooleanValue(cacheResult, sknCode, false));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 根据redis返回结果按顺序构结果，只返回为true的
|
|
|
|
*
|
|
|
|
* @param recommedSknList
|
|
|
|
* @param cacheQueryResult
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
private List<Integer> filterSknByCacheQueryResult(List<Integer> recommedSknList, Map<Integer, Integer> productSknToIdMap, Map<Integer, Boolean> cacheQueryResult) {
|
|
|
|
private List<Integer> filterResults(List<Integer> recommedSknList, Map<Integer, Integer> sknCodeMap, Transfer<Integer, Boolean> transfer) {
|
|
|
|
List<Integer> existProductSkns = new ArrayList<>();
|
|
|
|
for (Integer recommedSkn : recommedSknList) {
|
|
|
|
Integer productId = productSknToIdMap.get(recommedSkn);
|
|
|
|
if (productId == null) {
|
|
|
|
Integer sknCode = sknCodeMap.get(recommedSkn);
|
|
|
|
if (sknCode == null) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!cacheQueryResult.getOrDefault(productId, false)) {
|
|
|
|
if (!transfer.transfer(sknCode)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
existProductSkns.add(recommedSkn);
|
...
|
...
|
|
