Authored by hugufei

Merge branch 'master' into zj_skn分词

Showing 17 changed files with 330 additions and 188 deletions
... ... @@ -3,10 +3,7 @@ package com.yoho.search.common.cache.impls;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.common.cache.model.CacheObject;
import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.*;
public interface CacheInterface {
... ... @@ -22,6 +19,6 @@ public interface CacheInterface {
public boolean setBitSet(RedisKeyBuilder redisKeyBuilder, BitSet bitSet, int expiredTimeInMinute);
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets);
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, Collection<Integer> offsets);
}
... ...
... ... @@ -124,7 +124,7 @@ public class EhCache implements CacheInterface {
}
@Override
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets){
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, Collection<Integer> offsets){
try {
Element e = this.cache.get(key.getKey());
if(e==null){
... ...
... ... @@ -80,7 +80,7 @@ public class SearchRedis implements CacheInterface {
}
@Override
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets) {
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, Collection<Integer> offsets) {
try {
YHRedisTemplate redisTemplate = yohoSearchRedisComponent.getRedisTemplate();
if (!redisTemplate.hasKey(key)) {
... ...
... ... @@ -63,7 +63,7 @@ public class RedisCacheUtils {
* @param offsets
* @return
*/
public static Map<Integer, Boolean> getBits(YHRedisTemplate redisTemplate, RedisKeyBuilder redisKeyBuilder, List<Integer> offsets) {
public static Map<Integer, Boolean> getBits(YHRedisTemplate redisTemplate, RedisKeyBuilder redisKeyBuilder, Collection<Integer> offsets) {
List<Long> LongOffsets = CollectionUtils.toList(offsets,(a)->a.longValue());
Map<Long, Boolean> redisResults = redisTemplate.getBits(redisKeyBuilder,LongOffsets);
if(redisResults==null || redisResults.isEmpty()){
... ...
... ... @@ -20,32 +20,6 @@ public class SknBaseInfoCacheBean extends AbstractCacheBean<SknBaseInfoRequest,
private SearchCommonService searchCommonService;
/**
* 查询skn对应的productId
*
* @param productSknList
* @return
*/
public Map<Integer, Integer> queryProductSknToProductIdMap(List<Integer> productSknList) {
//1、参数判断
if (productSknList == null || productSknList.isEmpty()) {
return new HashMap<>();
}
//2、执行查询
List<SknBaseInfoRequestResponse> baseInfoList = this.batchQuery(productSknList);
//3、构造结果
Map<Integer, Integer> results = new HashMap<>();
for (SknBaseInfoRequestResponse requestResponse : baseInfoList) {
SknBaseInfoResponse response = requestResponse.getResponse();
if (response == null) {
continue;
}
results.put(response.getProductSkn(), response.getProductId());
}
return results;
}
/**
* 查询skn对应的基本信息
*
* @param productSknList
... ... @@ -71,7 +45,6 @@ public class SknBaseInfoCacheBean extends AbstractCacheBean<SknBaseInfoRequest,
return results;
}
private List<SknBaseInfoRequestResponse> batchQuery(List<Integer> productSknList) {
//1、参数判断
if (productSknList == null || productSknList.isEmpty()) {
... ... @@ -111,7 +84,7 @@ public class SknBaseInfoCacheBean extends AbstractCacheBean<SknBaseInfoRequest,
searchParam.setOffset(0);
searchParam.setSize(productSkns.size());
searchParam.setFiter(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, productSkns));
searchParam.setIncludeFields(missCacheRequests.get(0).getRequest().includeFields());
searchParam.setIncludeFields(SknBaseInfoRequest.includeFields);
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
//4、构建基于ProductId的临时结果
... ...
package com.yoho.search.recall.beans.cache;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.recall.models.req.SknCodeRequest;
import com.yoho.search.recall.models.req.SknCodeRequestResponse;
import com.yoho.search.service.base.SearchCommonService;
import org.apache.commons.collections.MapUtils;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class SknCodeCacheBean extends AbstractCacheBean<SknCodeRequest, Integer, SknCodeRequestResponse> {
@Autowired
private SearchCommonService searchCommonService;
@Override
protected boolean useEhCache() {
return false;
}
/**
* 查询skn对应的sknCode
*
* @param codeVersion
* @param productSknList
* @return
*/
public Map<Integer, Integer> queryProductSknCodeMap(String codeVersion, List<Integer> productSknList) {
//1、参数判断
if (productSknList == null || productSknList.isEmpty()) {
return new HashMap<>();
}
//2、执行查询
List<SknCodeRequestResponse> responseList = this.batchQuery(codeVersion,productSknList);
//3、构造结果
Map<Integer, Integer> results = new HashMap<>();
for (SknCodeRequestResponse requestResponse : responseList) {
Integer response = requestResponse.getResponse();
if (response == null || response.equals(0)){
continue;
}
results.put(requestResponse.getRequest().getProductSkn(), response);
}
return results;
}
private List<SknCodeRequestResponse> batchQuery(String codeVersion,List<Integer> productSknList) {
//1、参数判断
if (productSknList == null || productSknList.isEmpty()) {
return new ArrayList<>();
}
//2、构造请求
final List<SknCodeRequestResponse> requestResponses = new ArrayList<>();
for (Integer productSkn : productSknList) {
requestResponses.add(new SknCodeRequestResponse(new SknCodeRequest(productSkn, codeVersion)));
}
//2、调父类方法
super.bacthFillResponseWithCache(requestResponses, productSknList.size());
return requestResponses;
}
@Override
protected Map<SknCodeRequest, Integer> queryMissCacheRequestResults(List<SknCodeRequestResponse> missCacheRequests) {
//1、合法性判断
Map<SknCodeRequest, Integer> results = new HashMap<>();
if (missCacheRequests == null || missCacheRequests.isEmpty()) {
return results;
}
//2、获取productSkn
List<Integer> productSkns = new ArrayList<>();
for (SknCodeRequestResponse sknCodeRequestResponse : missCacheRequests) {
productSkns.add(sknCodeRequestResponse.getRequest().getProductSkn());
}
//3、查询sknCodeMap
Map<Integer, Integer> sknCodeMap = this.querySknCodeMapFromEs(productSkns);
//4、构建结果
for (SknCodeRequestResponse sknCodeRequestResponse : missCacheRequests) {
Integer productSkn = sknCodeRequestResponse.getRequest().getProductSkn();
Integer productSknCode = MapUtils.getIntValue(sknCodeMap, productSkn, 0);
results.put(sknCodeRequestResponse.getRequest(), productSknCode);
}
return results;
}
private Map<Integer, Integer> querySknCodeMapFromEs(List<Integer> productSkns) {
Map<Integer, Integer> sknCodeMap = new HashMap<>();
//1、构建SearchParam
SearchParam searchParam = new SearchParam();
searchParam.setSize(productSkns.size());
searchParam.setFiter(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, productSkns));
searchParam.setIncludeFields(SknCodeRequest.includeFields);
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult == null || searchResult.getResultList() == null) {
return sknCodeMap;
}
for (Map<String, Object> product : searchResult.getResultList()) {
sknCodeMap.put(MapUtils.getIntValue(product, ProductIndexEsField.productSkn, 0), MapUtils.getIntValue(product, ProductIndexEsField.productSknCode, 0));
}
return sknCodeMap;
}
}
... ...
package com.yoho.search.recall.beans.cache;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.beans.ZkConfigManager;
import com.yoho.search.base.constants.ZkConfigConstants;
import com.yoho.search.base.utils.CollectionUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.base.utils.Transfer;
import com.yoho.search.common.cache.impls.CacheInterface;
import com.yoho.search.common.cache.impls.EhCache;
import com.yoho.search.common.cache.impls.SearchRedis;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.recall.beans.strategy.StrategyEnum;
import com.yoho.search.recall.beans.persional.PageProductIdBitSetComponent;
import com.yoho.search.recall.beans.persional.PageSknCodeBitSetComponent;
import com.yoho.search.recall.beans.strategy.impls.IRecallSknStrategy;
import com.yoho.search.recall.beans.strategy.impls.RealTimeSimilarSknStrategy;
import com.yoho.search.recall.beans.strategy.impls.RecommendSknStrategy;
import com.yoho.search.recall.config.RecallConfigService;
import com.yoho.search.recall.models.common.ParamQueryFilter;
import com.yoho.search.recall.models.personal.PageProductIdBitSet;
import com.yoho.search.recall.models.personal.PageSknCodeBitSet;
import com.yoho.search.recall.models.personal.UserPersonalFactor;
import com.yoho.search.recall.models.req.RecallRequest;
import com.yoho.search.recall.models.req.RecallRequestResponse;
... ... @@ -22,6 +27,7 @@ import com.yoho.search.recall.models.req.RecallResponse;
import com.yoho.search.recall.models.req.UserRecallRequest;
import com.yoho.search.service.base.SearchCommonService;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
... ... @@ -37,19 +43,21 @@ public class SknRecallCacheBean {
private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
@Autowired
private SearchCommonService searchCommonService;
private SknCodeCacheBean sknCodeCacheBean;
@Autowired
private ZkConfigManager zkConfigManager;
@Autowired
private PageProductIdBitSetComponent pageProductIdBitSetComponent;
private SearchCommonService searchCommonService;
@Autowired
private SknBaseInfoCacheBean sknBaseInfoCacheBean;
private PageSknCodeBitSetComponent pageSknCodeBitSetComponent;
@Autowired
private SearchRedis searchRedis;
@Autowired
private EhCache ehCache;
@Autowired
private RecallConfigService recallConfigService;
// @Autowired
// private EhCache ehCache;
private static final Boolean recallWithCache = false;
private static final Boolean recallWithCache = true;
/**
* 将推荐的skn构造成召回对象
... ... @@ -69,6 +77,9 @@ public class SknRecallCacheBean {
if (rtSimSknCount > 0 && userPersonalFactor.getRealTimeSimilarSknList() != null) {
filterSknList.addAll(userPersonalFactor.getRealTimeSimilarSknList());
}
if(filterSknList.isEmpty()){
return new ArrayList<>();
}
//3、执行查询
List<Integer> filterSknResults;
if (recallWithCache) {
... ... @@ -89,7 +100,7 @@ public class SknRecallCacheBean {
}
private List<RecallRequestResponse> buildResults(ParamQueryFilter paramQueryFilter, List<Integer> filterSkns, Class<? extends IRecallSknStrategy> clazz, List<Integer> sknResults, int maxReturnCount) {
if (filterSkns == null || filterSkns.isEmpty() || sknResults == null || sknResults.isEmpty() || maxReturnCount<=0){
if (filterSkns == null || filterSkns.isEmpty() || sknResults == null || sknResults.isEmpty() || maxReturnCount <= 0) {
return new ArrayList<>();
}
List<RecallRequestResponse> results = new ArrayList<>();
... ... @@ -124,74 +135,63 @@ public class SknRecallCacheBean {
* @return
*/
private List<Integer> filterRecommendWithCache(UserRecallRequest userRecallRequest, List<Integer> recommedSknList) {
//1、skn转成productId
Map<Integer, Integer> productSknToIdMap = sknBaseInfoCacheBean.queryProductSknToProductIdMap(recommedSknList);
if (productSknToIdMap == null || productSknToIdMap.isEmpty()) {
//1、获取code版本
String codeVersion = zkConfigManager.getStringValueFromZk(ZkConfigConstants.productSknCodeVersion);
if(StringUtils.isBlank(codeVersion)){
return new ArrayList<>();
}
//2、查询sknCodeMap
Map<Integer, Integer> sknCodeMap = sknCodeCacheBean.queryProductSknCodeMap(codeVersion,recommedSknList);
if (sknCodeMap == null || sknCodeMap.isEmpty()) {
return new ArrayList<>();
}
List<Integer> recommedProductIds = new ArrayList<>();
recommedProductIds.addAll(productSknToIdMap.values());
//2、先从本地缓存获取
ParamQueryFilter paramQueryFilter = userRecallRequest.getParamQueryFilter();
RedisKeyBuilder redisKeyBuilder = pageProductIdBitSetComponent.genRedisKeyBuilder(paramQueryFilter);
Map<Integer, Boolean> ehcacheQueryResult = ehCache.getFromBitSet(redisKeyBuilder, recommedProductIds);
if (ehcacheQueryResult != null) {
RECALL_NEW_LOGGER.info("filterSknBy EhCache success");
return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap, ehcacheQueryResult);
RedisKeyBuilder redisKeyBuilder = pageSknCodeBitSetComponent.genRedisKeyBuilder(paramQueryFilter,codeVersion);
// //2、从本地缓存获取结果
// List<Integer> ehCacheResults = this.filterFromCache(ehCache, redisKeyBuilder, recommedSknList, sknCodeMap);
// if (ehCacheResults != null && !ehCacheResults.isEmpty()) {
// return ehCacheResults;
// }
//3、从redis获取结果
List<Integer> redisResults = this.filterFromCache(searchRedis, redisKeyBuilder, recommedSknList, sknCodeMap);
if (redisResults != null && !redisResults.isEmpty()) {
return redisResults;
}
//3、再从redis的bitset缓存结构中过滤
Map<Integer, Boolean> redisQueryResult = searchRedis.getFromBitSet(redisKeyBuilder, recommedProductIds);
if (redisQueryResult != null) {
RECALL_NEW_LOGGER.info("filterSknBy Redis success");
return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap, redisQueryResult);
//4、执行查询
PageSknCodeBitSet pageSknCodeBitSet = pageSknCodeBitSetComponent.queryPageSknCodeBitSet(userRecallRequest.getParamQueryFilter());
if (pageSknCodeBitSet == null) {
return new ArrayList<>();
}
//4、执行查询,添加到缓存
PageProductIdBitSet pageProductIdBitSet = pageProductIdBitSetComponent.queryPageProductIdBitSet(userRecallRequest.getParamQueryFilter());
//4.1)添加到本地缓存
this.ehCache.setBitSet(redisKeyBuilder, pageProductIdBitSet.getProductIdBitSet(), pageProductIdBitSetComponent.cacheTimeInMinute());
//4.2)异步添加到redis缓存
this.searchRedis.setBitSet(redisKeyBuilder, pageProductIdBitSet.getProductIdBitSet(), pageProductIdBitSetComponent.cacheTimeInMinute());
return this.filterSknByPageProductIdBitSet(recommedSknList, productSknToIdMap, pageProductIdBitSet.getProductIdBitSet());
//5、添加到缓存
//this.ehCache.setBitSet(redisKeyBuilder, pageSknCodeBitSet.getBitSet(), pageSknCodeBitSetComponent.cacheTimeInMinute());
this.searchRedis.setBitSet(redisKeyBuilder, pageSknCodeBitSet.getBitSet(), pageSknCodeBitSetComponent.cacheTimeInMinute());
//6、构造返回结果
return this.filterResults(recommedSknList, sknCodeMap, (sknCode) -> pageSknCodeBitSet.getBitSet().get(sknCode));
}
/**
* 从bitset对象中构造返回对象
*
* @param recommedSknList
* @param bitSet
* @return
*/
private List<Integer> filterSknByPageProductIdBitSet(List<Integer> recommedSknList, Map<Integer, Integer> productSknToIdMap, BitSet bitSet) {
List<Integer> existProductSkns = new ArrayList<>();
for (Integer recommedSkn : recommedSknList) {
Integer productId = productSknToIdMap.get(recommedSkn);
if (productId == null) {
continue;
}
if (!bitSet.get(productId)) {
continue;
}
existProductSkns.add(recommedSkn);
private List<Integer> filterFromCache(CacheInterface cacheInterface, RedisKeyBuilder redisKeyBuilder, List<Integer> recommedSknList, Map<Integer, Integer> sknCodeMap) {
Map<Integer, Boolean> cacheResult = cacheInterface.getFromBitSet(redisKeyBuilder, sknCodeMap.values());
if (cacheResult == null || cacheResult.isEmpty()) {
return new ArrayList<>();
}
return existProductSkns;
RECALL_NEW_LOGGER.info("filterSknBy {} success", cacheInterface.getClass().getSimpleName());
return this.filterResults(recommedSknList, sknCodeMap, (sknCode) -> MapUtils.getBooleanValue(cacheResult, sknCode, false));
}
/**
* 根据redis返回结果按顺序构结果,只返回为true的
*
* @param recommedSknList
* @param cacheQueryResult
* @return
*/
private List<Integer> filterSknByCacheQueryResult(List<Integer> recommedSknList, Map<Integer, Integer> productSknToIdMap, Map<Integer, Boolean> cacheQueryResult) {
private List<Integer> filterResults(List<Integer> recommedSknList, Map<Integer, Integer> sknCodeMap, Transfer<Integer, Boolean> transfer) {
List<Integer> existProductSkns = new ArrayList<>();
for (Integer recommedSkn : recommedSknList) {
Integer productId = productSknToIdMap.get(recommedSkn);
if (productId == null) {
Integer sknCode = sknCodeMap.get(recommedSkn);
if (sknCode == null) {
continue;
}
if (!cacheQueryResult.getOrDefault(productId, false)) {
if (!transfer.transfer(sknCode)) {
continue;
}
existProductSkns.add(recommedSkn);
... ...
package com.yoho.search.recall.beans.cache;
import com.alibaba.fastjson.JSON;
import com.yoho.search.recall.beans.builder.*;
import com.yoho.search.recall.beans.builder.CommonRecallRequestBuilder;
import com.yoho.search.recall.beans.builder.RecallMergerResultBuilder;
import com.yoho.search.recall.beans.builder.SortBrandRecallRequestBuilder;
import com.yoho.search.recall.beans.builder.UserRecallResponseBuilder;
import com.yoho.search.recall.beans.persional.QueryUserPersionalFactorBean;
import com.yoho.search.recall.beans.strategy.SortBrandType;
import com.yoho.search.recall.models.common.RecallMergerResult;
... ... @@ -12,7 +14,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
... ... @@ -126,16 +131,6 @@ public class UserRecallCacheBean extends AbstractCacheBean<UserRecallRequest, Us
batchRequestResults.addAll(this.getResultFromCompletableFuture(vectorRnnSortBrandCompletableFuture));
batchRequestResults.addAll(this.getResultFromCompletableFuture(configSortBrandCompletableFuture));
//9、日志打印
if (userRecallRequest.openDetailLog()) {
// for (RecallRequestResponse recallRequestResponse : batchRequestResults) {
// RecallRequest recallRequest = recallRequestResponse.getRequest();
// RecallResponse recallRespone = recallRequestResponse.getResponse();
// StringBuilder responseValue = new StringBuilder();
// responseValue.append(recallRespone == null ? "null" : JSON.toJSONString(recallRespone));
// RECALL_NEW_LOGGER.info("request_type is [{}], response is [{}] ", recallRequest.requestStrategy().name(), responseValue.toString());
// }
}
return batchRequestResults;
}
... ...
... ... @@ -18,24 +18,11 @@ import java.util.concurrent.Executors;
public abstract class AbstractPageComponent<T> {
private static final Logger logger = LoggerFactory.getLogger(AbstractPageComponent.class);
@Autowired
private EhCache ehCache;
@Autowired
private SearchRedis searchRedis;
private ConcurrentHashMap<String, Integer> mapLock;
private ExecutorService executorService;
private boolean synchronous = true;//是否同步
@PostConstruct
void init() {
mapLock = new ConcurrentHashMap<String, Integer>(20);//使用一个Map来限流
executorService = Executors.newFixedThreadPool(5);
}
public T queryWithCache(ParamQueryFilter paramQueryFilter, Class<T> clazz) {
//1、生成RedisKeyBuilder
RedisKeyBuilder redisKeyBuilder = this.genRedisKeyBuilder(paramQueryFilter);
... ... @@ -53,28 +40,7 @@ public abstract class AbstractPageComponent<T> {
this.addJavaObjectToEhcache(redisKeyBuilder, result);
return result;
}
//4、如果是同步查询,则直接调子类方法查询
if (synchronous) {
return this.doInnerQuery(redisKeyBuilder, paramQueryFilter);
}
//5、执行异步流程
if (mapLock.size() > 20 || mapLock.putIfAbsent(redisKeyBuilder.getKey(), 1) != null) {
return null;
}
//6、异步执行查询并加入缓存
executorService.submit(() -> {
try {
this.doInnerQuery(redisKeyBuilder, paramQueryFilter);
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
mapLock.remove(redisKeyBuilder.getKey());
}
});
return null;
return this.doInnerQuery(redisKeyBuilder, paramQueryFilter);
}
private T doInnerQuery(RedisKeyBuilder redisKeyBuilder, ParamQueryFilter paramQueryFilter) {
... ...
... ... @@ -7,7 +7,7 @@ import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.recall.config.CacheTimeConstants;
import com.yoho.search.recall.models.common.ParamQueryFilter;
import com.yoho.search.recall.models.personal.PageProductIdBitSet;
import com.yoho.search.recall.models.personal.PageSknCodeBitSet;
import com.yoho.search.service.base.SearchCommonService;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
... ... @@ -25,33 +25,26 @@ import java.util.List;
import java.util.Map;
@Component
public class PageProductIdBitSetComponent extends AbstractPageComponent<PageProductIdBitSet>{
public class PageSknCodeBitSetComponent {
@Autowired
private SearchCommonService searchCommonService;
public RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter,String sknCodeVersion) {
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:PAGE_SKN_CODE:").appendVar(sknCodeVersion).appendFixed(":").appendVar(paramQueryFilter.getParamMd5Key());
}
public int cacheTimeInMinute() {
return CacheTimeConstants.PAGE_SKN_CODE_BITSET;
}
/**
* 获取页面上的skn的bitset
*
* @param paramQueryFilter
* @return
*/
public PageProductIdBitSet queryPageProductIdBitSet(ParamQueryFilter paramQueryFilter) {
return this.doRealQuery(paramQueryFilter);
}
@Override
public RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:PAGE_PRODUCT_BITSET:").appendVar(paramQueryFilter.getParamMd5Key());
}
@Override
public int cacheTimeInMinute() {
return CacheTimeConstants.PAGE_SKN_BITSET;
}
@Override
protected PageProductIdBitSet doRealQuery(ParamQueryFilter paramQueryFilter) {
public PageSknCodeBitSet queryPageSknCodeBitSet(ParamQueryFilter paramQueryFilter) {
//1、构造请求参数
SearchParam searchParam = new SearchParam();
searchParam.setQuery(paramQueryFilter.getParamQuery());
... ... @@ -64,7 +57,7 @@ public class PageProductIdBitSetComponent extends AbstractPageComponent<PageProd
//2、构造聚合参数
List<AbstractAggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
aggregationBuilders.add(AggregationBuilders.terms("productIdAgg").field(ProductIndexEsField.productId).size(100000).order(Terms.Order.term(false)));
aggregationBuilders.add(AggregationBuilders.terms("sknCodeAgg").field(ProductIndexEsField.productSknCode).size(100000).order(Terms.Order.term(false)));
searchParam.setAggregationBuilders(aggregationBuilders);
//3、执行查询
... ... @@ -72,27 +65,27 @@ public class PageProductIdBitSetComponent extends AbstractPageComponent<PageProd
//4、构造结果
Map<String, Aggregation> aggregationMap = searchResult.getAggMaps();
PageProductIdBitSet pageProductIdBitSet = this.getPageSknBitSetFromAggregationMap(aggregationMap, "productIdAgg");
return pageProductIdBitSet;
PageSknCodeBitSet pageSknCodeBitSet = this.getPageSknBitSetFromAggregationMap(aggregationMap, "sknCodeAgg");
return pageSknCodeBitSet;
}
private PageProductIdBitSet getPageSknBitSetFromAggregationMap(Map<String, Aggregation> aggregationMap, String firstAggName) {
private PageSknCodeBitSet getPageSknBitSetFromAggregationMap(Map<String, Aggregation> aggregationMap, String firstAggName) {
if (!aggregationMap.containsKey(firstAggName)) {
return null;
}
List<Integer> productIdList = new ArrayList<Integer>();
List<Integer> sknCodeList = new ArrayList<Integer>();
MultiBucketsAggregation firstAggregation = (MultiBucketsAggregation) aggregationMap.get(firstAggName);
Iterator<? extends MultiBucketsAggregation.Bucket> firstAggregationIterator = firstAggregation.getBuckets().iterator();
while (firstAggregationIterator.hasNext()) {
MultiBucketsAggregation.Bucket bucket = firstAggregationIterator.next();
Integer value = Integer.valueOf(bucket.getKeyAsString());
productIdList.add(value);
sknCodeList.add(value);
}
PageProductIdBitSet pageProductIdBitSet = new PageProductIdBitSet();
for (Integer productId : productIdList) {
pageProductIdBitSet.add(productId);
PageSknCodeBitSet pageSknCodeBitSet = new PageSknCodeBitSet();
for (Integer sknCode : sknCodeList) {
pageSknCodeBitSet.add(sknCode);
}
return pageProductIdBitSet;
return pageSknCodeBitSet;
}
}
... ...
... ... @@ -74,6 +74,9 @@ public class QueryUserPersionalFactorBean {
//2、构造推荐的skn列表
List<Integer> recommendSknList = userFactor.getRecommendSknList();
List<Integer> realTimeSimilarSknList = userFactor.getRealTimeSimilarSknList();
//TODO remove
//realTimeSimilarSknList.addAll(Arrays.asList(50003513,50013179,50018239,50023799));
//3、获取页面中存在的所有的key
Set<String> pageSortBrandKeys = new HashSet<>();
for (SortBrand pageSortBrand : pageFactor.getSortBrandList()) {
... ...
... ... @@ -18,6 +18,9 @@ public class CacheTimeConstants {
//SKN基本信息的缓存
public static final int SKN_BASE_INFO = 60;
//SKN编码缓存
public static final int SKN_CODE = 60;
//SKN返回信息的缓存
public static final int SKN_RETURN_INFO = 3;
... ... @@ -25,7 +28,7 @@ public class CacheTimeConstants {
public static final int USER_RECALL_SKN_LIST = 10;
//页面skn的bitset缓存
public static final int PAGE_SKN_BITSET = 60;
public static final int PAGE_SKN_CODE_BITSET = 60;
//页面个性化因子的缓存
public static final int PAGE_PERSIONAL_FACTOR = 60;
... ...
... ... @@ -3,26 +3,26 @@ package com.yoho.search.recall.models.personal;
import java.io.Serializable;
import java.util.BitSet;
public class PageProductIdBitSet implements Serializable{
public class PageSknCodeBitSet implements Serializable{
public static final int maxValue = 2000000;
public static final int maxValue = 100000;
private static final long serialVersionUID = 7185024266096124078L;
private BitSet productIdBitSet;
private BitSet bitSet;
public PageProductIdBitSet(){
this.productIdBitSet = new BitSet(maxValue);
public PageSknCodeBitSet(){
this.bitSet = new BitSet(maxValue);
}
public void add(int bitSetIndex) {
if(bitSetIndex>maxValue){
return;
}
this.productIdBitSet.set(bitSetIndex);
this.bitSet.set(bitSetIndex);
}
public BitSet getProductIdBitSet() {
return productIdBitSet;
public BitSet getBitSet() {
return bitSet;
}
}
... ...
... ... @@ -10,7 +10,7 @@ import java.util.List;
public class SknBaseInfoRequest implements ICacheRequest{
private static final List<String> includeFields = Arrays.asList(ProductIndexEsField.productId,ProductIndexEsField.productSkn,ProductIndexEsField.brandId,ProductIndexEsField.middleSortId,ProductIndexEsField.priceArea,ProductIndexEsField.productFeatureFactor,ProductIndexEsField.heatValue);
public static final List<String> includeFields = Arrays.asList(ProductIndexEsField.productId,ProductIndexEsField.productSkn,ProductIndexEsField.brandId,ProductIndexEsField.middleSortId,ProductIndexEsField.priceArea,ProductIndexEsField.productFeatureFactor,ProductIndexEsField.heatValue);
private Integer productSkn;
... ... @@ -32,8 +32,4 @@ public class SknBaseInfoRequest implements ICacheRequest{
return productSkn;
}
public List<String> includeFields(){
return includeFields;
}
}
... ...
package com.yoho.search.recall.models.req;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.recall.config.CacheTimeConstants;
import com.yoho.search.recall.models.common.ICacheRequest;
import java.util.Arrays;
import java.util.List;
public class SknCodeRequest implements ICacheRequest {
public static final List<String> includeFields = Arrays.asList(ProductIndexEsField.productSkn,ProductIndexEsField.productSknCode);
private Integer productSkn;
private String codeVersion;
public SknCodeRequest(Integer productSkn,String codeVersion){
this.productSkn = productSkn;
this.codeVersion = codeVersion;
}
@Override
public RedisKeyBuilder redisKeyBuilder() {
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("SKN_CODE:").appendVar(codeVersion).appendFixed(":").appendVar(cacheTimeInMinute()).appendFixed(":").appendVar(productSkn);
}
@Override
public int cacheTimeInMinute() {
return CacheTimeConstants.SKN_CODE;
}
public Integer getProductSkn() {
return productSkn;
}
}
... ...
package com.yoho.search.recall.models.req;
import com.yoho.search.base.utils.Transfer;
import com.yoho.search.recall.models.common.AbstractCacheRequestResponse;
public class SknCodeRequestResponse extends AbstractCacheRequestResponse<SknCodeRequest, Integer> {
public SknCodeRequestResponse(SknCodeRequest request) {
super(request);
}
@Override
public Transfer<String, Integer> getToResponseTransfer() {
return (v) -> Integer.valueOf(v);
}
@Override
public Transfer<Integer, String> getFromResponseTransfer() {
return (v) -> String.valueOf(v);
}
}
... ...
package com.yoho.search.restapi.tools;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.common.cache.impls.SearchRedis;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;
@Controller
public class TestRedisController {
@Autowired
private SearchRedis searchRedis;
@RequestMapping(value = "/testRedis/bitset")
@ResponseBody
public Map<String, Object> testBitset() {
long begin = System.currentTimeMillis();
for (int i = 0; i < 10000; i++) {
BitSet bitSet = new BitSet(100000);
for (int j = 0; j < 10000; j++) {
bitSet.set(j);
}
bitSet.set(100000);
RedisKeyBuilder redisKeyBuilder = new RedisKeyBuilder();
redisKeyBuilder.appendFixed("RedisBitSet:").appendVar(System.currentTimeMillis());
redisKeyBuilder.appendFixed(":").appendVar((int) (Math.random() * 1000));
searchRedis.setBitSet(redisKeyBuilder, bitSet, 100);
}
Map result = new HashMap<>();
result.put("code", 200);
result.put("messgae", "success");
result.put("cost", System.currentTimeMillis() - begin);
return result;
}
}
... ...