yoho-search / yoho-search-service · Commits

Go to a project

Authored by wangnan
Commit a5a383433e0381486236c0890b278a063a60d0ed 2 parents d86a4bd9 b49a3f34

Merge branch 'wn_promotion' of http://git.yoho.cn/yoho-search/yoho-search-service into wn_promotion

Inline Side-by-side
Showing 26 changed files with 376 additions and 138 deletions
... ... @@ -3,14 +3,25 @@ package com.yoho.search.common.cache.impls;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.common.cache.model.CacheObject;
import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
public interface CacheInterface {
public boolean exist(RedisKeyBuilder redisKeyBuilder);
public CacheObject get(RedisKeyBuilder redisKeyBuilder);
public void addOrUpdate(RedisKeyBuilder redisKeyBuilder, CacheObject value, int expiredTimeInMinute);
public boolean exist(RedisKeyBuilder redisKeyBuilder);
public void mutiSet(final Map<RedisKeyBuilder,String> map, final int expiredTimeInMinute);
public void delete(RedisKeyBuilder redisKeyBuilder);
public List<String> mutiGet(Collection<RedisKeyBuilder> redisKeyBuilders) ;
public CacheObject get(RedisKeyBuilder redisKeyBuilder);
public boolean setBitSet(RedisKeyBuilder redisKeyBuilder, BitSet bitSet, int expiredTimeInMinute);
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets);
}
... ...
package com.yoho.search.common.cache.impls;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.common.cache.model.CacheObject;
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
import net.sf.ehcache.Statistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import com.yoho.search.common.cache.model.CacheObject;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.InputStream;
import java.util.*;
@Service("ehCache")
public class EhCache implements CacheInterface {
... ... @@ -67,15 +61,6 @@ public class EhCache implements CacheInterface {
}
@Override
public void delete(RedisKeyBuilder redisKeyBuilder) {
try {
cache.remove(redisKeyBuilder.getKey());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Override
public CacheObject get(RedisKeyBuilder redisKeyBuilder) {
try {
Element e = cache.get(redisKeyBuilder.getKey());
... ... @@ -91,12 +76,13 @@ public class EhCache implements CacheInterface {
return this.get(redisKeyBuilder) == null ? false : true;
}
public void mutiSet(final Map<RedisKeyBuilder,String> map, final int timeoutInSecond) {
@Override
public void mutiSet(final Map<RedisKeyBuilder,String> map, final int expiredTimeInMinute) {
try {
Collection<Element> elements = new ArrayList<>();
for (Map.Entry<RedisKeyBuilder, ? extends String> entry: map.entrySet()) {
Element element = new Element(entry.getKey().getKey(), entry.getValue());
element.setTimeToLive(timeoutInSecond);
element.setTimeToLive(expiredTimeInMinute * 60);
elements.add(element);
}
cache.putAll(elements);
... ... @@ -105,6 +91,7 @@ public class EhCache implements CacheInterface {
}
}
@Override
public List<String> mutiGet(Collection<RedisKeyBuilder> redisKeyBuilders) {
Collection<String> keys = new ArrayList<>();
for (RedisKeyBuilder redisKeyBuilder:redisKeyBuilders) {
... ... @@ -121,6 +108,37 @@ public class EhCache implements CacheInterface {
}
}
return values;
}
@Override
public boolean setBitSet(RedisKeyBuilder redisKeyBuilder, BitSet bitSet, int expiredTimeInMinute) {
try {
Element element = new Element(redisKeyBuilder.getKey(),bitSet);
element.setTimeToLive(expiredTimeInMinute * 60);
cache.put(element);
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}
@Override
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets){
try {
Element e = this.cache.get(key.getKey());
if(e==null){
return null;
}
BitSet cacheBitSet = (BitSet)e.getObjectValue();
Map<Integer, Boolean> results = new HashMap<>();
for (Integer offset:offsets) {
results.put(offset,cacheBitSet.get(offset));
}
return results;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return null;
}
}
}
... ...
... ... @@ -11,6 +11,11 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@Service("searchRedis")
public class SearchRedis implements CacheInterface {
... ... @@ -20,6 +25,17 @@ public class SearchRedis implements CacheInterface {
private YohoSearchRedisComponent yohoSearchRedisComponent;
@Override
public boolean exist(RedisKeyBuilder redisKeyBuilder) {
try {
YHRedisTemplate redisTemplate = yohoSearchRedisComponent.getRedisTemplate();
return RedisCacheUtils.exist(redisTemplate, redisKeyBuilder);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}
@Override
public CacheObject get(RedisKeyBuilder redisKeyBuilder) {
try {
YHValueOperations valueOperations = yohoSearchRedisComponent.getValueOperations();
... ... @@ -38,28 +54,42 @@ public class SearchRedis implements CacheInterface {
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@Override
public void mutiSet(Map<RedisKeyBuilder, String> map, int expiredTimeInMinute) {
YHRedisTemplate redisTemplate = yohoSearchRedisComponent.getRedisTemplate();
RedisCacheUtils.mutiSet(redisTemplate, map, expiredTimeInMinute);
}
@Override
public void delete(RedisKeyBuilder redisKeyBuilder) {
public List<String> mutiGet(Collection<RedisKeyBuilder> redisKeyBuilders) {
YHValueOperations valueOperations = yohoSearchRedisComponent.getValueOperations();
return RedisCacheUtils.mutiGet(valueOperations, redisKeyBuilders);
}
@Override
public boolean setBitSet(RedisKeyBuilder redisKeyBuilder, BitSet bitSet, int expiredTimeInMinute) {
try {
YHRedisTemplate redisTemplate = yohoSearchRedisComponent.getRedisTemplate();
RedisCacheUtils.delete(redisTemplate, redisKeyBuilder);
return RedisCacheUtils.setBitSet(yohoSearchRedisComponent.getRedisTemplate(), redisKeyBuilder, bitSet, expiredTimeInMinute);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}
@Override
public boolean exist(RedisKeyBuilder redisKeyBuilder) {
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets) {
try {
YHRedisTemplate redisTemplate = yohoSearchRedisComponent.getRedisTemplate();
return RedisCacheUtils.exist(redisTemplate, redisKeyBuilder);
if (!redisTemplate.hasKey(key)) {
return null;
}
return RedisCacheUtils.getBits(redisTemplate, key, offsets);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
return null;
}
}
... ...
... ... @@ -13,11 +13,15 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
/**
* 使用有货共用的不同步redis
*
* @author hugufei
*
*/
@Service("yohoRedis")
... ... @@ -29,6 +33,16 @@ public class YohoRedis implements CacheInterface {
private YohoNoSyncRedisComponent yohoNoSyncRedisComponent;
@Override
public boolean exist(RedisKeyBuilder redisKeyBuilder) {
try {
return RedisCacheUtils.exist(yohoNoSyncRedisComponent.getRedisTemplate(), redisKeyBuilder);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}
@Override
public CacheObject get(RedisKeyBuilder redisKeyBuilder) {
try {
return RedisCacheUtils.get(yohoNoSyncRedisComponent.getValueOperations(), redisKeyBuilder, CacheObject.class);
... ... @@ -49,21 +63,37 @@ public class YohoRedis implements CacheInterface {
@Override
public void delete(RedisKeyBuilder redisKeyBuilder) {
public void mutiSet(Map<RedisKeyBuilder, String> map, int expiredTimeInMinute) {
YHRedisTemplate redisTemplate = yohoNoSyncRedisComponent.getRedisTemplate();
RedisCacheUtils.mutiSet(redisTemplate, map, expiredTimeInMinute);
}
@Override
public List<String> mutiGet(Collection<RedisKeyBuilder> redisKeyBuilders) {
YHValueOperations valueOperations = yohoNoSyncRedisComponent.getValueOperations();
return RedisCacheUtils.mutiGet(valueOperations, redisKeyBuilders);
}
@Override
public boolean setBitSet(RedisKeyBuilder redisKeyBuilder, BitSet bitSet, int expiredTimeInMinute) {
try {
RedisCacheUtils.delete(yohoNoSyncRedisComponent.getRedisTemplate(), redisKeyBuilder);
return RedisCacheUtils.setBitSet(yohoNoSyncRedisComponent.getRedisTemplate(), redisKeyBuilder, bitSet, expiredTimeInMinute);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
}
}
@Override
public boolean exist(RedisKeyBuilder redisKeyBuilder) {
public Map<Integer, Boolean> getFromBitSet(RedisKeyBuilder key, List<Integer> offsets) {
try {
return RedisCacheUtils.exist(yohoNoSyncRedisComponent.getRedisTemplate(), redisKeyBuilder);
if (!this.exist(key)) {
return null;
}
return RedisCacheUtils.getBits(yohoNoSyncRedisComponent.getRedisTemplate(), key, offsets);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return false;
return null;
}
}
... ...
... ... @@ -5,8 +5,10 @@ import com.alibaba.fastjson.serializer.SerializerFeature;
import com.yoho.core.redis.cluster.operations.nosync.YHRedisTemplate;
import com.yoho.core.redis.cluster.operations.nosync.YHValueOperations;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.utils.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
import java.util.concurrent.TimeUnit;
public class RedisCacheUtils {
... ... @@ -17,27 +19,63 @@ public class RedisCacheUtils {
if (StringUtils.isBlank(compressedVal)) {
return null;
}
String uncompressStr = SnappyUtils.uncompress(compressedVal);
return unserializeFromString(uncompressStr, clazz);
//String uncompressStr = SnappyUtils.uncompress(compressedVal);
return unserializeFromString(compressedVal, clazz);
}
public static <T> void add(YHValueOperations valueOperations, RedisKeyBuilder redisKeyBuilder, T t, int expiredTimeInMinute) {
String uncompressStr = serializeToString(t);
String compressedVal = SnappyUtils.compress(uncompressStr);
if (StringUtils.isBlank(compressedVal)) {
//String compressedVal = SnappyUtils.compress(uncompressStr);
if (StringUtils.isBlank(uncompressStr)) {
return;
}
valueOperations.set(redisKeyBuilder, compressedVal, expiredTimeInMinute, TimeUnit.MINUTES);
}
public static void delete(YHRedisTemplate redisTemplate, RedisKeyBuilder redisKeyBuilder) {
redisTemplate.delete(redisKeyBuilder);
valueOperations.set(redisKeyBuilder, uncompressStr, expiredTimeInMinute, TimeUnit.MINUTES);
}
public static boolean exist(YHRedisTemplate redisTemplate, RedisKeyBuilder redisKeyBuilder) {
return redisTemplate.hasKey(redisKeyBuilder);
}
public static void mutiSet(YHRedisTemplate redisTemplate, Map<RedisKeyBuilder, String> map, int expiredTimeInMinute) {
redisTemplate.mset(map, expiredTimeInMinute*60);
}
public static List<String> mutiGet(YHValueOperations valueOperations, Collection<RedisKeyBuilder> redisKeyBuilders) {
return valueOperations.multiGet(redisKeyBuilders);
}
/**
* 设置bitset-整个设置
* @param redisTemplate
* @param redisKeyBuilder
* @param bitSet
* @param expiredTimeInMinute
* @return
*/
public static boolean setBitSet(YHRedisTemplate redisTemplate, RedisKeyBuilder redisKeyBuilder, BitSet bitSet, int expiredTimeInMinute) {
return redisTemplate.setBits(redisKeyBuilder,bitSet,expiredTimeInMinute * 60);
}
/**
* 获取bitset
* @param redisTemplate
* @param redisKeyBuilder
* @param offsets
* @return
*/
public static Map<Integer, Boolean> getBits(YHRedisTemplate redisTemplate, RedisKeyBuilder redisKeyBuilder, List<Integer> offsets) {
List<Long> LongOffsets = CollectionUtils.toList(offsets,(a)->a.longValue());
Map<Long, Boolean> redisResults = redisTemplate.getBits(redisKeyBuilder,LongOffsets);
if(redisResults==null || redisResults.isEmpty()){
return null;
}
Map<Integer, Boolean> results = new HashMap<>();
for (Map.Entry<Long, Boolean> entry: redisResults.entrySet()) {
results.put(entry.getKey().intValue(),entry.getValue());
}
return results;
}
/**
* 序列化保持null值
*
... ...
... ... @@ -19,15 +19,14 @@ import com.yoho.search.service.base.ProductListSortKey;
import com.yoho.search.service.base.ProductListSortService;
import com.yoho.search.service.base.SearchDynamicConfigService;
import com.yoho.search.service.scorer.personal.PersonalVectorFeatureSearch;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import sun.applet.Main;
import java.util.*;
@Component
public class UserRecallResponseBuilder {
... ...
... ... @@ -4,7 +4,7 @@ import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.utils.CollectionUtils;
import com.yoho.search.base.utils.Transfer;
import com.yoho.search.common.cache.impls.EhCache;
import com.yoho.search.core.redis.components.YohoSearchRedisComponent;
import com.yoho.search.common.cache.impls.SearchRedis;
import com.yoho.search.recall.scene.models.common.AbstractCacheRequestResponse;
import com.yoho.search.recall.scene.models.common.ICacheRequest;
import org.apache.commons.lang.StringUtils;
... ... @@ -19,9 +19,9 @@ public abstract class AbstractCacheBean<Request extends ICacheRequest, Response,
private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
@Autowired
private YohoSearchRedisComponent yohoSearchRedisComponent;
@Autowired
private EhCache ehCache;
@Autowired
private SearchRedis searchRedis;
/**
* 批量处理请求，以及添加缓存
... ... @@ -170,7 +170,7 @@ public abstract class AbstractCacheBean<Request extends ICacheRequest, Response,
if(request==null){
return 0;
}
return request.cacheTimeInSecond();
return request.cacheTimeInMinute();
}
});
//2、按缓存时间大小直接加入缓存
... ... @@ -179,7 +179,7 @@ public abstract class AbstractCacheBean<Request extends ICacheRequest, Response,
}
}
private void batchAddResponseToCache(List<RequestResponse> requestResponses,int timeOutInSecond) {
private void batchAddResponseToCache(List<RequestResponse> requestResponses,int expiredTimeInMinute) {
try {
Map<RedisKeyBuilder, String> toCacheMap = new HashMap<>();
for (RequestResponse requestResponse : requestResponses) {
... ... @@ -205,7 +205,7 @@ public abstract class AbstractCacheBean<Request extends ICacheRequest, Response,
if (toCacheMap.isEmpty()) {
return;
}
this.batchAddToCache(toCacheMap, timeOutInSecond);
this.batchAddToCache(toCacheMap, expiredTimeInMinute);
} catch (Exception e) {
RECALL_NEW_LOGGER.error(e.getMessage(), e);
}
... ... @@ -215,15 +215,15 @@ public abstract class AbstractCacheBean<Request extends ICacheRequest, Response,
if (useEhCache()){
return ehCache.mutiGet(keys);
} else {
return yohoSearchRedisComponent.getValueOperations().multiGet(keys);
return searchRedis.mutiGet(keys);
}
}
private void batchAddToCache(Map<RedisKeyBuilder, String> toCacheMap,int timeOutInSecond) {
private void batchAddToCache(Map<RedisKeyBuilder, String> toCacheMap,int expiredTimeInMinute) {
if (useEhCache()) {
ehCache.mutiSet(toCacheMap, timeOutInSecond);
ehCache.mutiSet(toCacheMap, expiredTimeInMinute);
} else {
yohoSearchRedisComponent.getRedisTemplate().mset(toCacheMap, timeOutInSecond);
searchRedis.mutiSet(toCacheMap,expiredTimeInMinute);
}
}
... ...
package com.yoho.search.recall.scene.beans.cache;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.impls.EhCache;
import com.yoho.search.common.cache.impls.SearchRedis;
import com.yoho.search.common.cache.model.CacheObject;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.recall.scene.beans.persional.PageProductIdBitSetComponent;
import com.yoho.search.recall.scene.beans.strategy.impls.RecommendSknStrategy;
import com.yoho.search.recall.scene.models.common.ParamQueryFilter;
import com.yoho.search.recall.scene.models.personal.PageProductIdBitSet;
import com.yoho.search.recall.scene.models.req.RecallRequest;
import com.yoho.search.recall.scene.models.req.RecallRequestResponse;
import com.yoho.search.recall.scene.models.req.RecallResponse;
... ... @@ -15,13 +20,17 @@ import com.yoho.search.service.base.SearchCommonService;
import org.apache.commons.collections.MapUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
@Component
public class SknRecallCacheBean{
public class SknRecallCacheBean {
private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
@Autowired
private SearchCommonService searchCommonService;
... ... @@ -29,22 +38,132 @@ public class SknRecallCacheBean{
private PageProductIdBitSetComponent pageProductIdBitSetComponent;
@Autowired
private SknBaseInfoCacheBean sknBaseInfoCacheBean;
@Autowired
private SearchRedis searchRedis;
@Autowired
private EhCache ehCache;
private static final Boolean recallWithCache = true;
/**
* 将推荐的skn构造成召回对象
*
* @return
*/
public List<RecallRequestResponse> batchRecallRecommedSknList(UserRecallRequest userRecallRequest,List<Integer> recommedSknList,int maxReturnCount){
if(recommedSknList==null || recommedSknList.isEmpty()){
public List<RecallRequestResponse> batchRecallRecommedSknList(UserRecallRequest userRecallRequest, List<Integer> recommedSknList, int maxReturnCount) {
try {
if (recommedSknList == null || recommedSknList.isEmpty()) {
return new ArrayList<>();
//localhost:8080/yohosearch/productindex/productList.json?uid=11145&status=1&gender=1,2,3,4,5
//recommedSknList = Arrays.asList(50020667,50026235,50026280,50024584,50026611,50028826,50031356,50031387,50029959,51001108);
}
List<Integer> filterSknList;
if (recallWithCache) {
filterSknList = this.filterRecommendWithCache(userRecallRequest, recommedSknList);
} else {
filterSknList = this.filterRecommedSknListByEs(userRecallRequest, recommedSknList);
}
return this.buildResults(userRecallRequest, filterSknList, maxReturnCount);
}catch (Exception e){
RECALL_NEW_LOGGER.error(e.getMessage(),e);
return new ArrayList<>();
}
List<Integer> filterSknList = this.filterRecommedSknList(userRecallRequest,recommedSknList,maxReturnCount);
return this.buildResults(userRecallRequest,filterSknList);
}
private List<Integer> filterRecommedSknList(UserRecallRequest userRecallRequest, List<Integer> recommedSknList,int maxReturnCount){
if(recommedSknList==null || recommedSknList.isEmpty()){
/**
* 从ehcahc或者redis中构造返回结果
* @param userRecallRequest
* @param recommedSknList
* @return
*/
private List<Integer> filterRecommendWithCache(UserRecallRequest userRecallRequest, List<Integer> recommedSknList) {
//1、skn转成productId
Map<Integer, Integer> productSknToIdMap = sknBaseInfoCacheBean.queryProductSknToProductIdMap(recommedSknList);
if(productSknToIdMap==null || productSknToIdMap.isEmpty()){
return new ArrayList<>();
}
List<Integer> recommedProductIds = new ArrayList<>();
recommedProductIds.addAll(productSknToIdMap.values());
//2、先从本地缓存获取
ParamQueryFilter paramQueryFilter = userRecallRequest.getParamQueryFilter();
RedisKeyBuilder redisKeyBuilder = pageProductIdBitSetComponent.genRedisKeyBuilder(paramQueryFilter);
Map<Integer, Boolean> ehcacheQueryResult = ehCache.getFromBitSet(redisKeyBuilder,recommedProductIds);
if (ehcacheQueryResult != null) {
return this.filterSknByCacheQueryResult(recommedSknList,productSknToIdMap, ehcacheQueryResult);
}
//3、再从redis的bitset缓存结构中过滤
Map<Integer, Boolean> redisQueryResult = searchRedis.getFromBitSet(redisKeyBuilder, recommedProductIds);
if (redisQueryResult != null) {
//TODO 回写本地缓存
return this.filterSknByCacheQueryResult(recommedSknList, productSknToIdMap,redisQueryResult);
}
//4、执行查询，添加到缓存
PageProductIdBitSet pageProductIdBitSet = pageProductIdBitSetComponent.queryPageProductIdBitSet(userRecallRequest.getParamQueryFilter());
//3.1)添加到本地缓存
this.ehCache.setBitSet(redisKeyBuilder,pageProductIdBitSet.getProductIdBitSet(),pageProductIdBitSetComponent.cacheTimeInMinute());
//3.2)添加到redis缓存
//TODO 异步加入redis缓存
this.searchRedis.setBitSet(redisKeyBuilder,pageProductIdBitSet.getProductIdBitSet(),pageProductIdBitSetComponent.cacheTimeInMinute());
return this.filterSknByPageProductIdBitSet(recommedSknList,productSknToIdMap, pageProductIdBitSet.getProductIdBitSet());
}
/**
* 从bitset对象中构造返回对象
*
* @param recommedSknList
* @param bitSet
* @return
*/
private List<Integer> filterSknByPageProductIdBitSet(List<Integer> recommedSknList,Map<Integer, Integer> productSknToIdMap, BitSet bitSet) {
List<Integer> existProductSkns = new ArrayList<>();
for (Integer recommedSkn : recommedSknList) {
Integer productId = productSknToIdMap.get(recommedSkn);
if(productId==null){
continue;
}
if (!bitSet.get(productId)) {
continue;
}
existProductSkns.add(recommedSkn);
}
return existProductSkns;
}
/**
* 根据redis返回结果按顺序构结果，只返回为true的
*
* @param recommedSknList
* @param cacheQueryResult
* @return
*/
private List<Integer> filterSknByCacheQueryResult(List<Integer> recommedSknList,Map<Integer, Integer> productSknToIdMap, Map<Integer, Boolean> cacheQueryResult) {
List<Integer> existProductSkns = new ArrayList<>();
for (Integer recommedSkn : recommedSknList) {
Integer productId = productSknToIdMap.get(recommedSkn);
if(productId==null){
continue;
}
if (!cacheQueryResult.getOrDefault(productId, false)) {
continue;
}
existProductSkns.add(recommedSkn);
}
return existProductSkns;
}
/**
* 直接从es召回
*
* @param userRecallRequest
* @param recommedSknList
* @return
*/
private List<Integer> filterRecommedSknListByEs(UserRecallRequest userRecallRequest, List<Integer> recommedSknList) {
if (recommedSknList == null || recommedSknList.isEmpty()) {
return new ArrayList<>();
}
//1、构造searchParam
... ... @@ -53,7 +172,7 @@ public class SknRecallCacheBean{
searchParam.setQuery(paramQueryFilter.getParamQuery());
BoolQueryBuilder realFilter = QueryBuilders.boolQuery();
realFilter.must(paramQueryFilter.getParamFilter());
realFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.productSkn,recommedSknList));
realFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, recommedSknList));
searchParam.setFiter(realFilter);
//2、设置分页参数
... ... @@ -74,11 +193,8 @@ public class SknRecallCacheBean{
//5、按顺序构造最终结果
List<Integer> results = new ArrayList<>();
for (Integer recommedSkn: recommedSknList) {
if(results.size()>=maxReturnCount){
continue;
}
if(existsProductSkns.contains(recommedSkn)){
for (Integer recommedSkn : recommedSknList) {
if (existsProductSkns.contains(recommedSkn)) {
results.add(recommedSkn);
}
}
... ... @@ -87,18 +203,22 @@ public class SknRecallCacheBean{
/**
* 将推荐的skn构造成召回对象
*
* @return
*/
private List<RecallRequestResponse> buildResults(UserRecallRequest userRecallRequest,final List<Integer> recommedSknList){
private List<RecallRequestResponse> buildResults(UserRecallRequest userRecallRequest, final List<Integer> recommedSknList, int maxReturnCount) {
List<RecallRequestResponse> results = new ArrayList<>();
if(recommedSknList==null || recommedSknList.isEmpty()){
if (recommedSknList == null || recommedSknList.isEmpty()) {
return results;
}
ParamQueryFilter paramQueryFilter = userRecallRequest.getParamQueryFilter();
for (Integer recommendSkn : recommedSknList){
RecallRequest recallRequest = new RecallRequest(paramQueryFilter,new RecommendSknStrategy(recommendSkn));
for (Integer recommendSkn : recommedSknList) {
if (results.size() >= maxReturnCount) {
break;
}
RecallRequest recallRequest = new RecallRequest(paramQueryFilter, new RecommendSknStrategy(recommendSkn));
RecallRequestResponse recallRequestResponse = new RecallRequestResponse(recallRequest);
recallRequestResponse.setResponse(new RecallResponse(1L, Arrays.asList(recommendSkn)),false);
recallRequestResponse.setResponse(new RecallResponse(1L, Arrays.asList(recommendSkn)), false);
results.add(recallRequestResponse);
}
return results;
... ...
... ... @@ -32,7 +32,7 @@ public abstract class AbstractPageComponent<T> {
@PostConstruct
void init() {
mapLock = new ConcurrentHashMap(20);//使用一个Map来限流
mapLock = new ConcurrentHashMap<String, Integer>(20);//使用一个Map来限流
executorService = Executors.newFixedThreadPool(5);
}
... ... @@ -43,20 +43,20 @@ public abstract class AbstractPageComponent<T> {
return null;
}
//2、取本地缓存，缓存命中，则直接返回
T result = this.getJavaObjectFromEhcache(redisKeyBuilder);
T result = this.getJavaObjectFromEhcache(redisKeyBuilder, clazz);
if (result != null) {
return result;
}
//3、取redis缓存，缓存命中，则回写ehcahce
result = this.getJavaObjectFromRedis(redisKeyBuilder,clazz);
result = this.getJavaObjectFromRedis(redisKeyBuilder, clazz);
if (result != null) {
this.addJavaObjectToEhcache(redisKeyBuilder,result);
this.addJavaObjectToEhcache(redisKeyBuilder, result);
return result;
}
//4、如果是同步查询，则直接调子类方法查询
if (synchronous) {
return this.doInnerQuery(redisKeyBuilder,paramQueryFilter);
return this.doInnerQuery(redisKeyBuilder, paramQueryFilter);
}
//5、执行异步流程
... ... @@ -67,7 +67,7 @@ public abstract class AbstractPageComponent<T> {
//6、异步执行查询并加入缓存
executorService.submit(() -> {
try {
this.doInnerQuery(redisKeyBuilder,paramQueryFilter);
this.doInnerQuery(redisKeyBuilder, paramQueryFilter);
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
... ... @@ -77,19 +77,27 @@ public abstract class AbstractPageComponent<T> {
return null;
}
private T getJavaObjectFromEhcache(RedisKeyBuilder redisKeyBuilder){
private T doInnerQuery(RedisKeyBuilder redisKeyBuilder, ParamQueryFilter paramQueryFilter) {
T result = this.doRealQuery(paramQueryFilter);
this.addJavaObjectToEhcache(redisKeyBuilder, result);
this.addJavaObjectToRedis(redisKeyBuilder, result);
return result;
}
private T getJavaObjectFromEhcache(RedisKeyBuilder redisKeyBuilder, Class<T> clazz) {
CacheObject cacheObject = ehCache.get(redisKeyBuilder);
if (cacheObject != null) {
return (T)cacheObject.toObject();
Object object = cacheObject.toObject();
return (T) object;
}
return null;
}
private void addJavaObjectToEhcache(RedisKeyBuilder redisKeyBuilder,T result){
ehCache.addOrUpdate(redisKeyBuilder, new CacheObject(result), this.cacheTimeInSecond() / 60);
private void addJavaObjectToEhcache(RedisKeyBuilder redisKeyBuilder, T result) {
ehCache.addOrUpdate(redisKeyBuilder, new CacheObject(result), this.cacheTimeInMinute());
}
private T getJavaObjectFromRedis(RedisKeyBuilder redisKeyBuilder,Class<T> clazz){
protected T getJavaObjectFromRedis(RedisKeyBuilder redisKeyBuilder, Class<T> clazz) {
CacheObject cacheObject = searchRedis.get(redisKeyBuilder);
if (cacheObject == null) {
return null;
... ... @@ -99,23 +107,15 @@ public abstract class AbstractPageComponent<T> {
return result;
}
private void addJavaObjectToRedis(RedisKeyBuilder redisKeyBuilder,T result){
protected void addJavaObjectToRedis(RedisKeyBuilder redisKeyBuilder, T result) {
JSONObject jsonObject = JSON.parseObject(JSON.toJSONString(result));
CacheObject toCacheResult = new CacheObject(jsonObject);
searchRedis.addOrUpdate(redisKeyBuilder, toCacheResult, this.cacheTimeInSecond() / 60);
}
private T doInnerQuery(RedisKeyBuilder redisKeyBuilder, ParamQueryFilter paramQueryFilter) {
T queryResult = doRealQuery(paramQueryFilter);
JSONObject jsonObject = JSON.parseObject(JSON.toJSONString(queryResult));
CacheObject toCacheResult = new CacheObject(jsonObject);
searchRedis.addOrUpdate(redisKeyBuilder, toCacheResult, this.cacheTimeInSecond() / 60);
return queryResult;
searchRedis.addOrUpdate(redisKeyBuilder, toCacheResult, this.cacheTimeInMinute());
}
protected abstract RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter);
protected abstract int cacheTimeInSecond();
protected abstract int cacheTimeInMinute();
protected abstract T doRealQuery(ParamQueryFilter paramQueryFilter);
... ...
... ... @@ -46,7 +46,7 @@ public class PagePersionalFactorComponent extends AbstractPageComponent<PagePers
}
@Override
protected int cacheTimeInSecond() {
protected int cacheTimeInMinute() {
return CacheTimeConstants.PAGE_PERSIONAL_FACTOR;
}
... ...
package com.yoho.search.recall.scene.beans.persional;
import com.alibaba.fastjson.JSON;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
... ... @@ -20,7 +19,6 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import sun.applet.Main;
import java.util.ArrayList;
import java.util.Iterator;
... ... @@ -40,19 +38,16 @@ public class PageProductIdBitSetComponent extends AbstractPageComponent<PageProd
* @return
*/
public PageProductIdBitSet queryPageProductIdBitSet(ParamQueryFilter paramQueryFilter) {
//Object value = super.queryWithCache(paramQueryFilter);
//return value == null ? null : (PageProductIdBitSet) value;
//return doRealQuery(paramQueryFilter);
return null;
return this.doRealQuery(paramQueryFilter);
}
@Override
protected RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
public RedisKeyBuilder genRedisKeyBuilder(ParamQueryFilter paramQueryFilter) {
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:PAGE_PRODUCT_BITSET:").appendVar(paramQueryFilter.getParamMd5Key());
}
@Override
protected int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.PAGE_SKN_BITSET;
}
... ...
... ... @@ -4,10 +4,8 @@ import com.alibaba.fastjson.JSON;
import com.yoho.search.core.personalized.models.SortBrand;
import com.yoho.search.core.personalized.models.SortPriceAreas;
import com.yoho.search.core.personalized.models.UserPersonalFactorRspNew;
import com.yoho.search.recall.scene.beans.cache.SknBaseInfoCacheBean;
import com.yoho.search.recall.scene.models.personal.PageBrandSorts;
import com.yoho.search.recall.scene.models.personal.PagePersonalFactor;
import com.yoho.search.recall.scene.models.personal.PageProductIdBitSet;
import com.yoho.search.recall.scene.models.personal.UserPersonalFactor;
import com.yoho.search.recall.scene.models.req.UserRecallRequest;
import com.yoho.search.service.base.SearchDynamicConfigService;
... ... @@ -15,7 +13,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import sun.applet.Main;
import java.util.*;
... ...
... ... @@ -15,12 +15,12 @@ public interface IStrategy {
String strategyCacheKey();// 可用于缓存的key
int cacheTimeInSecond();//缓存时间
int cacheTimeInMinute();//缓存时间
default StringBuilder defaultStrategyKey(){
StringBuilder sb = new StringBuilder();
sb.append(this.strategtEnum().name());
sb.append(this.cacheTimeInSecond());
sb.append(this.cacheTimeInMinute());
sb.append(this.size());
return sb;
}
... ...
... ... @@ -43,7 +43,7 @@ public class CommonAddFlowStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.COMMON_RECALL_STRATEGY_CACHE_TIME;
}
... ...
... ... @@ -43,7 +43,7 @@ public class CommonDirectTrainStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.COMMON_RECALL_STRATEGY_CACHE_TIME;
}
... ...
... ... @@ -48,7 +48,7 @@ public class CommonFirstSknStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.COMMON_RECALL_STRATEGY_CACHE_TIME;
}
... ...
... ... @@ -43,7 +43,7 @@ public class CommonHeatValueStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.COMMON_RECALL_STRATEGY_CACHE_TIME;
}
... ...
... ... @@ -43,7 +43,7 @@ public class CommonNewShopStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.COMMON_RECALL_STRATEGY_CACHE_TIME;
}
... ...
... ... @@ -43,7 +43,7 @@ public class RecommendSknStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.RECOMMEND_SKN_CACHE_TIME;
}
... ...
... ... @@ -21,7 +21,7 @@ public abstract class SortBrandAbstractStrategy implements IStrategy {
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.SORT_BRAND_RECALL_STRATEGY_CACHE_TIME;
}
... ...
... ... @@ -7,27 +7,27 @@ package com.yoho.search.recall.scene.constants;
public class CacheTimeConstants {
//通用召回的缓存 - 10分钟
public static final int COMMON_RECALL_STRATEGY_CACHE_TIME = 10 * 60;
public static final int COMMON_RECALL_STRATEGY_CACHE_TIME = 10 ;
//SKN的的缓存 - 60分钟
public static final int RECOMMEND_SKN_CACHE_TIME = 60 * 60;
public static final int RECOMMEND_SKN_CACHE_TIME = 60 ;
//品类+品牌的缓存 - 60分钟
public static final int SORT_BRAND_RECALL_STRATEGY_CACHE_TIME = 60 * 60;
public static final int SORT_BRAND_RECALL_STRATEGY_CACHE_TIME = 60 ;
//SKN基本信息的缓存-一个小时
public static final int SKN_BASE_INFO = 60 * 60;
public static final int SKN_BASE_INFO = 60 ;
//SKN返回信息的缓存 - 10分钟
public static final int SKN_RETURN_INFO = 10 * 60;
public static final int SKN_RETURN_INFO = 10 ;
//用户召回结果的缓存-三分钟
public static final int USER_RECALL_SKN_LIST = 3 * 60;
public static final int USER_RECALL_SKN_LIST = 3 ;
//页面skn的bitset缓存
public static final int PAGE_SKN_BITSET = 60 * 60;
public static final int PAGE_SKN_BITSET = 60 ;
//页面个性化因子的缓存
public static final int PAGE_PERSIONAL_FACTOR = 60 * 60;
public static final int PAGE_PERSIONAL_FACTOR = 60 ;
}
... ...
... ... @@ -13,5 +13,5 @@ public interface ICacheRequest {
* 缓存时间
* @return
*/
int cacheTimeInSecond();
int cacheTimeInMinute();
}
... ...
... ... @@ -38,7 +38,7 @@ public class RecallRequest implements ICacheRequest, IRecallRequest {
RedisKeyBuilder redisKeyBuilder = RedisKeyBuilder.newInstance();
redisKeyBuilder.appendFixed("YOHOSEARCH:").appendFixed("RECALL:");
redisKeyBuilder.appendFixed(strategy.strategtEnum().name()).appendFixed(":");
redisKeyBuilder.appendVar(cacheTimeInSecond()).appendFixed(":");
redisKeyBuilder.appendVar(cacheTimeInMinute()).appendFixed(":");
redisKeyBuilder.appendVar(cacheKey);
return redisKeyBuilder;
}
... ... @@ -49,8 +49,8 @@ public class RecallRequest implements ICacheRequest, IRecallRequest {
}
@Override
public int cacheTimeInSecond() {
return this.strategy.cacheTimeInSecond();
public int cacheTimeInMinute() {
return this.strategy.cacheTimeInMinute();
}
@Override
... ...
... ... @@ -18,11 +18,11 @@ public class SknBaseInfoRequest implements ICacheRequest{
@Override
public RedisKeyBuilder redisKeyBuilder() {
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("SKN_BASE_INFO:").appendVar(cacheTimeInSecond()).appendFixed(":").appendVar(productSkn);
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("SKN_BASE_INFO:").appendVar(cacheTimeInMinute()).appendFixed(":").appendVar(productSkn);
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.SKN_BASE_INFO;
}
... ...
... ... @@ -14,11 +14,11 @@ public class SknReturnInfoResquest implements ICacheRequest {
@Override
public RedisKeyBuilder redisKeyBuilder() {
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("SKN_INFO:").appendVar(cacheTimeInSecond()).appendFixed(":").appendVar(productSkn);
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("SKN_INFO:").appendVar(cacheTimeInMinute()).appendFixed(":").appendVar(productSkn);
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.SKN_RETURN_INFO;
}
... ...
... ... @@ -55,11 +55,11 @@ public class UserRecallRequest implements ICacheRequest {
stringBuilder.append(uid);
stringBuilder.append(udid==null?"": udid);
String value = MD5Util.string2MD5(stringBuilder.toString());
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("USER_RECALL_SKN_LIST:").appendVar(cacheTimeInSecond()).appendFixed(":").appendVar(value);
return RedisKeyBuilder.newInstance().appendFixed("YOHOSEARCH:").appendFixed("USER_RECALL_SKN_LIST:").appendVar(cacheTimeInMinute()).appendFixed(":").appendVar(value);
}
@Override
public int cacheTimeInSecond() {
public int cacheTimeInMinute() {
return CacheTimeConstants.USER_RECALL_SKN_LIST;
}
... ...