Authored by hugufei

针对提前聚合做缓存

package com.yoho.search.cache;
import java.util.concurrent.atomic.AtomicLong;
public class CacheCount {
private AtomicLong totalCount = new AtomicLong(0);
private AtomicLong matchCount = new AtomicLong(0);
public void clear() {
totalCount = new AtomicLong(0);
matchCount = new AtomicLong(0);
}
public void incTotalCount() {
totalCount.incrementAndGet();
}
public void incMatchCount() {
totalCount.incrementAndGet();
}
public AtomicLong getTotalCount() {
return totalCount;
}
public AtomicLong getMatchCount() {
return matchCount;
}
public int getMatchPercent() {
long matchCnt = matchCount.longValue();
long totalCnt = totalCount.longValue();
if (totalCnt == 0) {
return 0;
}
return (int) (matchCnt * 100L / totalCnt);
}
@Override
public String toString() {
return "CacheCount [totalCount=" + totalCount + ", matchCount=" + matchCount + ", matchPercent=" + getMatchPercent() + "]";
}
}
... ...
package com.yoho.search.cache;
public class CacheObject {
private Object object;
private long expireTime;// 过期秒数
public CacheObject(Object object,long expireInSecond) {
super();
this.object = object;
this.expireTime = System.currentTimeMillis() + expireInSecond * 1000L;
}
public Object getObject() {
return object;
}
public long getExpireTime() {
return expireTime;
}
}
... ...
package com.yoho.search.cache;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import com.yoho.search.dal.model.SearchParam;
import com.yoho.search.utils.MD5Util;
import com.yoho.search.utils.SearchParamUtils;
@Service
public class CacheService {
private static final Logger logger = LoggerFactory.getLogger(CacheService.class);
private Map<String,CacheObject> cache = new ConcurrentHashMap<String,CacheObject>();
private CacheCount cacheCount = new CacheCount();
private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2);
@PostConstruct
void init(){
//每5分钟记录下缓存命中率
scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
logger.info("do log cache matchPercent,[{}]",cacheCount);
cacheCount.clear();
}
}, 1, 5, TimeUnit.MINUTES);
//每30分钟清除缓存对象,防止内存爆掉
scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
logger.info("do clear cacheObject...");
cache = new ConcurrentHashMap<String,CacheObject>();
}
}, 30, 30, TimeUnit.MINUTES);
}
private Object getObjectFromCache(String key){
cacheCount.incTotalCount();
CacheObject cacheObject = cache.get(key);
if(cacheObject==null){
return null;
}
if(cacheObject.getExpireTime() < System.currentTimeMillis()){
cache.remove(key);
return null;
}
cacheCount.incMatchCount();
return cacheObject.getObject();
}
private void addObjectToCache(String key,Object object,long expireInSecond){
CacheObject cacheObject = new CacheObject(object, expireInSecond);
cache.put(key, cacheObject);
}
/**
* 根据SearchParam生成的ES报文
* @param searchParam
* @return
*/
private String genSearchParamString(String indexName,SearchParam searchParam){
StringBuilder sb = new StringBuilder();
//拼装索引名称
sb.append("indexName:").append(indexName).append(";");
//拼装搜索类型
sb.append("searchType:").append(searchParam.getSearchType()==null?"":searchParam.getSearchType().name()).append(";");
//拼装报文
SearchSourceBuilder searchSourceBuilder = SearchParamUtils.genSearchSourceBuilderFromSearchParam(searchParam);
sb.append("searchSource:").append(searchSourceBuilder.toString()).append(";");
//打印拼装结果
logger.info("cacheKey is [{}]",sb.toString());
return MD5Util.string2MD5(sb.toString());
}
public Object getObjectFromCache(String indexName,SearchParam searchParam){
String key = this.genSearchParamString(indexName,searchParam);
return this.getObjectFromCache(key);
}
public void addObjectToCache(String indexName,SearchParam searchParam,Object object){
String key = this.genSearchParamString(indexName,searchParam);
this.addObjectToCache(key, object, 5 * 60);
}
}
... ...
... ... @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service;
import com.yoho.search.aggregations.IAggregation;
import com.yoho.search.aggregations.impls.AggregationFactoryService;
import com.yoho.search.cache.CacheService;
import com.yoho.search.dal.model.SearchParam;
import com.yoho.search.dal.model.SearchResult;
import com.yoho.search.utils.ISearchConstans;
... ... @@ -39,6 +40,8 @@ public class SearchProductsService {
private SearchCommonService searchCommonService;
@Autowired
private AggregationFactoryService aggregationFactoryService;
@Autowired
private CacheService cacheService;
/**
* 搜索接口[商品列表以及一堆聚合结果]
... ... @@ -131,12 +134,17 @@ public class SearchProductsService {
private Map<String, Object> preAggregationSearch(SearchParam searchParam, Map<String, String> paramMap) throws Exception {
Map<String, Object> preAggregationResult = new HashMap<String, Object>();
// 1)年龄层
// if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_AGELEVEL) && StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_AGELEVEL))) {
// IAggregation aggregation = aggregationFactoryService.getAgeLevelAggregation();
// searchParam.setAggregationBuilders(Arrays.asList(aggregation.getBuilder()));
// searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap, ISearchConstans.PARAM_SEARCH_AGELEVEL));
// preAggregationResult.put(aggregation.aggName(), doPreSearch(searchParam.clone(), paramMap));
// }
// if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_AGELEVEL) &&
// StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_AGELEVEL)))
// {
// IAggregation aggregation =
// aggregationFactoryService.getAgeLevelAggregation();
// searchParam.setAggregationBuilders(Arrays.asList(aggregation.getBuilder()));
// searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap,
// ISearchConstans.PARAM_SEARCH_AGELEVEL));
// preAggregationResult.put(aggregation.aggName(),
// doPreSearch(searchParam.clone(), paramMap));
// }
// 2)性别
if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_GENDER) && StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_GENDER))) {
IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap);
... ... @@ -176,7 +184,7 @@ public class SearchProductsService {
}
/**
* 预聚合,并获取聚合结果[因为filter的参数不同,包含了一定的业务逻辑]
* 预聚合,并获取聚合结果[因为filter的参数不同,包含了一定的业务逻辑][针对这里做一下缓存]
*
* @param searchParam
* @param paramMap
... ... @@ -192,40 +200,56 @@ public class SearchProductsService {
searchParam.setOffset(1);
searchParam.setSize(1);
final String indexName = ISearchConstans.INDEX_NAME_PRODUCT_INDEX;
// 从缓存中取聚合结果
Object objectFromCache = cacheService.getObjectFromCache(indexName, searchParam);
if (objectFromCache != null) {
return null;
}
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
if (aggMaps == null) {
return null;
}
// 1)获取年龄层的聚合结果
// IAggregation ageLevelAggregation = aggregationFactoryService.getAgeLevelAggregation();
// if (aggMaps.containsKey(ageLevelAggregation.aggName())) {
// return ageLevelAggregation.getAggregationResponseMap(aggMaps);
// }
// IAggregation ageLevelAggregation = aggregationFactoryService.getAgeLevelAggregation();
// if (aggMaps.containsKey(ageLevelAggregation.aggName())) {
// return ageLevelAggregation.getAggregationResponseMap(aggMaps);
// }
// 2)获取性别层的聚合结果
IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap);
if (aggMaps.containsKey(genderAggregation.aggName())) {
return genderAggregation.getAggregationResponseMap(aggMaps);
Object genderResult = genderAggregation.getAggregationResponseMap(aggMaps);
cacheService.addObjectToCache(indexName, searchParam, genderResult);
return genderResult;
}
// 3)获取价格层的聚合结果
IAggregation priceAggregation = aggregationFactoryService.getPriceAggregation();
if (aggMaps.containsKey(priceAggregation.aggName())) {
return priceAggregation.getAggregationResponseMap(aggMaps);
Object result = priceAggregation.getAggregationResponseMap(aggMaps);
cacheService.addObjectToCache(indexName, searchParam, result);
return result;
}
// 4)获取颜色层的聚合结果
IAggregation collorAggregation = aggregationFactoryService.getColorAggregation(paramMap);
if (aggMaps.containsKey(collorAggregation.aggName())) {
return collorAggregation.getAggregationResponseMap(aggMaps);
Object result = collorAggregation.getAggregationResponseMap(aggMaps);
cacheService.addObjectToCache(indexName, searchParam, result);
return result;
}
// 5)获取风格层面的聚合结果
IAggregation styleAggregation = aggregationFactoryService.getStyleAggregation(paramMap);
if (aggMaps.containsKey(styleAggregation.aggName())) {
return styleAggregation.getAggregationResponseMap(aggMaps);
Object result = styleAggregation.getAggregationResponseMap(aggMaps);
cacheService.addObjectToCache(indexName, searchParam, result);
return result;
}
// 6)获取品牌层面的聚合结果
IAggregation brandaAggregation = aggregationFactoryService.getBrandAggregation(paramMap);
if (aggMaps.containsKey(brandaAggregation.aggName())) {
return brandaAggregation.getAggregationResponseMap(aggMaps);
Object result = brandaAggregation.getAggregationResponseMap(aggMaps);
cacheService.addObjectToCache(indexName, searchParam, result);
return result;
}
return null;
}
... ... @@ -240,9 +264,10 @@ public class SearchProductsService {
private List<AbstractAggregationBuilder> getAllAggregations(SearchParam searchParam, Map<String, String> paramMap) {
List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>();
// 年龄层
// if (!paramMap.containsKey("ageLevel") || StringUtils.isBlank(paramMap.get("ageLevel"))) {
// list.add(aggregationFactoryService.getAgeLevelAggregation().getBuilder());
// }
// if (!paramMap.containsKey("ageLevel") ||
// StringUtils.isBlank(paramMap.get("ageLevel"))) {
// list.add(aggregationFactoryService.getAgeLevelAggregation().getBuilder());
// }
// 价格
if (!paramMap.containsKey("price") || StringUtils.isBlank(paramMap.get("price"))) {
list.add(aggregationFactoryService.getPriceAggregation().getBuilder());
... ... @@ -307,11 +332,14 @@ public class SearchProductsService {
private Map<String, Object> buildFilterResult(Map<String, Aggregation> aggMaps, Map<String, Object> preAggregationResult, Map<String, String> paramMap) throws Exception {
Map<String, Object> filter = new HashMap<String, Object>();
// 1)获取年龄的聚合结果
// IAggregation ageLevelAggregation = aggregationFactoryService.getAgeLevelAggregation();
// Object ageLevelResponse = this.getResponseByIAggregation(ageLevelAggregation, preAggregationResult, aggMaps);
// if (ageLevelResponse != null) {
// filter.put("ageLevel", ageLevelResponse);
// }
// IAggregation ageLevelAggregation =
// aggregationFactoryService.getAgeLevelAggregation();
// Object ageLevelResponse =
// this.getResponseByIAggregation(ageLevelAggregation,
// preAggregationResult, aggMaps);
// if (ageLevelResponse != null) {
// filter.put("ageLevel", ageLevelResponse);
// }
// 2)获取性别的聚合结果
IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap);
Object genderResponse = this.getResponseByIAggregation(genderAggregation, preAggregationResult, aggMaps);
... ...