yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 126cc2e398a546cd98b4a144f5856c72d40b6b69 1 parent 74dd4403

针对提前聚合做缓存

Inline Side-by-side
Showing 4 changed files with 217 additions and 24 deletions
  1 +package com.yoho.search.cache;
  2 +
  3 +import java.util.concurrent.atomic.AtomicLong;
  4 +
  5 +public class CacheCount {
  6 +
  7 + private AtomicLong totalCount = new AtomicLong(0);
  8 + private AtomicLong matchCount = new AtomicLong(0);
  9 +
  10 + public void clear() {
  11 + totalCount = new AtomicLong(0);
  12 + matchCount = new AtomicLong(0);
  13 + }
  14 +
  15 + public void incTotalCount() {
  16 + totalCount.incrementAndGet();
  17 + }
  18 +
  19 + public void incMatchCount() {
  20 + totalCount.incrementAndGet();
  21 + }
  22 +
  23 + public AtomicLong getTotalCount() {
  24 + return totalCount;
  25 + }
  26 +
  27 + public AtomicLong getMatchCount() {
  28 + return matchCount;
  29 + }
  30 +
  31 + public int getMatchPercent() {
  32 + long matchCnt = matchCount.longValue();
  33 + long totalCnt = totalCount.longValue();
  34 + if (totalCnt == 0) {
  35 + return 0;
  36 + }
  37 + return (int) (matchCnt * 100L / totalCnt);
  38 + }
  39 +
  40 + @Override
  41 + public String toString() {
  42 + return "CacheCount [totalCount=" + totalCount + ", matchCount=" + matchCount + ", matchPercent=" + getMatchPercent() + "]";
  43 + }
  44 +
  45 +}
  1 +package com.yoho.search.cache;
  2 +
  3 +public class CacheObject {
  4 +
  5 + private Object object;
  6 + private long expireTime;// 过期秒数
  7 +
  8 + public CacheObject(Object object,long expireInSecond) {
  9 + super();
  10 + this.object = object;
  11 + this.expireTime = System.currentTimeMillis() + expireInSecond * 1000L;
  12 + }
  13 +
  14 + public Object getObject() {
  15 + return object;
  16 + }
  17 +
  18 + public long getExpireTime() {
  19 + return expireTime;
  20 + }
  21 +
  22 +}
  1 +package com.yoho.search.cache;
  2 +
  3 +import java.util.Map;
  4 +import java.util.concurrent.ConcurrentHashMap;
  5 +import java.util.concurrent.Executors;
  6 +import java.util.concurrent.ScheduledExecutorService;
  7 +import java.util.concurrent.TimeUnit;
  8 +
  9 +import javax.annotation.PostConstruct;
  10 +
  11 +import org.elasticsearch.search.builder.SearchSourceBuilder;
  12 +import org.slf4j.Logger;
  13 +import org.slf4j.LoggerFactory;
  14 +import org.springframework.stereotype.Service;
  15 +
  16 +import com.yoho.search.dal.model.SearchParam;
  17 +import com.yoho.search.utils.MD5Util;
  18 +import com.yoho.search.utils.SearchParamUtils;
  19 +
  20 +@Service
  21 +public class CacheService {
  22 +
  23 + private static final Logger logger = LoggerFactory.getLogger(CacheService.class);
  24 +
  25 + private Map<String,CacheObject> cache = new ConcurrentHashMap<String,CacheObject>();
  26 + private CacheCount cacheCount = new CacheCount();
  27 + private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2);
  28 +
  29 + @PostConstruct
  30 + void init(){
  31 + //每5分钟记录下缓存命中率
  32 + scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
  33 + @Override
  34 + public void run() {
  35 + logger.info("do log cache matchPercent,[{}]",cacheCount);
  36 + cacheCount.clear();
  37 + }
  38 + }, 1, 5, TimeUnit.MINUTES);
  39 +
  40 + //每30分钟清除缓存对象，防止内存爆掉
  41 + scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
  42 + @Override
  43 + public void run() {
  44 + logger.info("do clear cacheObject...");
  45 + cache = new ConcurrentHashMap<String,CacheObject>();
  46 + }
  47 + }, 30, 30, TimeUnit.MINUTES);
  48 + }
  49 +
  50 + private Object getObjectFromCache(String key){
  51 + cacheCount.incTotalCount();
  52 + CacheObject cacheObject = cache.get(key);
  53 + if(cacheObject==null){
  54 + return null;
  55 + }
  56 + if(cacheObject.getExpireTime() < System.currentTimeMillis()){
  57 + cache.remove(key);
  58 + return null;
  59 + }
  60 + cacheCount.incMatchCount();
  61 + return cacheObject.getObject();
  62 + }
  63 +
  64 + private void addObjectToCache(String key,Object object,long expireInSecond){
  65 + CacheObject cacheObject = new CacheObject(object, expireInSecond);
  66 + cache.put(key, cacheObject);
  67 + }
  68 +
  69 + /**
  70 + * 根据SearchParam生成的ES报文
  71 + * @param searchParam
  72 + * @return
  73 + */
  74 + private String genSearchParamString(String indexName,SearchParam searchParam){
  75 + StringBuilder sb = new StringBuilder();
  76 + //拼装索引名称
  77 + sb.append("indexName:").append(indexName).append(";");
  78 + //拼装搜索类型
  79 + sb.append("searchType:").append(searchParam.getSearchType()==null?"":searchParam.getSearchType().name()).append(";");
  80 + //拼装报文
  81 + SearchSourceBuilder searchSourceBuilder = SearchParamUtils.genSearchSourceBuilderFromSearchParam(searchParam);
  82 + sb.append("searchSource:").append(searchSourceBuilder.toString()).append(";");
  83 + //打印拼装结果
  84 + logger.info("cacheKey is [{}]",sb.toString());
  85 + return MD5Util.string2MD5(sb.toString());
  86 + }
  87 +
  88 + public Object getObjectFromCache(String indexName,SearchParam searchParam){
  89 + String key = this.genSearchParamString(indexName,searchParam);
  90 + return this.getObjectFromCache(key);
  91 + }
  92 +
  93 + public void addObjectToCache(String indexName,SearchParam searchParam,Object object){
  94 + String key = this.genSearchParamString(indexName,searchParam);
  95 + this.addObjectToCache(key, object, 5 * 60);
  96 + }
  97 +
  98 +}
@@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service;
24 24
25 import com.yoho.search.aggregations.IAggregation; 25 import com.yoho.search.aggregations.IAggregation;
26 import com.yoho.search.aggregations.impls.AggregationFactoryService; 26 import com.yoho.search.aggregations.impls.AggregationFactoryService;
  27 +import com.yoho.search.cache.CacheService;
27 import com.yoho.search.dal.model.SearchParam; 28 import com.yoho.search.dal.model.SearchParam;
28 import com.yoho.search.dal.model.SearchResult; 29 import com.yoho.search.dal.model.SearchResult;
29 import com.yoho.search.utils.ISearchConstans; 30 import com.yoho.search.utils.ISearchConstans;
@@ -39,6 +40,8 @@ public class SearchProductsService { @@ -39,6 +40,8 @@ public class SearchProductsService {
39 private SearchCommonService searchCommonService; 40 private SearchCommonService searchCommonService;
40 @Autowired 41 @Autowired
41 private AggregationFactoryService aggregationFactoryService; 42 private AggregationFactoryService aggregationFactoryService;
  43 + @Autowired
  44 + private CacheService cacheService;
42 45
43 /** 46 /**
44 * 搜索接口[商品列表以及一堆聚合结果] 47 * 搜索接口[商品列表以及一堆聚合结果]
@@ -131,12 +134,17 @@ public class SearchProductsService { @@ -131,12 +134,17 @@ public class SearchProductsService {
131 private Map<String, Object> preAggregationSearch(SearchParam searchParam, Map<String, String> paramMap) throws Exception { 134 private Map<String, Object> preAggregationSearch(SearchParam searchParam, Map<String, String> paramMap) throws Exception {
132 Map<String, Object> preAggregationResult = new HashMap<String, Object>(); 135 Map<String, Object> preAggregationResult = new HashMap<String, Object>();
133 // 1)年龄层 136 // 1)年龄层
134 -// if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_AGELEVEL) && StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_AGELEVEL))) {  
135 -// IAggregation aggregation = aggregationFactoryService.getAgeLevelAggregation();  
136 -// searchParam.setAggregationBuilders(Arrays.asList(aggregation.getBuilder()));  
137 -// searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap, ISearchConstans.PARAM_SEARCH_AGELEVEL));  
138 -// preAggregationResult.put(aggregation.aggName(), doPreSearch(searchParam.clone(), paramMap));  
139 -// } 137 + // if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_AGELEVEL) &&
  138 + // StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_AGELEVEL)))
  139 + // {
  140 + // IAggregation aggregation =
  141 + // aggregationFactoryService.getAgeLevelAggregation();
  142 + // searchParam.setAggregationBuilders(Arrays.asList(aggregation.getBuilder()));
  143 + // searchParam.setFiter(searchServiceHelper.constructFilterBuilder(paramMap,
  144 + // ISearchConstans.PARAM_SEARCH_AGELEVEL));
  145 + // preAggregationResult.put(aggregation.aggName(),
  146 + // doPreSearch(searchParam.clone(), paramMap));
  147 + // }
140 // 2)性别 148 // 2)性别
141 if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_GENDER) && StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_GENDER))) { 149 if (paramMap.containsKey(ISearchConstans.PARAM_SEARCH_GENDER) && StringUtils.isNotBlank(paramMap.get(ISearchConstans.PARAM_SEARCH_GENDER))) {
142 IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap); 150 IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap);
@@ -176,7 +184,7 @@ public class SearchProductsService { @@ -176,7 +184,7 @@ public class SearchProductsService {
176 } 184 }
177 185
178 /** 186 /**
179 - * 预聚合，并获取聚合结果[因为filter的参数不同，包含了一定的业务逻辑] 187 + * 预聚合，并获取聚合结果[因为filter的参数不同，包含了一定的业务逻辑][针对这里做一下缓存]
180 * 188 *
181 * @param searchParam 189 * @param searchParam
182 * @param paramMap 190 * @param paramMap
@@ -192,40 +200,56 @@ public class SearchProductsService { @@ -192,40 +200,56 @@ public class SearchProductsService {
192 searchParam.setOffset(1); 200 searchParam.setOffset(1);
193 searchParam.setSize(1); 201 searchParam.setSize(1);
194 final String indexName = ISearchConstans.INDEX_NAME_PRODUCT_INDEX; 202 final String indexName = ISearchConstans.INDEX_NAME_PRODUCT_INDEX;
  203 +
  204 + // 从缓存中取聚合结果
  205 + Object objectFromCache = cacheService.getObjectFromCache(indexName, searchParam);
  206 + if (objectFromCache != null) {
  207 + return null;
  208 + }
195 SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam); 209 SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
196 Map<String, Aggregation> aggMaps = searchResult.getAggMaps(); 210 Map<String, Aggregation> aggMaps = searchResult.getAggMaps();
197 if (aggMaps == null) { 211 if (aggMaps == null) {
198 return null; 212 return null;
199 } 213 }
200 // 1)获取年龄层的聚合结果 214 // 1)获取年龄层的聚合结果
201 -// IAggregation ageLevelAggregation = aggregationFactoryService.getAgeLevelAggregation();  
202 -// if (aggMaps.containsKey(ageLevelAggregation.aggName())) {  
203 -// return ageLevelAggregation.getAggregationResponseMap(aggMaps);  
204 -// } 215 + // IAggregation ageLevelAggregation = aggregationFactoryService.getAgeLevelAggregation();
  216 + // if (aggMaps.containsKey(ageLevelAggregation.aggName())) {
  217 + // return ageLevelAggregation.getAggregationResponseMap(aggMaps);
  218 + // }
205 // 2)获取性别层的聚合结果 219 // 2)获取性别层的聚合结果
206 IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap); 220 IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap);
207 if (aggMaps.containsKey(genderAggregation.aggName())) { 221 if (aggMaps.containsKey(genderAggregation.aggName())) {
208 - return genderAggregation.getAggregationResponseMap(aggMaps); 222 + Object genderResult = genderAggregation.getAggregationResponseMap(aggMaps);
  223 + cacheService.addObjectToCache(indexName, searchParam, genderResult);
  224 + return genderResult;
209 } 225 }
210 // 3)获取价格层的聚合结果 226 // 3)获取价格层的聚合结果
211 IAggregation priceAggregation = aggregationFactoryService.getPriceAggregation(); 227 IAggregation priceAggregation = aggregationFactoryService.getPriceAggregation();
212 if (aggMaps.containsKey(priceAggregation.aggName())) { 228 if (aggMaps.containsKey(priceAggregation.aggName())) {
213 - return priceAggregation.getAggregationResponseMap(aggMaps); 229 + Object result = priceAggregation.getAggregationResponseMap(aggMaps);
  230 + cacheService.addObjectToCache(indexName, searchParam, result);
  231 + return result;
214 } 232 }
215 // 4)获取颜色层的聚合结果 233 // 4)获取颜色层的聚合结果
216 IAggregation collorAggregation = aggregationFactoryService.getColorAggregation(paramMap); 234 IAggregation collorAggregation = aggregationFactoryService.getColorAggregation(paramMap);
217 if (aggMaps.containsKey(collorAggregation.aggName())) { 235 if (aggMaps.containsKey(collorAggregation.aggName())) {
218 - return collorAggregation.getAggregationResponseMap(aggMaps); 236 + Object result = collorAggregation.getAggregationResponseMap(aggMaps);
  237 + cacheService.addObjectToCache(indexName, searchParam, result);
  238 + return result;
219 } 239 }
220 // 5)获取风格层面的聚合结果 240 // 5)获取风格层面的聚合结果
221 IAggregation styleAggregation = aggregationFactoryService.getStyleAggregation(paramMap); 241 IAggregation styleAggregation = aggregationFactoryService.getStyleAggregation(paramMap);
222 if (aggMaps.containsKey(styleAggregation.aggName())) { 242 if (aggMaps.containsKey(styleAggregation.aggName())) {
223 - return styleAggregation.getAggregationResponseMap(aggMaps); 243 + Object result = styleAggregation.getAggregationResponseMap(aggMaps);
  244 + cacheService.addObjectToCache(indexName, searchParam, result);
  245 + return result;
224 } 246 }
225 // 6)获取品牌层面的聚合结果 247 // 6)获取品牌层面的聚合结果
226 IAggregation brandaAggregation = aggregationFactoryService.getBrandAggregation(paramMap); 248 IAggregation brandaAggregation = aggregationFactoryService.getBrandAggregation(paramMap);
227 if (aggMaps.containsKey(brandaAggregation.aggName())) { 249 if (aggMaps.containsKey(brandaAggregation.aggName())) {
228 - return brandaAggregation.getAggregationResponseMap(aggMaps); 250 + Object result = brandaAggregation.getAggregationResponseMap(aggMaps);
  251 + cacheService.addObjectToCache(indexName, searchParam, result);
  252 + return result;
229 } 253 }
230 return null; 254 return null;
231 } 255 }
@@ -240,9 +264,10 @@ public class SearchProductsService { @@ -240,9 +264,10 @@ public class SearchProductsService {
240 private List<AbstractAggregationBuilder> getAllAggregations(SearchParam searchParam, Map<String, String> paramMap) { 264 private List<AbstractAggregationBuilder> getAllAggregations(SearchParam searchParam, Map<String, String> paramMap) {
241 List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>(); 265 List<AbstractAggregationBuilder> list = new ArrayList<AbstractAggregationBuilder>();
242 // 年龄层 266 // 年龄层
243 -// if (!paramMap.containsKey("ageLevel") || StringUtils.isBlank(paramMap.get("ageLevel"))) {  
244 -// list.add(aggregationFactoryService.getAgeLevelAggregation().getBuilder());  
245 -// } 267 + // if (!paramMap.containsKey("ageLevel") ||
  268 + // StringUtils.isBlank(paramMap.get("ageLevel"))) {
  269 + // list.add(aggregationFactoryService.getAgeLevelAggregation().getBuilder());
  270 + // }
246 // 价格 271 // 价格
247 if (!paramMap.containsKey("price") || StringUtils.isBlank(paramMap.get("price"))) { 272 if (!paramMap.containsKey("price") || StringUtils.isBlank(paramMap.get("price"))) {
248 list.add(aggregationFactoryService.getPriceAggregation().getBuilder()); 273 list.add(aggregationFactoryService.getPriceAggregation().getBuilder());
@@ -307,11 +332,14 @@ public class SearchProductsService { @@ -307,11 +332,14 @@ public class SearchProductsService {
307 private Map<String, Object> buildFilterResult(Map<String, Aggregation> aggMaps, Map<String, Object> preAggregationResult, Map<String, String> paramMap) throws Exception { 332 private Map<String, Object> buildFilterResult(Map<String, Aggregation> aggMaps, Map<String, Object> preAggregationResult, Map<String, String> paramMap) throws Exception {
308 Map<String, Object> filter = new HashMap<String, Object>(); 333 Map<String, Object> filter = new HashMap<String, Object>();
309 // 1)获取年龄的聚合结果 334 // 1)获取年龄的聚合结果
310 -// IAggregation ageLevelAggregation = aggregationFactoryService.getAgeLevelAggregation();  
311 -// Object ageLevelResponse = this.getResponseByIAggregation(ageLevelAggregation, preAggregationResult, aggMaps);  
312 -// if (ageLevelResponse != null) {  
313 -// filter.put("ageLevel", ageLevelResponse);  
314 -// } 335 + // IAggregation ageLevelAggregation =
  336 + // aggregationFactoryService.getAgeLevelAggregation();
  337 + // Object ageLevelResponse =
  338 + // this.getResponseByIAggregation(ageLevelAggregation,
  339 + // preAggregationResult, aggMaps);
  340 + // if (ageLevelResponse != null) {
  341 + // filter.put("ageLevel", ageLevelResponse);
  342 + // }
315 // 2)获取性别的聚合结果 343 // 2)获取性别的聚合结果
316 IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap); 344 IAggregation genderAggregation = aggregationFactoryService.getGenderAggregation(paramMap);
317 Object genderResponse = this.getResponseByIAggregation(genderAggregation, preAggregationResult, aggMaps); 345 Object genderResponse = this.getResponseByIAggregation(genderAggregation, preAggregationResult, aggMaps);