yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 84f7f6aea2a6d8c6cb3717562cf63471149cfd71 1 parent 2d4fa12a

添加service缓存

Inline Side-by-side
Showing 3 changed files with 86 additions and 45 deletions
@@ -9,6 +9,8 @@ public class CacheObject { @@ -9,6 +9,8 @@ public class CacheObject {
9 super(); 9 super();
10 this.object = object; 10 this.object = object;
11 this.expireTime = System.currentTimeMillis() + expireInSecond * 1000L; 11 this.expireTime = System.currentTimeMillis() + expireInSecond * 1000L;
  12 + System.out.println(System.currentTimeMillis());
  13 + System.out.println(expireTime);
12 } 14 }
13 15
14 public Object getObject() { 16 public Object getObject() {
@@ -19,4 +21,11 @@ public class CacheObject { @@ -19,4 +21,11 @@ public class CacheObject {
19 return expireTime; 21 return expireTime;
20 } 22 }
21 23
  24 +
  25 + public static void main(String[] args) {
  26 + long current = System.currentTimeMillis();
  27 + System.out.println(current);
  28 + System.out.println(current + 3000 * 60);
  29 + }
  30 +
22 } 31 }
1 package com.yoho.search.cache; 1 package com.yoho.search.cache;
2 2
  3 +import java.util.List;
3 import java.util.Map; 4 import java.util.Map;
  5 +import java.util.Set;
4 import java.util.concurrent.ConcurrentHashMap; 6 import java.util.concurrent.ConcurrentHashMap;
5 import java.util.concurrent.Executors; 7 import java.util.concurrent.Executors;
6 import java.util.concurrent.ScheduledExecutorService; 8 import java.util.concurrent.ScheduledExecutorService;
@@ -14,6 +16,7 @@ import org.slf4j.LoggerFactory; @@ -14,6 +16,7 @@ import org.slf4j.LoggerFactory;
14 import org.springframework.stereotype.Service; 16 import org.springframework.stereotype.Service;
15 17
16 import com.yoho.search.dal.model.SearchParam; 18 import com.yoho.search.dal.model.SearchParam;
  19 +import com.yoho.search.dal.model.SearchResult;
17 import com.yoho.search.utils.MD5Util; 20 import com.yoho.search.utils.MD5Util;
18 import com.yoho.search.utils.SearchParamUtils; 21 import com.yoho.search.utils.SearchParamUtils;
19 22
@@ -26,9 +29,9 @@ public class CacheService { @@ -26,9 +29,9 @@ public class CacheService {
26 private CacheCount cacheCount = new CacheCount(); 29 private CacheCount cacheCount = new CacheCount();
27 private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2); 30 private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2);
28 31
29 - private static final int cacheTime = 3;//结果缓存的时间  
30 - private static final int logPeriod = 5;//记录缓存命中的时间  
31 - private static final int clearCachePeriod = 15;//清除整个缓存对象的时间 32 + private static final int cacheTimeInMinute = 3;//结果缓存的时间
  33 + private static final int logPeriodInMinute = 3;//记录缓存命中的时间
  34 + private static final int clearCachePeriodInMinute = 15;//清除整个缓存对象的时间
32 35
33 @PostConstruct 36 @PostConstruct
34 void init(){ 37 void init(){
@@ -36,10 +39,10 @@ public class CacheService { @@ -36,10 +39,10 @@ public class CacheService {
36 scheduledExecutorService.scheduleAtFixedRate(new Runnable() { 39 scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
37 @Override 40 @Override
38 public void run() { 41 public void run() {
39 - logger.info("do log cache matchPercent,logPeriod is [{}],[{}]",logPeriod,cacheCount); 42 + logger.info("do log cache matchPercent,logPeriodInMinute is [{}],[{}]",logPeriodInMinute,cacheCount);
40 cacheCount.clear(); 43 cacheCount.clear();
41 } 44 }
42 - }, logPeriod, logPeriod, TimeUnit.MINUTES); 45 + }, logPeriodInMinute, logPeriodInMinute, TimeUnit.MINUTES);
43 46
44 //每15分钟清除缓存对象，防止内存爆掉 47 //每15分钟清除缓存对象，防止内存爆掉
45 scheduledExecutorService.scheduleAtFixedRate(new Runnable() { 48 scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@@ -48,7 +51,7 @@ public class CacheService { @@ -48,7 +51,7 @@ public class CacheService {
48 logger.info("do clear cacheObject..."); 51 logger.info("do clear cacheObject...");
49 cache = new ConcurrentHashMap<String,CacheObject>(); 52 cache = new ConcurrentHashMap<String,CacheObject>();
50 } 53 }
51 - }, clearCachePeriod, clearCachePeriod, TimeUnit.MINUTES); 54 + }, clearCachePeriodInMinute, clearCachePeriodInMinute, TimeUnit.MINUTES);
52 } 55 }
53 56
54 private Object getObjectFromCache(String key){ 57 private Object getObjectFromCache(String key){
@@ -65,38 +68,59 @@ public class CacheService { @@ -65,38 +68,59 @@ public class CacheService {
65 return cacheObject.getObject(); 68 return cacheObject.getObject();
66 } 69 }
67 70
68 - private void addObjectToCache(String key,Object object,long expireInSecond){  
69 - CacheObject cacheObject = new CacheObject(object, expireInSecond); 71 + private void addObjectToCache(String key,Object object){
  72 + CacheObject cacheObject = new CacheObject(object, cacheTimeInMinute * 60);
70 cache.put(key, cacheObject); 73 cache.put(key, cacheObject);
71 } 74 }
72 75
73 - /**  
74 - * 根据SearchParam生成的ES报文  
75 - * @param searchParam  
76 - * @return  
77 - */  
78 - private String genSearchParamString(String indexName,SearchParam searchParam){ 76 + /***********************************for SearchParam and SearchResult*****************************************/
  77 + private String genSearchParamString(String indexName, SearchParam searchParam) {
79 StringBuilder sb = new StringBuilder(); 78 StringBuilder sb = new StringBuilder();
80 - //拼装索引名称 79 + // 拼装索引名称
81 sb.append("indexName:").append(indexName).append(";"); 80 sb.append("indexName:").append(indexName).append(";");
82 - //拼装搜索类型  
83 - sb.append("searchType:").append(searchParam.getSearchType()==null?"":searchParam.getSearchType().name()).append(";");  
84 - //拼装报文 81 + // 拼装搜索类型
  82 + sb.append("searchType:").append(searchParam.getSearchType() == null ? "" : searchParam.getSearchType().name()).append(";");
  83 + // 拼装报文
85 SearchSourceBuilder searchSourceBuilder = SearchParamUtils.genSearchSourceBuilderFromSearchParam(searchParam); 84 SearchSourceBuilder searchSourceBuilder = SearchParamUtils.genSearchSourceBuilderFromSearchParam(searchParam);
86 sb.append("searchSource:").append(searchSourceBuilder.toString()).append(";"); 85 sb.append("searchSource:").append(searchSourceBuilder.toString()).append(";");
87 - //打印拼装结果  
88 - //logger.info("cacheKey is [{}]",sb.toString()); 86 + // 打印拼装结果
  87 + // logger.info("cacheKey is [{}]",sb.toString());
89 return MD5Util.string2MD5(sb.toString()); 88 return MD5Util.string2MD5(sb.toString());
90 } 89 }
91 90
92 - public Object getObjectFromCache(String indexName,SearchParam searchParam){  
93 - String key = this.genSearchParamString(indexName,searchParam);  
94 - return this.getObjectFromCache(key); 91 + public SearchResult getSearchResultFromCache(String indexName, SearchParam searchParam) {
  92 + String key = this.genSearchParamString(indexName, searchParam);
  93 + return (SearchResult) this.getObjectFromCache(key);
95 } 94 }
96 95
97 - public void addObjectToCache(String indexName,SearchParam searchParam,Object object){  
98 - String key = this.genSearchParamString(indexName,searchParam);  
99 - this.addObjectToCache(key, object, cacheTime); 96 + public void addSearchResultToCache(String indexName, SearchParam searchParam, SearchResult result) {
  97 + String key = this.genSearchParamString(indexName, searchParam);
  98 + this.addObjectToCache(key, result);
  99 + }
  100 +
  101 + /***********************************for multiGet*****************************************/
  102 + private String genMultiGetParamString(String indexName, Set<String> idList, List<String> fields) {
  103 + StringBuilder sb = new StringBuilder();
  104 + // 拼装索引名称
  105 + sb.append("indexName:").append(indexName).append(";");
  106 + // 拼装搜索类型
  107 + sb.append("idList:").append(idList.toString()).append(";");
  108 + // 拼装报文
  109 + sb.append("fields:").append(fields==null?"":fields.toString()).append(";");
  110 + // 打印拼装结果
  111 + // logger.info("cacheKey is [{}]",sb.toString());
  112 + return MD5Util.string2MD5(sb.toString());
  113 + }
  114 +
  115 + @SuppressWarnings("unchecked")
  116 + public List<Map<String, Object>> getMultiGetResultFromCache(String indexName, Set<String> idList, List<String> fields) {
  117 + String key = this.genMultiGetParamString(indexName, idList, fields);
  118 + return (List<Map<String, Object>>) this.getObjectFromCache(key);
  119 + }
  120 +
  121 + public void addMultiGetResultResultToCache(String indexName, Set<String> idList, List<String> fields, List<Map<String, Object>> result) {
  122 + String key = this.genMultiGetParamString(indexName, idList, fields);
  123 + this.addObjectToCache(key, result);
100 } 124 }
101 125
102 } 126 }
@@ -2,6 +2,7 @@ package com.yoho.search.service; @@ -2,6 +2,7 @@ package com.yoho.search.service;
2 2
3 import java.util.ArrayList; 3 import java.util.ArrayList;
4 import java.util.HashMap; 4 import java.util.HashMap;
  5 +import java.util.HashSet;
5 import java.util.LinkedList; 6 import java.util.LinkedList;
6 import java.util.List; 7 import java.util.List;
7 import java.util.Map; 8 import java.util.Map;
@@ -27,6 +28,7 @@ public class SearchCommonService { @@ -27,6 +28,7 @@ public class SearchCommonService {
27 @Autowired 28 @Autowired
28 private CacheService cacheService; 29 private CacheService cacheService;
29 30
  31 +
30 /** 32 /**
31 * 通用的查询接口 33 * 通用的查询接口
32 * 34 *
@@ -36,9 +38,9 @@ public class SearchCommonService { @@ -36,9 +38,9 @@ public class SearchCommonService {
36 */ 38 */
37 public SearchResult doSearch(final String indexName, final SearchParam searchParam) { 39 public SearchResult doSearch(final String indexName, final SearchParam searchParam) {
38 // 1、先从缓存中取结果 40 // 1、先从缓存中取结果
39 - Object objectFromCache = cacheService.getObjectFromCache(indexName, searchParam);  
40 - if (objectFromCache != null) {  
41 - return (SearchResult) objectFromCache; 41 + SearchResult resultFromCache = cacheService.getSearchResultFromCache(indexName, searchParam);
  42 + if (resultFromCache != null) {
  43 + return resultFromCache;
42 } 44 }
43 // 2、取不到再从ES获取结果 45 // 2、取不到再从ES获取结果
44 SearchResult searchResult = null; 46 SearchResult searchResult = null;
@@ -52,7 +54,9 @@ public class SearchCommonService { @@ -52,7 +54,9 @@ public class SearchCommonService {
52 }, firstIndex); 54 }, firstIndex);
53 } 55 }
54 // 3、将ES中的缓存结果放进缓存中 56 // 3、将ES中的缓存结果放进缓存中
55 - cacheService.addObjectToCache(indexName, searchParam, searchResult); 57 + if (searchResult != null) {
  58 + cacheService.addSearchResultToCache(indexName, searchParam, searchResult);
  59 + }
56 return searchResult; 60 return searchResult;
57 } 61 }
58 62
@@ -89,40 +93,44 @@ public class SearchCommonService { @@ -89,40 +93,44 @@ public class SearchCommonService {
89 * @param fields 93 * @param fields
90 * @return 94 * @return
91 */ 95 */
92 - @SuppressWarnings({ "unchecked", "rawtypes" })  
93 public List<Map<String, Object>> doMultiGet(final String indexName, final String[] idList, final List<String> fields) { 96 public List<Map<String, Object>> doMultiGet(final String indexName, final String[] idList, final List<String> fields) {
94 List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(); 97 List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
95 if (idList == null || idList.length == 0) { 98 if (idList == null || idList.length == 0) {
96 return list; 99 return list;
97 } 100 }
98 - Index firstIndex = indexService.getIndex(indexName);  
99 - if (firstIndex != null) {  
100 - list = indexService.execute(new SearchCallback<List>() {  
101 - @Override  
102 - public List<Map<String, Object>> execute(IndexClient indexClient) {  
103 - return indexClient.multiGet(indexName, idList, fields);  
104 - }  
105 - }, firstIndex); 101 + Set<String> idSet = new HashSet<String>();
  102 + for (String id : idList) {
  103 + idSet.add(id);
106 } 104 }
107 - return list; 105 + return this.doMultiGet(indexName, idList, fields);
108 } 106 }
109 107
110 @SuppressWarnings({ "unchecked", "rawtypes" }) 108 @SuppressWarnings({ "unchecked", "rawtypes" })
111 public List<Map<String, Object>> doMultiGet(final String indexName, final Set<String> idList, final List<String> fields) throws Exception { 109 public List<Map<String, Object>> doMultiGet(final String indexName, final Set<String> idList, final List<String> fields) throws Exception {
112 - List<Map<String, Object>> list = new LinkedList<Map<String, Object>>();  
113 - if (idList==null || idList.size() == 0){  
114 - return list; 110 + List<Map<String, Object>> result = new LinkedList<Map<String, Object>>();
  111 + if (idList == null || idList.size() == 0) {
  112 + return result;
115 } 113 }
  114 + // 1、先从缓存中取结果
  115 + List<Map<String, Object>> resultFromCache = cacheService.getMultiGetResultFromCache(indexName, idList, fields);
  116 + if (resultFromCache != null) {
  117 + return resultFromCache;
  118 + }
  119 + // 2、先ES中批量获取结果
116 Index firstIndex = indexService.getIndex(indexName); 120 Index firstIndex = indexService.getIndex(indexName);
117 if (firstIndex != null) { 121 if (firstIndex != null) {
118 - list = indexService.execute(new SearchCallback<List>() { 122 + result = indexService.execute(new SearchCallback<List>() {
119 @Override 123 @Override
120 public List<Map<String, Object>> execute(IndexClient indexClient) { 124 public List<Map<String, Object>> execute(IndexClient indexClient) {
121 return indexClient.multiGet(indexName, idList, fields); 125 return indexClient.multiGet(indexName, idList, fields);
122 } 126 }
123 }, firstIndex); 127 }, firstIndex);
124 } 128 }
125 - return list; 129 + // 3、将搜索结果加入到缓存中
  130 + if (result != null) {
  131 + cacheService.addMultiGetResultResultToCache(indexName, idList,fields, result);
  132 + }
  133 + return result;
126 } 134 }
127 135
128 } 136 }