yoho-search / yoho-search-service · Commits

Go to a project

Authored by 胡古飞
Commit 9f835ab2caebbcacb8bee1d1570542565de1962d 1 parent 90e52a4e

fix SearchKeyWordService

Inline Side-by-side
Showing 1 changed file with 217 additions and 218 deletions
1 package com.yoho.search.service.service; 1 package com.yoho.search.service.service;
2 2
  3 +import java.util.ArrayList;
  4 +import java.util.Date;
  5 +import java.util.HashMap;
  6 +import java.util.List;
  7 +import java.util.Map;
  8 +import java.util.Set;
  9 +import java.util.concurrent.ExecutionException;
  10 +import java.util.concurrent.ExecutorService;
  11 +import java.util.concurrent.Executors;
  12 +import java.util.concurrent.TimeUnit;
  13 +
  14 +import javax.annotation.Resource;
  15 +
  16 +import org.apache.commons.lang.StringUtils;
  17 +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
  18 +import org.slf4j.Logger;
  19 +import org.slf4j.LoggerFactory;
  20 +import org.springframework.beans.factory.annotation.Autowired;
  21 +import org.springframework.data.redis.core.ZSetOperations;
  22 +import org.springframework.data.redis.core.ZSetOperations.TypedTuple;
  23 +import org.springframework.stereotype.Service;
  24 +import org.springframework.util.Assert;
  25 +
3 import com.google.common.cache.CacheBuilder; 26 import com.google.common.cache.CacheBuilder;
4 import com.google.common.cache.CacheLoader; 27 import com.google.common.cache.CacheLoader;
5 import com.yoho.core.redis.YHRedisTemplate; 28 import com.yoho.core.redis.YHRedisTemplate;
@@ -11,22 +34,6 @@ import com.yoho.search.base.utils.RedisKeys; @@ -11,22 +34,6 @@ import com.yoho.search.base.utils.RedisKeys;
11 import com.yoho.search.core.es.IElasticsearchClient; 34 import com.yoho.search.core.es.IElasticsearchClient;
12 import com.yoho.search.core.es.impl.YohoIndexHelper; 35 import com.yoho.search.core.es.impl.YohoIndexHelper;
13 import com.yoho.search.service.vo.KeyWordWithCount; 36 import com.yoho.search.service.vo.KeyWordWithCount;
14 -import org.apache.commons.lang.StringUtils;  
15 -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;  
16 -import org.slf4j.Logger;  
17 -import org.slf4j.LoggerFactory;  
18 -import org.springframework.beans.factory.annotation.Autowired;  
19 -import org.springframework.data.redis.core.ZSetOperations;  
20 -import org.springframework.data.redis.core.ZSetOperations.TypedTuple;  
21 -import org.springframework.stereotype.Service;  
22 -import org.springframework.util.Assert;  
23 -  
24 -import javax.annotation.Resource;  
25 -import java.util.*;  
26 -import java.util.concurrent.ExecutionException;  
27 -import java.util.concurrent.ExecutorService;  
28 -import java.util.concurrent.Executors;  
29 -import java.util.concurrent.TimeUnit;  
30 37
31 /** 38 /**
32 * 将关键字结果存进redis中 39 * 将关键字结果存进redis中
@@ -37,207 +44,199 @@ import java.util.concurrent.TimeUnit; @@ -37,207 +44,199 @@ import java.util.concurrent.TimeUnit;
37 @Service 44 @Service
38 public class SearchKeyWordService { 45 public class SearchKeyWordService {
39 46
40 - private static final Logger logger = LoggerFactory.getLogger(SearchKeyWordService.class);  
41 -  
42 - @Resource(name = "yhNoSyncZSetOperations")  
43 - private YHZSetOperations<String, String> yhNoSyncZSetOperations;  
44 -  
45 -  
46 - @Resource(name = "yhNoSyncRedisTemplate")  
47 - private YHRedisTemplate<String, String> yhNoSyncRedisTemplate;  
48 -  
49 - @Autowired  
50 - private ESClientMgr esClientMgr;  
51 -  
52 - @Autowired  
53 - private YohoIndexHelper yohoIndexHelper;  
54 -  
55 - private ExecutorService service = Executors.newFixedThreadPool(5);  
56 -  
57 - // 保存rediskey中的日期  
58 - private volatile String dateForRedisKey = null;  
59 -  
60 - public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {  
61 - List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();  
62 - final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;  
63 - IElasticsearchClient client = esClientMgr.getClient(yohoIndexName);  
64 - List<String> realIndexNames = yohoIndexHelper.getRealIndexNames(yohoIndexName, client);  
65 - if (realIndexNames == null || realIndexNames.isEmpty() || realIndexNames.size() > 1) {  
66 - return analyzeTokens;  
67 - }  
68 - return client.getAnalyzeResponse(yohoIndexName, text, analyzer).getTokens();  
69 - }  
70 -  
71 - /**  
72 - * 获取分词结果  
73 - *  
74 - * @param keyWord  
75 - * @return  
76 - */  
77 - public List<String> getAnalyzeTerms(final String keyWord, final String analyzer, boolean useCache) {  
78 - if (!useCache) {  
79 - return getAnalyzeTermsDirect(keyWord, analyzer);  
80 - }  
81 -  
82 - try {  
83 - return CacheBuilder.newBuilder()  
84 - .maximumSize(10000)  
85 - .expireAfterWrite(10, TimeUnit.MINUTES)  
86 - .build(new CacheLoader<String, List<String>>() {  
87 - @Override  
88 - public List<String> load(String cacheKey) throws Exception {  
89 - String[] arrays = cacheKey.split("@", 2);  
90 - Assert.isTrue(arrays != null && arrays.length == 2);  
91 - return getAnalyzeTermsDirect(arrays[1], arrays[0]);  
92 - }  
93 - }).get(analyzer + "@" + keyWord);  
94 - } catch (ExecutionException e) {  
95 - logger.error(keyWord, e);  
96 - return new ArrayList<>();  
97 - }  
98 - }  
99 -  
100 - public List<String> getAnalyzeTermsDirect(String keyWord, String analyzer) {  
101 - try {  
102 - if (StringUtils.isEmpty(keyWord)) {  
103 - return new ArrayList<>();  
104 - }  
105 -  
106 - List<AnalyzeToken> tokens = getAnalyzeTokens(keyWord, analyzer);  
107 - List<String> results = new ArrayList<String>();  
108 - for (AnalyzeToken analyzeToken : tokens) {  
109 - results.add(analyzeToken.getTerm());  
110 - }  
111 - return results;  
112 - } catch (Exception e) {  
113 - logger.error(keyWord, e);  
114 - return new ArrayList<>();  
115 - }  
116 - }  
117 -  
118 - // 异步的做法是防止redis报错影响搜索主流程  
119 - private void recordKeyWord(String redisKeyTemplate, String queryWord) {  
120 - service.submit(new Runnable() {  
121 - @Override  
122 - public void run() {  
123 - try {  
124 - // 按照当前时间和rediKey格式生成真正的redisKey  
125 - // 如果是第一次生成的话 给redis设置30个小时的失效时间  
126 - String currentDate = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);  
127 - String redisKey = String.format(redisKeyTemplate, currentDate);  
128 - boolean hasIncreased = false;  
129 - if (!currentDate.equals(dateForRedisKey)) {  
130 - synchronized (this) {  
131 - if (!currentDate.equals(dateForRedisKey)) {  
132 - dateForRedisKey = currentDate;  
133 - yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);  
134 - yhNoSyncRedisTemplate.longExpire(redisKey, 48L, TimeUnit.HOURS);  
135 - hasIncreased = true;  
136 - logger.info("update new redis key date {}.", dateForRedisKey);  
137 - }  
138 - }  
139 - }  
140 -  
141 - if (!hasIncreased) {  
142 - yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);  
143 - }  
144 - } catch (Exception e) {  
145 - logger.error(queryWord + "/" + redisKeyTemplate, e);  
146 - }  
147 - }  
148 - });  
149 - }  
150 -  
151 - /**  
152 - * 增加热搜词的次数  
153 - *  
154 - * @param queryWord  
155 - */  
156 - public void recordKeyWord(String queryWord) {  
157 - this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, queryWord);  
158 - }  
159 -  
160 - /**  
161 - * 根据【搜索结果数】记录搜索词  
162 - *  
163 - * @param queryWord  
164 - * @param total  
165 - */  
166 - public void recordKeyWordByResultCount(String queryWord, long total) {  
167 - // 1、如果搜索结果为0，则加入【空结果列表】  
168 - if (total == 0) {  
169 - this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);  
170 - return;  
171 - }  
172 - // 2、如果搜索结果为小于20，则加入【结果<20个的结果集】  
173 - if (total <= 20) {  
174 - this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, queryWord);  
175 - return;  
176 - }  
177 - }  
178 -  
179 - public void recordSearchRecommendCase(String caseType) {  
180 - recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, caseType);  
181 - }  
182 -  
183 - public Double getKeywordCount(String redisKeyTemplate, String queryWord) {  
184 - try {  
185 - return yhNoSyncZSetOperations.score(RedisKeys.getRedisKey4Yesterday(redisKeyTemplate), queryWord);  
186 - } catch (Exception e) {  
187 - return null;  
188 - }  
189 - }  
190 -  
191 - // 获取【热搜】toplist  
192 - public Map<String, Object> getHotkeyWords(int limit, String dateStr) {  
193 - return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, dateStr);  
194 - }  
195 -  
196 - // 获取空结果的toplist  
197 - public Map<String, Object> getEmptyKeyWords(int limit, String dateStr) {  
198 - return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit, dateStr);  
199 - }  
200 -  
201 - // 获取只有一页结果的toplist  
202 - public Map<String, Object> getLessKeyWords(int limit, String dateStr) {  
203 - return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit, dateStr);  
204 - }  
205 -  
206 - // 获取需要搜索推荐的关键词  
207 - public Map<String, Object> getNeedRecomKeyWords(int limit, String dateStr) {  
208 - return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, limit, dateStr);  
209 - }  
210 -  
211 - private Map<String, Object> getListByScoreDesc(String redisKeyTemplate, int limit, String dateStr) {  
212 - Map<String, Object> resultMap = new HashMap<>(3);  
213 -  
214 - String date = dateStr;  
215 - if (StringUtils.isEmpty(date)) {  
216 - date = this.dateForRedisKey;  
217 - }  
218 - if (StringUtils.isEmpty(date)) {  
219 - date = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);  
220 - }  
221 -  
222 - String redisKey = String.format(redisKeyTemplate, date);  
223 - Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);  
224 - List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();  
225 - for (TypedTuple<String> typedTuple : redisResults) {  
226 - results.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));  
227 - }  
228 -  
229 - resultMap.put("redisKey", redisKey);  
230 - resultMap.put("keywords", results);  
231 - return resultMap;  
232 - }  
233 -  
234 - public String deleteRedisKey(String redisKey) {  
235 - if (yhNoSyncRedisTemplate.hasKey(redisKey)) {  
236 - yhNoSyncRedisTemplate.delete(redisKey);  
237 - return "The key has been deleted succede!";  
238 - } else {  
239 - return "The key doesn't exist.";  
240 - }  
241 - } 47 + private static final Logger logger = LoggerFactory.getLogger(SearchKeyWordService.class);
  48 +
  49 + @Resource(name = "yhNoSyncZSetOperations")
  50 + private YHZSetOperations<String, String> yhNoSyncZSetOperations;
  51 + @Resource(name = "yhNoSyncRedisTemplate")
  52 + private YHRedisTemplate<String, String> yhNoSyncRedisTemplate;
  53 + @Autowired
  54 + private ESClientMgr esClientMgr;
  55 + @Autowired
  56 + private YohoIndexHelper yohoIndexHelper;
  57 +
  58 + private ExecutorService service = Executors.newFixedThreadPool(5);
  59 +
  60 + // 保存rediskey中的日期
  61 + private volatile String dateForRedisKey = null;
  62 +
  63 + public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {
  64 + List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();
  65 + final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
  66 + IElasticsearchClient client = esClientMgr.getClient(yohoIndexName);
  67 + List<String> realIndexNames = yohoIndexHelper.getRealIndexNames(yohoIndexName, client);
  68 + if (realIndexNames == null || realIndexNames.isEmpty() || realIndexNames.size() > 1) {
  69 + return analyzeTokens;
  70 + }
  71 + return client.getAnalyzeResponse(yohoIndexName, text, analyzer).getTokens();
  72 + }
  73 +
  74 + /**
  75 + * 获取分词结果
  76 + *
  77 + * @param keyWord
  78 + * @return
  79 + */
  80 + public List<String> getAnalyzeTerms(final String keyWord, final String analyzer, boolean useCache) {
  81 + if (!useCache) {
  82 + return getAnalyzeTermsDirect(keyWord, analyzer);
  83 + }
  84 + try {
  85 + return CacheBuilder.newBuilder().maximumSize(10000).expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<String, List<String>>() {
  86 + @Override
  87 + public List<String> load(String cacheKey){
  88 + String[] arrays = cacheKey.split("@", 2);
  89 + Assert.isTrue(arrays != null && arrays.length == 2);
  90 + return getAnalyzeTermsDirect(arrays[1], arrays[0]);
  91 + }
  92 + }).get(analyzer + "@" + keyWord);
  93 + } catch (ExecutionException e) {
  94 + logger.error(keyWord, e);
  95 + return new ArrayList<>();
  96 + }
  97 + }
  98 +
  99 + public List<String> getAnalyzeTermsDirect(String keyWord, String analyzer) {
  100 + try {
  101 + if (StringUtils.isEmpty(keyWord)) {
  102 + return new ArrayList<>();
  103 + }
  104 +
  105 + List<AnalyzeToken> tokens = getAnalyzeTokens(keyWord, analyzer);
  106 + List<String> results = new ArrayList<String>();
  107 + for (AnalyzeToken analyzeToken : tokens) {
  108 + results.add(analyzeToken.getTerm());
  109 + }
  110 + return results;
  111 + } catch (Exception e) {
  112 + logger.error(keyWord, e);
  113 + return new ArrayList<>();
  114 + }
  115 + }
  116 +
  117 + // 异步的做法是防止redis报错影响搜索主流程
  118 + private void recordKeyWord(String redisKeyTemplate, String queryWord) {
  119 + service.submit(new Runnable() {
  120 + @Override
  121 + public void run() {
  122 + try {
  123 + // 按照当前时间和rediKey格式生成真正的redisKey
  124 + // 如果是第一次生成的话 给redis设置30个小时的失效时间
  125 + String currentDate = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
  126 + String redisKey = String.format(redisKeyTemplate, currentDate);
  127 + boolean hasIncreased = false;
  128 + if (!currentDate.equals(dateForRedisKey)) {
  129 + synchronized (this) {
  130 + if (!currentDate.equals(dateForRedisKey)) {
  131 + dateForRedisKey = currentDate;
  132 + yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
  133 + yhNoSyncRedisTemplate.longExpire(redisKey, 48L, TimeUnit.HOURS);
  134 + hasIncreased = true;
  135 + logger.info("update new redis key date {}.", dateForRedisKey);
  136 + }
  137 + }
  138 + }
  139 +
  140 + if (!hasIncreased) {
  141 + yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
  142 + }
  143 + } catch (Exception e) {
  144 + logger.error(queryWord + "/" + redisKeyTemplate, e);
  145 + }
  146 + }
  147 + });
  148 + }
  149 +
  150 + /**
  151 + * 增加热搜词的次数
  152 + *
  153 + * @param queryWord
  154 + */
  155 + public void recordKeyWord(String queryWord) {
  156 + this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, queryWord);
  157 + }
  158 +
  159 + /**
  160 + * 根据【搜索结果数】记录搜索词
  161 + *
  162 + * @param queryWord
  163 + * @param total
  164 + */
  165 + public void recordKeyWordByResultCount(String queryWord, long total) {
  166 + // 1、如果搜索结果为0，则加入【空结果列表】
  167 + if (total == 0) {
  168 + this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);
  169 + return;
  170 + }
  171 + // 2、如果搜索结果为小于20，则加入【结果<20个的结果集】
  172 + if (total <= 20) {
  173 + this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, queryWord);
  174 + return;
  175 + }
  176 + }
  177 +
  178 + public void recordSearchRecommendCase(String caseType) {
  179 + recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, caseType);
  180 + }
  181 +
  182 + public Double getKeywordCount(String redisKeyTemplate, String queryWord) {
  183 + try {
  184 + return yhNoSyncZSetOperations.score(RedisKeys.getRedisKey4Yesterday(redisKeyTemplate), queryWord);
  185 + } catch (Exception e) {
  186 + return null;
  187 + }
  188 + }
  189 +
  190 + // 获取【热搜】toplist
  191 + public Map<String, Object> getHotkeyWords(int limit, String dateStr) {
  192 + return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, dateStr);
  193 + }
  194 +
  195 + // 获取空结果的toplist
  196 + public Map<String, Object> getEmptyKeyWords(int limit, String dateStr) {
  197 + return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit, dateStr);
  198 + }
  199 +
  200 + // 获取只有一页结果的toplist
  201 + public Map<String, Object> getLessKeyWords(int limit, String dateStr) {
  202 + return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit, dateStr);
  203 + }
  204 +
  205 + // 获取需要搜索推荐的关键词
  206 + public Map<String, Object> getNeedRecomKeyWords(int limit, String dateStr) {
  207 + return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, limit, dateStr);
  208 + }
  209 +
  210 + private Map<String, Object> getListByScoreDesc(String redisKeyTemplate, int limit, String dateStr) {
  211 + Map<String, Object> resultMap = new HashMap<>(3);
  212 +
  213 + String date = dateStr;
  214 + if (StringUtils.isEmpty(date)) {
  215 + date = this.dateForRedisKey;
  216 + }
  217 + if (StringUtils.isEmpty(date)) {
  218 + date = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
  219 + }
  220 +
  221 + String redisKey = String.format(redisKeyTemplate, date);
  222 + Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);
  223 + List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();
  224 + for (TypedTuple<String> typedTuple : redisResults) {
  225 + results.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
  226 + }
  227 +
  228 + resultMap.put("redisKey", redisKey);
  229 + resultMap.put("keywords", results);
  230 + return resultMap;
  231 + }
  232 +
  233 + public String deleteRedisKey(String redisKey) {
  234 + if (yhNoSyncRedisTemplate.hasKey(redisKey)) {
  235 + yhNoSyncRedisTemplate.delete(redisKey);
  236 + return "The key has been deleted succede!";
  237 + } else {
  238 + return "The key doesn't exist.";
  239 + }
  240 + }
242 241
243 } 242 }