|
|
|
package com.yoho.search.service.service;
|
|
|
|
|
|
|
|
import com.yoho.core.redis.YHRedisTemplate;
|
|
|
|
import com.yoho.core.redis.YHZSetOperations;
|
|
|
|
import com.yoho.search.base.utils.DateStyle;
|
|
|
|
import com.yoho.search.base.utils.DateUtil;
|
|
|
|
import com.yoho.search.base.utils.ISearchConstants;
|
|
|
|
import com.yoho.search.core.es.IElasticsearchClient;
|
|
|
|
import com.yoho.search.core.es.impl.YohoIndexHelper;
|
|
|
|
import com.yoho.search.service.common.RedisKeys;
|
|
|
|
import com.yoho.search.base.utils.RedisKeys;
|
|
|
|
import com.yoho.search.service.vo.KeyWordWithCount;
|
|
|
|
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
|
|
|
|
import org.slf4j.Logger;
|
|
...
|
...
|
@@ -16,18 +19,15 @@ import org.springframework.stereotype.Service; |
|
|
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
import java.text.SimpleDateFormat;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.*;
|
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
|
import java.util.concurrent.Executors;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 将关键字结果存进redis中
|
|
|
|
*
|
|
|
|
* @author hugufei
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
@Service
|
|
...
|
...
|
@@ -40,6 +40,10 @@ public class SearchKeyWordService { |
|
|
|
@Resource(name = "yhNoSyncZSetOperations")
|
|
|
|
private YHZSetOperations<String, String> yhNoSyncZSetOperations;
|
|
|
|
|
|
|
|
|
|
|
|
@Resource(name = "yhNoSyncRedisTemplate")
|
|
|
|
private YHRedisTemplate<String, String> yhNoSyncRedisTemplate;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private ESClientMgr esClientMgr;
|
|
|
|
|
|
...
|
...
|
@@ -48,6 +52,9 @@ public class SearchKeyWordService { |
|
|
|
|
|
|
|
private ExecutorService service = Executors.newFixedThreadPool(5);
|
|
|
|
|
|
|
|
// 保存rediskey中的日期
|
|
|
|
private volatile String dateForRedisKey = null;
|
|
|
|
|
|
|
|
public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {
|
|
|
|
List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();
|
|
|
|
final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
|
|
...
|
...
|
@@ -79,34 +86,34 @@ public class SearchKeyWordService { |
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 获取分词结果
|
|
|
|
*
|
|
|
|
* @param keyWord
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
private List<String> getAnalyzeTokens(String keyWord) {
|
|
|
|
return getAnalyzeTerms(keyWord, "ik_complex");
|
|
|
|
}
|
|
|
|
|
|
|
|
// 异步的做法是防止redis报错影响搜索主流程
|
|
|
|
private void recordKeyWord(String redisKey, String queryWord) {
|
|
|
|
private void recordKeyWord(String redisKeyTemplate, String queryWord) {
|
|
|
|
service.submit(new Runnable() {
|
|
|
|
@Override
|
|
|
|
public void run() {
|
|
|
|
try {
|
|
|
|
// 1、记录整词
|
|
|
|
String keyWord = queryWord;
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKey, keyWord, 1);
|
|
|
|
// 按照当前时间和rediKey格式生成真正的redisKey
|
|
|
|
// 如果是第一次生成的话 给redis设置30个小时的失效时间
|
|
|
|
String currentDate = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
|
|
|
|
String redisKey = String.format(redisKeyTemplate, currentDate);
|
|
|
|
boolean hasIncreased = false;
|
|
|
|
if (!currentDate.equals(dateForRedisKey)) {
|
|
|
|
synchronized (this) {
|
|
|
|
if (!currentDate.equals(dateForRedisKey)) {
|
|
|
|
dateForRedisKey = currentDate;
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
|
|
|
|
yhNoSyncRedisTemplate.longExpire(redisKey, 30L, TimeUnit.HOURS);
|
|
|
|
hasIncreased = true;
|
|
|
|
logger.info("update new redis key date {}.", dateForRedisKey);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 2、记录分词结果
|
|
|
|
String redisKeyWithToken = redisKey + ".TOEKNS";
|
|
|
|
List<String> tokens = getAnalyzeTokens(keyWord);
|
|
|
|
for (String token : tokens) {
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKeyWithToken, token, 1);
|
|
|
|
if (!hasIncreased) {
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(e.getMessage(), e);
|
|
|
|
logger.error(queryWord + "/" + redisKeyTemplate, e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
...
|
...
|
@@ -141,50 +148,49 @@ public class SearchKeyWordService { |
|
|
|
}
|
|
|
|
|
|
|
|
// 获取【热搜】toplist
|
|
|
|
public List<KeyWordWithCount> getHotkeyWords(int limit) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取【热搜】toplist
|
|
|
|
public List<KeyWordWithCount> getHotkeyWordTokens(int limit) {
|
|
|
|
//return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT_TOKENS, limit);
|
|
|
|
// token没什么用处 先不存储
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取空结果的toplist
|
|
|
|
public List<KeyWordWithCount> getEmptyKeyWords(int limit) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit);
|
|
|
|
public Map<String, Object> getHotkeyWords(int limit, boolean isReturnTodayRecords) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, isReturnTodayRecords);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取空结果的toplist
|
|
|
|
public List<KeyWordWithCount> getEmptyKeyWordTokens(int limit) {
|
|
|
|
//return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY_TOKENS, limit);
|
|
|
|
// token没什么用处 先不存储
|
|
|
|
return new ArrayList<>();
|
|
|
|
public Map<String, Object> getEmptyKeyWords(int limit, boolean isReturnTodayRecords) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit, isReturnTodayRecords);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取只有一页结果的toplist
|
|
|
|
public List<KeyWordWithCount> getLessKeyWords(int limit) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit);
|
|
|
|
public Map<String, Object> getLessKeyWords(int limit, boolean isReturnTodayRecords) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit, isReturnTodayRecords);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取只有一页结果的toplist
|
|
|
|
public List<KeyWordWithCount> getLessKeyWordTokens(int limit) {
|
|
|
|
//return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS_TOKENS, limit);
|
|
|
|
// token没什么用处 先不存储
|
|
|
|
return new ArrayList<>();
|
|
|
|
private Map<String, Object> getListByScoreDesc(String redisKeyTemplate, int limit, boolean isReturnTodayRecords) {
|
|
|
|
Map<String, Object> resultMap = new HashMap<>(3);
|
|
|
|
resultMap.put("dateForRedisKey", dateForRedisKey);
|
|
|
|
if (this.dateForRedisKey == null) {
|
|
|
|
return resultMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<KeyWordWithCount> getListByScoreDesc(String redisKey, int limit) {
|
|
|
|
String redisKey = RedisKeys.getRedisKey4Yesterday(redisKeyTemplate);
|
|
|
|
Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);
|
|
|
|
List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();
|
|
|
|
for (TypedTuple<String> typedTuple : redisResults) {
|
|
|
|
String keyWord = typedTuple.getValue();
|
|
|
|
KeyWordWithCount result = new KeyWordWithCount(keyWord, (int) typedTuple.getScore().doubleValue());
|
|
|
|
results.add(result);
|
|
|
|
results.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
|
|
|
|
if (isReturnTodayRecords) {
|
|
|
|
// 也返回今天的数据
|
|
|
|
String redisKey4Today = RedisKeys.getRedisKey4Today(redisKeyTemplate);
|
|
|
|
Set<ZSetOperations.TypedTuple<String>> redisResults4Today = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey4Today, 0, limit);
|
|
|
|
List<KeyWordWithCount> results4Today = new ArrayList<KeyWordWithCount>();
|
|
|
|
for (TypedTuple<String> typedTuple : redisResults4Today) {
|
|
|
|
results4Today.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
|
|
|
|
}
|
|
|
|
|
|
|
|
resultMap.put("0", results4Today);
|
|
|
|
}
|
|
|
|
|
|
|
|
resultMap.put("redisKey", redisKey);
|
|
|
|
resultMap.put("-1", results);
|
|
|
|
return resultMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
public void handleEmptyRecords(Map<String, String> paramMap) {
|
...
|
...
|
|
