|
|
|
package com.yoho.search.service.service;
|
|
|
|
|
|
|
|
import com.yoho.core.redis.YHRedisTemplate;
|
|
|
|
import com.yoho.core.redis.YHZSetOperations;
|
|
|
|
import com.yoho.search.base.utils.DateStyle;
|
|
|
|
import com.yoho.search.base.utils.DateUtil;
|
|
|
|
import com.yoho.search.base.utils.ISearchConstants;
|
|
|
|
import com.yoho.search.core.es.IElasticsearchClient;
|
|
|
|
import com.yoho.search.core.es.impl.YohoIndexHelper;
|
|
|
|
import com.yoho.search.service.common.RedisKeys;
|
|
|
|
import com.yoho.search.base.utils.RedisKeys;
|
|
|
|
import com.yoho.search.service.vo.KeyWordWithCount;
|
|
|
|
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
|
|
|
|
import org.slf4j.Logger;
|
|
...
|
...
|
@@ -16,194 +19,197 @@ import org.springframework.stereotype.Service; |
|
|
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
import java.text.SimpleDateFormat;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.*;
|
|
|
|
import java.util.concurrent.ExecutorService;
|
|
|
|
import java.util.concurrent.Executors;
|
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 将关键字结果存进redis中
|
|
|
|
*
|
|
|
|
* @author hugufei
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
@Service
|
|
|
|
public class SearchKeyWordService {
|
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(SearchKeyWordService.class);
|
|
|
|
|
|
|
|
private static final Logger EMPTY_RESULT = LoggerFactory.getLogger("EMPTY_RESULT");
|
|
|
|
|
|
|
|
@Resource(name = "yhNoSyncZSetOperations")
|
|
|
|
private YHZSetOperations<String, String> yhNoSyncZSetOperations;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private ESClientMgr esClientMgr;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private YohoIndexHelper yohoIndexHelper;
|
|
|
|
|
|
|
|
private ExecutorService service = Executors.newFixedThreadPool(5);
|
|
|
|
|
|
|
|
public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {
|
|
|
|
List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();
|
|
|
|
final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
|
|
|
|
IElasticsearchClient client = esClientMgr.getClient(yohoIndexName);
|
|
|
|
List<String> realIndexNames = yohoIndexHelper.getRealIndexNames(yohoIndexName, client);
|
|
|
|
if (realIndexNames == null || realIndexNames.isEmpty() || realIndexNames.size() > 1) {
|
|
|
|
return analyzeTokens;
|
|
|
|
}
|
|
|
|
return client.getAnalyzeResponse(yohoIndexName, text, analyzer).getTokens();
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 获取分词结果
|
|
|
|
*
|
|
|
|
* @param keyWord
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
public List<String> getAnalyzeTerms(String keyWord, String analyzer) {
|
|
|
|
try {
|
|
|
|
List<AnalyzeToken> tokens = getAnalyzeTokens(keyWord, analyzer);
|
|
|
|
List<String> results = new ArrayList<String>();
|
|
|
|
for (AnalyzeToken analyzeToken : tokens) {
|
|
|
|
results.add(analyzeToken.getTerm());
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(keyWord, e);
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 获取分词结果
|
|
|
|
*
|
|
|
|
* @param keyWord
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
private List<String> getAnalyzeTokens(String keyWord) {
|
|
|
|
return getAnalyzeTerms(keyWord, "ik_complex");
|
|
|
|
}
|
|
|
|
|
|
|
|
// 异步的做法是防止redis报错影响搜索主流程
|
|
|
|
private void recordKeyWord(String redisKey, String queryWord) {
|
|
|
|
service.submit(new Runnable() {
|
|
|
|
@Override
|
|
|
|
public void run() {
|
|
|
|
try {
|
|
|
|
// 1、记录整词
|
|
|
|
String keyWord = queryWord;
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKey, keyWord, 1);
|
|
|
|
|
|
|
|
// 2、记录分词结果
|
|
|
|
String redisKeyWithToken = redisKey + ".TOEKNS";
|
|
|
|
List<String> tokens = getAnalyzeTokens(keyWord);
|
|
|
|
for (String token : tokens) {
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKeyWithToken, token, 1);
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(e.getMessage(), e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 增加热搜词的次数
|
|
|
|
*
|
|
|
|
* @param queryWord
|
|
|
|
*/
|
|
|
|
public void recordKeyWord(String queryWord) {
|
|
|
|
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, queryWord);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 根据【搜索结果数】记录搜索词
|
|
|
|
*
|
|
|
|
* @param queryWord
|
|
|
|
* @param total
|
|
|
|
*/
|
|
|
|
public void recordKeyWordByResultCount(String queryWord, long total) {
|
|
|
|
// 1、如果搜索结果为0，则加入【空结果列表】
|
|
|
|
if (total == 0) {
|
|
|
|
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
// 2、如果搜索结果为小于10，则加入【结果<10个的结果集】
|
|
|
|
if (total <= 10) {
|
|
|
|
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, queryWord);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取【热搜】toplist
|
|
|
|
public List<KeyWordWithCount> getHotkeyWords(int limit) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取【热搜】toplist
|
|
|
|
public List<KeyWordWithCount> getHotkeyWordTokens(int limit) {
|
|
|
|
//return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT_TOKENS, limit);
|
|
|
|
// token没什么用处 先不存储
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取空结果的toplist
|
|
|
|
public List<KeyWordWithCount> getEmptyKeyWords(int limit) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取空结果的toplist
|
|
|
|
public List<KeyWordWithCount> getEmptyKeyWordTokens(int limit) {
|
|
|
|
//return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY_TOKENS, limit);
|
|
|
|
// token没什么用处 先不存储
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取只有一页结果的toplist
|
|
|
|
public List<KeyWordWithCount> getLessKeyWords(int limit) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取只有一页结果的toplist
|
|
|
|
public List<KeyWordWithCount> getLessKeyWordTokens(int limit) {
|
|
|
|
//return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS_TOKENS, limit);
|
|
|
|
// token没什么用处 先不存储
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<KeyWordWithCount> getListByScoreDesc(String redisKey, int limit) {
|
|
|
|
Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);
|
|
|
|
List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();
|
|
|
|
for (TypedTuple<String> typedTuple : redisResults) {
|
|
|
|
String keyWord = typedTuple.getValue();
|
|
|
|
KeyWordWithCount result = new KeyWordWithCount(keyWord, (int) typedTuple.getScore().doubleValue());
|
|
|
|
results.add(result);
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
}
|
|
|
|
|
|
|
|
public void handleEmptyRecords(Map<String, String> paramMap) {
|
|
|
|
StringBuilder paramStringBuilder = new StringBuilder();
|
|
|
|
for (Map.Entry<String, String> entry : paramMap.entrySet()) {
|
|
|
|
paramStringBuilder.append("&").append(entry.getKey()).append("=").append(entry.getValue());
|
|
|
|
}
|
|
|
|
String paramString = paramStringBuilder.toString().replaceFirst("&", "");
|
|
|
|
EMPTY_RESULT.info("empty records for search.json: time[{}], param [{}]", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(System.currentTimeMillis()), paramString);
|
|
|
|
|
|
|
|
// 商品池或品类列表 为空，则告警
|
|
|
|
if (paramMap.get("filter_poolId") != null) {// ||
|
|
|
|
// paramMap.get("msort")!=null
|
|
|
|
// ||
|
|
|
|
// paramMap.get("misort")!=null
|
|
|
|
// ||
|
|
|
|
// paramMap.get("sort")!=null
|
|
|
|
// /TODO
|
|
|
|
}
|
|
|
|
}
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(SearchKeyWordService.class);
|
|
|
|
|
|
|
|
private static final Logger EMPTY_RESULT = LoggerFactory.getLogger("EMPTY_RESULT");
|
|
|
|
|
|
|
|
@Resource(name = "yhNoSyncZSetOperations")
|
|
|
|
private YHZSetOperations<String, String> yhNoSyncZSetOperations;
|
|
|
|
|
|
|
|
|
|
|
|
@Resource(name = "yhNoSyncRedisTemplate")
|
|
|
|
private YHRedisTemplate<String, String> yhNoSyncRedisTemplate;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private ESClientMgr esClientMgr;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private YohoIndexHelper yohoIndexHelper;
|
|
|
|
|
|
|
|
private ExecutorService service = Executors.newFixedThreadPool(5);
|
|
|
|
|
|
|
|
// 保存rediskey中的日期
|
|
|
|
private volatile String dateForRedisKey = null;
|
|
|
|
|
|
|
|
public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {
|
|
|
|
List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();
|
|
|
|
final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
|
|
|
|
IElasticsearchClient client = esClientMgr.getClient(yohoIndexName);
|
|
|
|
List<String> realIndexNames = yohoIndexHelper.getRealIndexNames(yohoIndexName, client);
|
|
|
|
if (realIndexNames == null || realIndexNames.isEmpty() || realIndexNames.size() > 1) {
|
|
|
|
return analyzeTokens;
|
|
|
|
}
|
|
|
|
return client.getAnalyzeResponse(yohoIndexName, text, analyzer).getTokens();
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 获取分词结果
|
|
|
|
*
|
|
|
|
* @param keyWord
|
|
|
|
* @return
|
|
|
|
*/
|
|
|
|
public List<String> getAnalyzeTerms(String keyWord, String analyzer) {
|
|
|
|
try {
|
|
|
|
List<AnalyzeToken> tokens = getAnalyzeTokens(keyWord, analyzer);
|
|
|
|
List<String> results = new ArrayList<String>();
|
|
|
|
for (AnalyzeToken analyzeToken : tokens) {
|
|
|
|
results.add(analyzeToken.getTerm());
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(keyWord, e);
|
|
|
|
return new ArrayList<>();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 异步的做法是防止redis报错影响搜索主流程
|
|
|
|
private void recordKeyWord(String redisKeyTemplate, String queryWord) {
|
|
|
|
service.submit(new Runnable() {
|
|
|
|
@Override
|
|
|
|
public void run() {
|
|
|
|
try {
|
|
|
|
// 按照当前时间和rediKey格式生成真正的redisKey
|
|
|
|
// 如果是第一次生成的话 给redis设置30个小时的失效时间
|
|
|
|
String currentDate = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
|
|
|
|
String redisKey = String.format(redisKeyTemplate, currentDate);
|
|
|
|
boolean hasIncreased = false;
|
|
|
|
if (!currentDate.equals(dateForRedisKey)) {
|
|
|
|
synchronized (this) {
|
|
|
|
if (!currentDate.equals(dateForRedisKey)) {
|
|
|
|
dateForRedisKey = currentDate;
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
|
|
|
|
yhNoSyncRedisTemplate.longExpire(redisKey, 30L, TimeUnit.HOURS);
|
|
|
|
hasIncreased = true;
|
|
|
|
logger.info("update new redis key date {}.", dateForRedisKey);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!hasIncreased) {
|
|
|
|
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
|
|
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(queryWord + "/" + redisKeyTemplate, e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 增加热搜词的次数
|
|
|
|
*
|
|
|
|
* @param queryWord
|
|
|
|
*/
|
|
|
|
public void recordKeyWord(String queryWord) {
|
|
|
|
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, queryWord);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 根据【搜索结果数】记录搜索词
|
|
|
|
*
|
|
|
|
* @param queryWord
|
|
|
|
* @param total
|
|
|
|
*/
|
|
|
|
public void recordKeyWordByResultCount(String queryWord, long total) {
|
|
|
|
// 1、如果搜索结果为0，则加入【空结果列表】
|
|
|
|
if (total == 0) {
|
|
|
|
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
// 2、如果搜索结果为小于10，则加入【结果<10个的结果集】
|
|
|
|
if (total <= 10) {
|
|
|
|
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, queryWord);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取【热搜】toplist
|
|
|
|
public Map<String, Object> getHotkeyWords(int limit, boolean isReturnTodayRecords) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, isReturnTodayRecords);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取空结果的toplist
|
|
|
|
public Map<String, Object> getEmptyKeyWords(int limit, boolean isReturnTodayRecords) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit, isReturnTodayRecords);
|
|
|
|
}
|
|
|
|
|
|
|
|
// 获取只有一页结果的toplist
|
|
|
|
public Map<String, Object> getLessKeyWords(int limit, boolean isReturnTodayRecords) {
|
|
|
|
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit, isReturnTodayRecords);
|
|
|
|
}
|
|
|
|
|
|
|
|
private Map<String, Object> getListByScoreDesc(String redisKeyTemplate, int limit, boolean isReturnTodayRecords) {
|
|
|
|
Map<String, Object> resultMap = new HashMap<>(3);
|
|
|
|
resultMap.put("dateForRedisKey", dateForRedisKey);
|
|
|
|
if (this.dateForRedisKey == null) {
|
|
|
|
return resultMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
String redisKey = RedisKeys.getRedisKey4Yesterday(redisKeyTemplate);
|
|
|
|
Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);
|
|
|
|
List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();
|
|
|
|
for (TypedTuple<String> typedTuple : redisResults) {
|
|
|
|
results.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (isReturnTodayRecords) {
|
|
|
|
// 也返回今天的数据
|
|
|
|
String redisKey4Today = RedisKeys.getRedisKey4Today(redisKeyTemplate);
|
|
|
|
Set<ZSetOperations.TypedTuple<String>> redisResults4Today = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey4Today, 0, limit);
|
|
|
|
List<KeyWordWithCount> results4Today = new ArrayList<KeyWordWithCount>();
|
|
|
|
for (TypedTuple<String> typedTuple : redisResults4Today) {
|
|
|
|
results4Today.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
|
|
|
|
}
|
|
|
|
|
|
|
|
resultMap.put("0", results4Today);
|
|
|
|
}
|
|
|
|
|
|
|
|
resultMap.put("redisKey", redisKey);
|
|
|
|
resultMap.put("-1", results);
|
|
|
|
return resultMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
public void handleEmptyRecords(Map<String, String> paramMap) {
|
|
|
|
StringBuilder paramStringBuilder = new StringBuilder();
|
|
|
|
for (Map.Entry<String, String> entry : paramMap.entrySet()) {
|
|
|
|
paramStringBuilder.append("&").append(entry.getKey()).append("=").append(entry.getValue());
|
|
|
|
}
|
|
|
|
String paramString = paramStringBuilder.toString().replaceFirst("&", "");
|
|
|
|
EMPTY_RESULT.info("empty records for search.json: time[{}], param [{}]", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(System.currentTimeMillis()), paramString);
|
|
|
|
|
|
|
|
// 商品池或品类列表 为空，则告警
|
|
|
|
if (paramMap.get("filter_poolId") != null) {// ||
|
|
|
|
// paramMap.get("msort")!=null
|
|
|
|
// ||
|
|
|
|
// paramMap.get("misort")!=null
|
|
|
|
// ||
|
|
|
|
// paramMap.get("sort")!=null
|
|
|
|
// /TODO
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
} |
...
|
...
|
|
