yoho-search / yoho-search-service · Commits

Go to a project

Authored by 胡古飞
Commit 9f835ab2caebbcacb8bee1d1570542565de1962d 1 parent 90e52a4e

fix SearchKeyWordService

Inline Side-by-side
Showing 1 changed file with 217 additions and 218 deletions
package com.yoho.search.service.service;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import javax.annotation.Resource;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.ZSetOperations;
import org.springframework.data.redis.core.ZSetOperations.TypedTuple;
import org.springframework.stereotype.Service;
import org.springframework.util.Assert;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.yoho.core.redis.YHRedisTemplate;
... ... @@ -11,22 +34,6 @@ import com.yoho.search.base.utils.RedisKeys;
import com.yoho.search.core.es.IElasticsearchClient;
import com.yoho.search.core.es.impl.YohoIndexHelper;
import com.yoho.search.service.vo.KeyWordWithCount;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.ZSetOperations;
import org.springframework.data.redis.core.ZSetOperations.TypedTuple;
import org.springframework.stereotype.Service;
import org.springframework.util.Assert;
import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* 将关键字结果存进redis中
... ... @@ -37,207 +44,199 @@ import java.util.concurrent.TimeUnit;
@Service
public class SearchKeyWordService {
private static final Logger logger = LoggerFactory.getLogger(SearchKeyWordService.class);
@Resource(name = "yhNoSyncZSetOperations")
private YHZSetOperations<String, String> yhNoSyncZSetOperations;
@Resource(name = "yhNoSyncRedisTemplate")
private YHRedisTemplate<String, String> yhNoSyncRedisTemplate;
@Autowired
private ESClientMgr esClientMgr;
@Autowired
private YohoIndexHelper yohoIndexHelper;
private ExecutorService service = Executors.newFixedThreadPool(5);
// 保存rediskey中的日期
private volatile String dateForRedisKey = null;
public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {
List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();
final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
IElasticsearchClient client = esClientMgr.getClient(yohoIndexName);
List<String> realIndexNames = yohoIndexHelper.getRealIndexNames(yohoIndexName, client);
if (realIndexNames == null || realIndexNames.isEmpty() || realIndexNames.size() > 1) {
return analyzeTokens;
}
return client.getAnalyzeResponse(yohoIndexName, text, analyzer).getTokens();
}
/**
* 获取分词结果
*
* @param keyWord
* @return
*/
public List<String> getAnalyzeTerms(final String keyWord, final String analyzer, boolean useCache) {
if (!useCache) {
return getAnalyzeTermsDirect(keyWord, analyzer);
}
try {
return CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build(new CacheLoader<String, List<String>>() {
@Override
public List<String> load(String cacheKey) throws Exception {
String[] arrays = cacheKey.split("@", 2);
Assert.isTrue(arrays != null && arrays.length == 2);
return getAnalyzeTermsDirect(arrays[1], arrays[0]);
}
}).get(analyzer + "@" + keyWord);
} catch (ExecutionException e) {
logger.error(keyWord, e);
return new ArrayList<>();
}
}
public List<String> getAnalyzeTermsDirect(String keyWord, String analyzer) {
try {
if (StringUtils.isEmpty(keyWord)) {
return new ArrayList<>();
}
List<AnalyzeToken> tokens = getAnalyzeTokens(keyWord, analyzer);
List<String> results = new ArrayList<String>();
for (AnalyzeToken analyzeToken : tokens) {
results.add(analyzeToken.getTerm());
}
return results;
} catch (Exception e) {
logger.error(keyWord, e);
return new ArrayList<>();
}
}
// 异步的做法是防止redis报错影响搜索主流程
private void recordKeyWord(String redisKeyTemplate, String queryWord) {
service.submit(new Runnable() {
@Override
public void run() {
try {
// 按照当前时间和rediKey格式生成真正的redisKey
// 如果是第一次生成的话 给redis设置30个小时的失效时间
String currentDate = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
String redisKey = String.format(redisKeyTemplate, currentDate);
boolean hasIncreased = false;
if (!currentDate.equals(dateForRedisKey)) {
synchronized (this) {
if (!currentDate.equals(dateForRedisKey)) {
dateForRedisKey = currentDate;
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
yhNoSyncRedisTemplate.longExpire(redisKey, 48L, TimeUnit.HOURS);
hasIncreased = true;
logger.info("update new redis key date {}.", dateForRedisKey);
}
}
}
if (!hasIncreased) {
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
}
} catch (Exception e) {
logger.error(queryWord + "/" + redisKeyTemplate, e);
}
}
});
}
/**
* 增加热搜词的次数
*
* @param queryWord
*/
public void recordKeyWord(String queryWord) {
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, queryWord);
}
/**
* 根据【搜索结果数】记录搜索词
*
* @param queryWord
* @param total
*/
public void recordKeyWordByResultCount(String queryWord, long total) {
// 1、如果搜索结果为0，则加入【空结果列表】
if (total == 0) {
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);
return;
}
// 2、如果搜索结果为小于20，则加入【结果<20个的结果集】
if (total <= 20) {
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, queryWord);
return;
}
}
public void recordSearchRecommendCase(String caseType) {
recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, caseType);
}
public Double getKeywordCount(String redisKeyTemplate, String queryWord) {
try {
return yhNoSyncZSetOperations.score(RedisKeys.getRedisKey4Yesterday(redisKeyTemplate), queryWord);
} catch (Exception e) {
return null;
}
}
// 获取【热搜】toplist
public Map<String, Object> getHotkeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, dateStr);
}
// 获取空结果的toplist
public Map<String, Object> getEmptyKeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit, dateStr);
}
// 获取只有一页结果的toplist
public Map<String, Object> getLessKeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit, dateStr);
}
// 获取需要搜索推荐的关键词
public Map<String, Object> getNeedRecomKeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, limit, dateStr);
}
private Map<String, Object> getListByScoreDesc(String redisKeyTemplate, int limit, String dateStr) {
Map<String, Object> resultMap = new HashMap<>(3);
String date = dateStr;
if (StringUtils.isEmpty(date)) {
date = this.dateForRedisKey;
}
if (StringUtils.isEmpty(date)) {
date = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
}
String redisKey = String.format(redisKeyTemplate, date);
Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);
List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();
for (TypedTuple<String> typedTuple : redisResults) {
results.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
}
resultMap.put("redisKey", redisKey);
resultMap.put("keywords", results);
return resultMap;
}
public String deleteRedisKey(String redisKey) {
if (yhNoSyncRedisTemplate.hasKey(redisKey)) {
yhNoSyncRedisTemplate.delete(redisKey);
return "The key has been deleted succede!";
} else {
return "The key doesn't exist.";
}
}
private static final Logger logger = LoggerFactory.getLogger(SearchKeyWordService.class);
@Resource(name = "yhNoSyncZSetOperations")
private YHZSetOperations<String, String> yhNoSyncZSetOperations;
@Resource(name = "yhNoSyncRedisTemplate")
private YHRedisTemplate<String, String> yhNoSyncRedisTemplate;
@Autowired
private ESClientMgr esClientMgr;
@Autowired
private YohoIndexHelper yohoIndexHelper;
private ExecutorService service = Executors.newFixedThreadPool(5);
// 保存rediskey中的日期
private volatile String dateForRedisKey = null;
public List<AnalyzeToken> getAnalyzeTokens(String text, String analyzer) {
List<AnalyzeToken> analyzeTokens = new ArrayList<AnalyzeToken>();
final String yohoIndexName = ISearchConstants.INDEX_NAME_PRODUCT_INDEX;
IElasticsearchClient client = esClientMgr.getClient(yohoIndexName);
List<String> realIndexNames = yohoIndexHelper.getRealIndexNames(yohoIndexName, client);
if (realIndexNames == null || realIndexNames.isEmpty() || realIndexNames.size() > 1) {
return analyzeTokens;
}
return client.getAnalyzeResponse(yohoIndexName, text, analyzer).getTokens();
}
/**
* 获取分词结果
*
* @param keyWord
* @return
*/
public List<String> getAnalyzeTerms(final String keyWord, final String analyzer, boolean useCache) {
if (!useCache) {
return getAnalyzeTermsDirect(keyWord, analyzer);
}
try {
return CacheBuilder.newBuilder().maximumSize(10000).expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<String, List<String>>() {
@Override
public List<String> load(String cacheKey){
String[] arrays = cacheKey.split("@", 2);
Assert.isTrue(arrays != null && arrays.length == 2);
return getAnalyzeTermsDirect(arrays[1], arrays[0]);
}
}).get(analyzer + "@" + keyWord);
} catch (ExecutionException e) {
logger.error(keyWord, e);
return new ArrayList<>();
}
}
public List<String> getAnalyzeTermsDirect(String keyWord, String analyzer) {
try {
if (StringUtils.isEmpty(keyWord)) {
return new ArrayList<>();
}
List<AnalyzeToken> tokens = getAnalyzeTokens(keyWord, analyzer);
List<String> results = new ArrayList<String>();
for (AnalyzeToken analyzeToken : tokens) {
results.add(analyzeToken.getTerm());
}
return results;
} catch (Exception e) {
logger.error(keyWord, e);
return new ArrayList<>();
}
}
// 异步的做法是防止redis报错影响搜索主流程
private void recordKeyWord(String redisKeyTemplate, String queryWord) {
service.submit(new Runnable() {
@Override
public void run() {
try {
// 按照当前时间和rediKey格式生成真正的redisKey
// 如果是第一次生成的话 给redis设置30个小时的失效时间
String currentDate = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
String redisKey = String.format(redisKeyTemplate, currentDate);
boolean hasIncreased = false;
if (!currentDate.equals(dateForRedisKey)) {
synchronized (this) {
if (!currentDate.equals(dateForRedisKey)) {
dateForRedisKey = currentDate;
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
yhNoSyncRedisTemplate.longExpire(redisKey, 48L, TimeUnit.HOURS);
hasIncreased = true;
logger.info("update new redis key date {}.", dateForRedisKey);
}
}
}
if (!hasIncreased) {
yhNoSyncZSetOperations.incrementScore(redisKey, queryWord, 1);
}
} catch (Exception e) {
logger.error(queryWord + "/" + redisKeyTemplate, e);
}
}
});
}
/**
* 增加热搜词的次数
*
* @param queryWord
*/
public void recordKeyWord(String queryWord) {
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, queryWord);
}
/**
* 根据【搜索结果数】记录搜索词
*
* @param queryWord
* @param total
*/
public void recordKeyWordByResultCount(String queryWord, long total) {
// 1、如果搜索结果为0，则加入【空结果列表】
if (total == 0) {
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, queryWord);
return;
}
// 2、如果搜索结果为小于20，则加入【结果<20个的结果集】
if (total <= 20) {
this.recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, queryWord);
return;
}
}
public void recordSearchRecommendCase(String caseType) {
recordKeyWord(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, caseType);
}
public Double getKeywordCount(String redisKeyTemplate, String queryWord) {
try {
return yhNoSyncZSetOperations.score(RedisKeys.getRedisKey4Yesterday(redisKeyTemplate), queryWord);
} catch (Exception e) {
return null;
}
}
// 获取【热搜】toplist
public Map<String, Object> getHotkeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_HOT, limit, dateStr);
}
// 获取空结果的toplist
public Map<String, Object> getEmptyKeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY, limit, dateStr);
}
// 获取只有一页结果的toplist
public Map<String, Object> getLessKeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_LESS, limit, dateStr);
}
// 获取需要搜索推荐的关键词
public Map<String, Object> getNeedRecomKeyWords(int limit, String dateStr) {
return this.getListByScoreDesc(RedisKeys.YOHO_SEARCH_KEYWORDS_TIPS, limit, dateStr);
}
private Map<String, Object> getListByScoreDesc(String redisKeyTemplate, int limit, String dateStr) {
Map<String, Object> resultMap = new HashMap<>(3);
String date = dateStr;
if (StringUtils.isEmpty(date)) {
date = this.dateForRedisKey;
}
if (StringUtils.isEmpty(date)) {
date = DateUtil.DateToString(new Date(), DateStyle.YYYYMMDD);
}
String redisKey = String.format(redisKeyTemplate, date);
Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(redisKey, 0, limit);
List<KeyWordWithCount> results = new ArrayList<KeyWordWithCount>();
for (TypedTuple<String> typedTuple : redisResults) {
results.add(new KeyWordWithCount(typedTuple.getValue(), (int) typedTuple.getScore().doubleValue()));
}
resultMap.put("redisKey", redisKey);
resultMap.put("keywords", results);
return resultMap;
}
public String deleteRedisKey(String redisKey) {
if (yhNoSyncRedisTemplate.hasKey(redisKey)) {
yhNoSyncRedisTemplate.delete(redisKey);
return "The key has been deleted succede!";
} else {
return "The key doesn't exist.";
}
}
}
... ...