yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by Gino Zhang
Commit 261e01c103dbe39b411815411cf77290d60b852d 1 parent 59890d44

suggest优化： 1.去除支持老的suggestion_keywords表； 2.suggest的计数支持blk和app；

Inline Side-by-side
Showing 15 changed files with 373 additions and 449 deletions
... ... @@ -11,6 +11,10 @@ public class SuggestWordDef {
private Integer count;
private Integer countForBlk;
private Integer countForApp;
private Integer weight;
private Integer type;
... ... @@ -49,6 +53,22 @@ public class SuggestWordDef {
this.count = count;
}
public Integer getCountForBlk() {
return countForBlk;
}
public void setCountForBlk(Integer countForBlk) {
this.countForBlk = countForBlk;
}
public Integer getCountForApp() {
return countForApp;
}
public void setCountForApp(Integer countForApp) {
this.countForApp = countForApp;
}
public Integer getWeight() {
return weight;
}
... ...
... ... @@ -6,21 +6,25 @@
<result column="keyword" property="keyword" jdbcType="VARCHAR" />
<result column="weight" property="weight" jdbcType="INTEGER" />
<result column="count" property="count" jdbcType="INTEGER" />
<result column="count_for_blk" property="countForBlk" jdbcType="INTEGER" />
<result column="count_for_global" property="countForGlobal" jdbcType="INTEGER" />
<result column="type" property="type" jdbcType="INTEGER" />
<result column="status" property="status" jdbcType="INTEGER" />
</resultMap>
<sql id="Base_Column_List">
id, keyword, weight, count, type, status
id, keyword, weight, count, count_for_blk, count_for_global, type, status
</sql>
<insert id="insertBatch" parameterType="java.util.List" timeout="20000">
insert ignore into suggest_word_def (keyword, weight, count, type)
insert ignore into suggest_word_def (keyword, weight, count, count_for_blk, count_for_global, type)
values
<foreach collection="list" item="item" index="index"
separator=",">
(#{item.keyword, jdbcType=VARCHAR},
#{item.weight, jdbcType=INTEGER},
#{item.count, jdbcType=INTEGER},
#{item.countForBlk, jdbcType=INTEGER},
#{item.countForGlobal, jdbcType=INTEGER},
#{item.type, jdbcType=INTEGER})
</foreach>
</insert>
... ... @@ -33,6 +37,16 @@
when id = #{item.id,jdbcType=INTEGER} then #{item.count,jdbcType=INTEGER}
</foreach>
</trim>
<trim prefix="count_for_blk =case" suffix="end,">
<foreach collection="suggestWordDefList" item="item" index="index">
when id = #{item.id,jdbcType=INTEGER} then #{item.countForBlk,jdbcType=INTEGER}
</foreach>
</trim>
<trim prefix="count_for_global =case" suffix="end,">
<foreach collection="suggestWordDefList" item="item" index="index">
when id = #{item.id,jdbcType=INTEGER} then #{item.countForGlobal,jdbcType=INTEGER}
</foreach>
</trim>
<trim prefix="weight =case" suffix="end,">
<foreach collection="suggestWordDefList" item="item" index="index">
when id = #{item.id,jdbcType=INTEGER} then #{item.weight,jdbcType=DECIMAL}
... ...
package com.yoho.search.consumer.index.fullbuild;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.MD5Util;
import com.yoho.search.consumer.common.DynamicConfigService;
import com.yoho.search.consumer.index.common.IIndexBuilder;
import com.yoho.search.consumer.service.base.SuggestWordDefService;
import com.yoho.search.consumer.service.bo.SuggestIndexBO;
import com.yoho.search.consumer.suggests.common.KeywordType;
import com.yoho.search.consumer.suggests.common.SuggestionConstants;
import com.yoho.search.dal.model.SuggestWordDef;
@Component
public class SuggestExtendedIndexBuilder extends IIndexBuilder {
@Autowired
private SuggestWordDefService suggestWordDefService;
@Autowired
private DynamicConfigService dynamicConfigService;
@Override
public int getTotalCount() throws Exception {
return suggestWordDefService.selectTotalCount();
}
@Override
public List<?> getPageLists(int offset, int limit) throws Exception {
Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType))
.map(KeywordType::getType).collect(Collectors.toSet());
// 获取分页列表
List<SuggestWordDef> list = suggestWordDefService.selectPageList(offset, limit);
// 构建结果
List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>();
for (SuggestWordDef suggestWordDef : list) {
if (suggestWordDef.getCount() == null || suggestWordDef.getCount() == 0) {
continue;
}
if (!suggestWordDef.getStatus().equals(SuggestionConstants.VALID_STATUS) || !enabledKeywordTypes.contains(suggestWordDef.getType())) {
continue;
}
// 根据Type取权重
int weight = KeywordType.getWeightValueByType(suggestWordDef.getType());
results.add(new SuggestIndexBO(suggestWordDef.getKeyword(), suggestWordDef.getType(), weight, suggestWordDef.getCount()));
}
return results;
}
@Override
public String getId(Object object) {
return MD5Util.string2MD5(((SuggestIndexBO) object).getKeyword().trim().toLowerCase());
}
}
package com.yoho.search.consumer.index.fullbuild;
import java.util.ArrayList;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.MD5Util;
import com.yoho.search.consumer.common.DynamicConfigService;
import com.yoho.search.consumer.index.common.IIndexBuilder;
import com.yoho.search.consumer.service.base.SuggestionKeywordsService;
import com.yoho.search.consumer.service.base.SuggestWordDefService;
import com.yoho.search.consumer.service.bo.SuggestIndexBO;
import com.yoho.search.consumer.suggests.common.KeywordType;
import com.yoho.search.consumer.suggests.common.SuggestionConstants;
import com.yoho.search.dal.model.SuggestWordDef;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class SuggestIndexBuilder extends IIndexBuilder {
@Autowired
private SuggestionKeywordsService suggestionKeywordsService;
@Override
public int getTotalCount() throws Exception {
// return suggestionKeywordsService.count();
return 0;
}
@Override
public List<?> getPageLists(int offset, int limit) throws Exception {
// List<SuggestionKeywords> guggestionKeywordList =
// suggestionKeywordsService.getPageLists(offset, limit);
// List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>();
// for (SuggestionKeywords kw : guggestionKeywordList) {
// if (kw.getCount() == null || kw.getCount() == 0) {
// continue;
// }
// int type = KeywordType.Customized.getType();
// int weight = KeywordType.Customized.getWeightValue();
// results.add(new SuggestIndexBO(kw.getKeyword(), type, weight,
// kw.getCount()));
// }
return new ArrayList<SuggestIndexBO>();
}
@Override
public String getId(Object object) {
return MD5Util.string2MD5(((SuggestIndexBO) object).getKeyword().trim().toLowerCase());
}
@Autowired
private SuggestWordDefService suggestWordDefService;
@Autowired
private DynamicConfigService dynamicConfigService;
@Override
public int getTotalCount() throws Exception {
return suggestWordDefService.selectTotalCount();
}
@Override
public List<?> getPageLists(int offset, int limit) throws Exception {
Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType))
.map(KeywordType::getType).collect(Collectors.toSet());
// 获取分页列表
List<SuggestWordDef> list = suggestWordDefService.selectPageList(offset, limit);
// 构建结果
List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>();
for (SuggestWordDef suggestWordDef : list) {
if (!suggestWordDef.getStatus().equals(SuggestionConstants.VALID_STATUS) || !enabledKeywordTypes.contains(suggestWordDef.getType())) {
continue;
}
// 根据Type取权重
int weight = KeywordType.getWeightValueByType(suggestWordDef.getType());
results.add(new SuggestIndexBO(suggestWordDef.getKeyword(), suggestWordDef.getType(), weight, suggestWordDef.getCount(), suggestWordDef.getCountForApp(), suggestWordDef.getCountForBlk()));
}
return results;
}
@Override
public String getId(Object object) {
return MD5Util.string2MD5(((SuggestIndexBO) object).getKeyword().trim().toLowerCase());
}
}
... ...
... ... @@ -284,25 +284,6 @@ public class IndexController implements ApplicationEventPublisherAware {
}
}
@RequestMapping(value = "/index/suggestion/count")
@ResponseBody
public Map<String, Object> suggestionDetail(@RequestParam String keyword) {
try {
Integer countInES = keywordCounterService.countInES(keyword);
Map<String, Object> rtnMap = getResultMap(200, "success");
Map<String, Integer> data = new HashMap<>(10);
data.put("countInES", countInES);
rtnMap.put("data", data);
return rtnMap;
} catch (Exception e) {
logger.error(e.getMessage(), e);
Map<String, Object> rtnMap = new HashMap<String, Object>();
rtnMap.put("code", 400);
rtnMap.put("msg", e.getMessage());
return rtnMap;
}
}
@RequestMapping(value = "/index/generateRule")
@ResponseBody
public Map<String, Object> generateRule() {
... ...
package com.yoho.search.consumer.suggests.common;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.consumer.suggests.counter.CountUsage;
import com.yoho.search.core.es.model.SearchParam;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.*;
import java.util.List;
/**
* Created by ginozhang on 2017/1/3.
*/
public final class SuggestSearchParamBuilder {
public static SearchParam build(String keyword, CountUsage countUsage) {
// TODO: 暂时先从service那边拷贝相关代码 后续考虑抽取到core
SearchParam searchParam = new SearchParam();
searchParam.setQuery(getMultiMatchQueryBuilder(keyword));
searchParam.setFiter(getBoolQueryBuilder(keyword, countUsage));
searchParam.setSize(0);
return searchParam;
}
private static BoolQueryBuilder getBoolQueryBuilder(String keyword, CountUsage countUsage) {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
// 增加默认的过滤条件
switch (countUsage) {
case BLK:
boolFilter.must(QueryBuilders.termQuery("appType", 1)); //BLK商品
case APP:
break;
case PC:
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y")); // 非全球购
break;
default:
break;
}
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y")); // 非秒杀
boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(1)); // 有库存
boolFilter.mustNot(QueryBuilders.termQuery("attribute", "2")); // 非赠品
boolFilter.must(QueryBuilders.termQuery("status", "1")); // 上架商品
// 1对男女关键字做特殊处理
boolean cotainBoy = keyword.contains("男");
boolean cotainGirl = keyword.contains("女");
// 对性别做特殊处理
String gender = null;
if (cotainBoy && !cotainGirl) {
gender = "1,3";
} else if (!cotainBoy && cotainGirl) {
gender = "2,3";
} else if (cotainBoy && cotainGirl) {
gender = "1,2,3";
}
if (gender != null) {
int[] genders = ConvertUtils.stringToIntArray(gender, ",");
boolFilter.must(QueryBuilders.termsQuery("gender", genders));
}
return boolFilter;
}
private static QueryBuilder getMultiMatchQueryBuilder(String keyword) {
if (StringUtils.isEmpty(keyword)) {
return QueryBuilders.matchAllQuery();
}
MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keyword);
MultiMatchQueryBuilder.Type multiMatchQueryBuilderType = getMultiMatchQueryBuilderType();
if (multiMatchQueryBuilderType != null) {
queryBuilder.type(multiMatchQueryBuilderType);
}
setDefaultSearchField(queryBuilder);
if (ISearchConstants.SEARCH_OPERATOR.equalsIgnoreCase("or")) {
queryBuilder.operator(MatchQueryBuilder.Operator.OR);
queryBuilder.minimumShouldMatch(ISearchConstants.SEARCH_MINIMUM_SHOULD_MATCH);
} else {
queryBuilder.operator(MatchQueryBuilder.Operator.AND);
}
return queryBuilder;
}
private static MultiMatchQueryBuilder.Type getMultiMatchQueryBuilderType() {
String configMultiMatchQueryType = ISearchConstants.SEARCH_MULTIMATCHQUERY_TYPE;
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.BEST_FIELDS.name())) {
return MultiMatchQueryBuilder.Type.BEST_FIELDS;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.MOST_FIELDS.name())) {
return MultiMatchQueryBuilder.Type.MOST_FIELDS;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.CROSS_FIELDS.name())) {
return MultiMatchQueryBuilder.Type.CROSS_FIELDS;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE.name())) {
return MultiMatchQueryBuilder.Type.PHRASE;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE_PREFIX.name())) {
return MultiMatchQueryBuilder.Type.PHRASE_PREFIX;
}
return null;
}
private static void setDefaultSearchField(MultiMatchQueryBuilder queryBuilder) {
List<String> fields = ISearchConstants.SEARCH_DEFAULT_FIELD;
for (String field : fields) {
String[] fieldBoost = field.split("^");
if (fieldBoost.length == 2) {
queryBuilder.field(fieldBoost[0], Float.parseFloat(fieldBoost[1]));
} else if (fieldBoost.length == 1) {
queryBuilder.field(fieldBoost[0]);
}
}
}
}
... ...
... ... @@ -52,17 +52,32 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli
return true;
}
Map<String, Integer> countMap = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()));
int countMapSize = countMap != null ? countMap.size() : 0;
if (countMapSize == 0) {
logger.info("[{} business][pageNo={}][countMapSize={}]", flowName(), pageNo, countMapSize);
Map<String, Integer> countMapForApp = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()), CountUsage.APP);
int countMapSizeForApp = countMapForApp != null ? countMapForApp.size() : 0;
if (countMapSizeForApp == 0) {
logger.info("[{} business][pageNo={}][countMapSizeForApp={}]", flowName(), pageNo, countMapSizeForApp);
return false;
}
Map<String, Integer> filterMap = filter(pageNo, countMap);
logger.info("[{} business][pageNo={}][fetchWordSize={}][countMapSize={}][filterMapSize={}]", flowName(), pageNo, fetchSize, countMapSize, filterMap.size());
logger.trace("[{} business][pageNo={}][keywordMap={}][countMap={}][filterMap={}]", flowName(), pageNo, keywordMap, countMap, filterMap);
return persistence(keywordMap, filterMap);
Map<String, Integer> countMapForPC = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()), CountUsage.PC);
int countMapSizeForPC = countMapForPC != null ? countMapForPC.size() : 0;
if (countMapSizeForPC == 0) {
logger.info("[{} business][pageNo={}][countMapForPC={}]", flowName(), pageNo, countMapSizeForPC);
return false;
}
Map<String, Integer> countMapForBlk = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()), CountUsage.BLK);
int countMapSizeForBlk = countMapForBlk != null ? countMapForBlk.size() : 0;
if (countMapSizeForBlk == 0) {
logger.info("[{} business][pageNo={}][countMapSizeForBlk={}]", flowName(), pageNo, countMapSizeForBlk);
return false;
}
// countMapForApp 的过滤条件最宽松，使用该Map来进行filter
Map<String, Integer> filterMap = filter(pageNo, countMapForApp);
logger.info("[{} business][pageNo={}][fetchWordSize={}][countMapForApp={}][countMapForPC={}][countMapForBlk={}][filterMapSize={}]", flowName(), pageNo, fetchSize, countMapForApp, countMapForPC, countMapForBlk, filterMap.size());
logger.trace("[{} business][pageNo={}][keywordMap={}][countMapForApp={}][filterMap={}]", flowName(), pageNo, keywordMap, countMapForApp, filterMap);
return persistence(keywordMap, filterMap, countMapForApp, countMapForPC, countMapForBlk);
}
protected Map<String, Integer> filter(int pageNo, Map<String, Integer> countMap) {
... ... @@ -88,5 +103,5 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli
abstract Map<String, Object> getKeywordMap(int pageNo, int batchSize);
abstract boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap);
abstract boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap, Map<String, Integer> countMapForApp, Map<String, Integer> countMapForPC, Map<String, Integer> countMapForBlk);
}
... ...
package com.yoho.search.consumer.suggests.counter;
/**
* Created by ginozhang on 2017/1/3.
*/
public enum CountUsage {
PC, BLK, APP
}
... ...
package com.yoho.search.consumer.suggests.counter;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.consumer.index.common.IYohoIndexService;
import com.yoho.search.core.es.agg.IAggregation;
import com.yoho.search.consumer.suggests.common.SuggestSearchParamBuilder;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
... ... @@ -27,123 +23,17 @@ public class KeywordCounterService {
@Autowired
private IYohoIndexService yohoIndexService;
public Integer countInES(String keyword) {
Assert.notNull(keyword);
SearchParam searchParam = buildSearchParam(keyword);
SearchResult result = yohoIndexService.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
return result != null ? Integer.valueOf(String.valueOf(result.getTotal())) : null;
}
@SuppressWarnings("unchecked")
public Map<String, Integer> batchCountByAgg(IAggregation aggregation) {
SearchParam searchParam = buildSearchParam("");
searchParam.addAbstractAggregationBuilder(aggregation.getBuilder());
SearchResult searchResult = yohoIndexService.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult != null) {
return (Map<String, Integer>) aggregation.getAggregationResponseMap(searchResult.getAggMaps());
}
return null;
}
public Map<String, Integer> batchCount(List<String> keywords) {
public Map<String, Integer> batchCount(List<String> keywords, CountUsage countUsage) {
Assert.notNull(keywords);
Map<String, Integer> countResultMap = new LinkedHashMap<>();
List<SearchParam> searchParams = keywords.stream().map(keyword -> buildSearchParam(keyword)).collect(Collectors.toList());
List<SearchParam> searchParams = keywords.stream().map(keyword -> SuggestSearchParamBuilder.build(keyword, countUsage)).collect(Collectors.toList());
List<SearchResult> results = yohoIndexService.multiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams);
Assert.notNull(results);
Assert.isTrue(results.size() == keywords.size());
for (int i = 0; i < keywords.size(); i++) {
countResultMap.put(keywords.get(i), Integer.valueOf(String.valueOf(results.get(i).getTotal())));
}
return countResultMap;
}
private SearchParam buildSearchParam(String keyword) {
// TODO: 暂时先从service那边拷贝相关代码 后续考虑抽取到core
SearchParam searchParam = new SearchParam();
searchParam.setQuery(getMultiMatchQueryBuilder(keyword));
searchParam.setFiter(getBoolQueryBuilder(keyword));
searchParam.setSize(0);
return searchParam;
}
private BoolQueryBuilder getBoolQueryBuilder(String keyword) {
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
// 增加默认的过滤条件 TODO: 支持可配置
boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y")); // 非全球购
boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y")); // 非秒杀
boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(1)); // 有库存
boolFilter.mustNot(QueryBuilders.termQuery("attribute", "2")); // 非赠品
boolFilter.must(QueryBuilders.termQuery("status", "1")); // 上架商品
// 1对男女关键字做特殊处理
boolean cotainBoy = keyword.contains("男");
boolean cotainGirl = keyword.contains("女");
// 对性别做特殊处理
String gender = null;
if (cotainBoy && !cotainGirl) {
gender = "1,3";
} else if (!cotainBoy && cotainGirl) {
gender = "2,3";
} else if (cotainBoy && cotainGirl) {
gender = "1,2,3";
}
if (gender != null) {
int[] genders = ConvertUtils.stringToIntArray(gender, ",");
boolFilter.must(QueryBuilders.termsQuery("gender", genders));
}
return boolFilter;
}
private QueryBuilder getMultiMatchQueryBuilder(String keyword) {
if (StringUtils.isEmpty(keyword)) {
return QueryBuilders.matchAllQuery();
}
MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keyword);
MultiMatchQueryBuilder.Type multiMatchQueryBuilderType = this.getMultiMatchQueryBuilderType();
if (multiMatchQueryBuilderType != null) {
queryBuilder.type(multiMatchQueryBuilderType);
}
setDefaultSearchField(queryBuilder);
if (ISearchConstants.SEARCH_OPERATOR.equalsIgnoreCase("or")) {
queryBuilder.operator(MatchQueryBuilder.Operator.OR);
queryBuilder.minimumShouldMatch(ISearchConstants.SEARCH_MINIMUM_SHOULD_MATCH);
} else {
queryBuilder.operator(MatchQueryBuilder.Operator.AND);
}
return queryBuilder;
}
private MultiMatchQueryBuilder.Type getMultiMatchQueryBuilderType() {
String configMultiMatchQueryType = ISearchConstants.SEARCH_MULTIMATCHQUERY_TYPE;
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.BEST_FIELDS.name())) {
return MultiMatchQueryBuilder.Type.BEST_FIELDS;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.MOST_FIELDS.name())) {
return MultiMatchQueryBuilder.Type.MOST_FIELDS;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.CROSS_FIELDS.name())) {
return MultiMatchQueryBuilder.Type.CROSS_FIELDS;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE.name())) {
return MultiMatchQueryBuilder.Type.PHRASE;
}
if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE_PREFIX.name())) {
return MultiMatchQueryBuilder.Type.PHRASE_PREFIX;
}
return null;
}
private void setDefaultSearchField(MultiMatchQueryBuilder queryBuilder) {
List<String> fields = ISearchConstants.SEARCH_DEFAULT_FIELD;
for (String field : fields) {
String[] fieldBoost = field.split("^");
if (fieldBoost.length == 2) {
queryBuilder.field(fieldBoost[0], Float.parseFloat(fieldBoost[1]));
} else if (fieldBoost.length == 1) {
queryBuilder.field(fieldBoost[0]);
}
}
return countResultMap;
}
}
... ...
package com.yoho.search.consumer.suggests.counter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.consumer.common.DynamicConfigService;
import com.yoho.search.consumer.service.base.SuggestWordDefService;
import com.yoho.search.consumer.suggests.common.KeywordType;
import com.yoho.search.consumer.suggests.common.SuggestionConstants;
import com.yoho.search.dal.model.SuggestWordDef;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Created by ginozhang on 2016/11/25.
... ... @@ -24,42 +19,49 @@ import com.yoho.search.dal.model.SuggestWordDef;
@Component
public class SuggestWordDefCounter extends AbstractSuggestionCounter {
@Autowired
private SuggestWordDefService suggestWordDefService;
@Autowired
private DynamicConfigService dynamicConfigService;
@Autowired
private SuggestWordDefService suggestWordDefService;
@Autowired
private DynamicConfigService dynamicConfigService;
@Override
public int getTotalCount() {
return suggestWordDefService.selectTotalCount();
}
@Override
public int getTotalCount() {
return suggestWordDefService.selectTotalCount();
}
@Override
Map<String, Object> getKeywordMap(int pageNo, int batchSize) {
int start = (pageNo - 1) * batchSize;
Map<String, Object> wordMap = new HashMap<>(batchSize);
List<SuggestWordDef> dataList = suggestWordDefService.selectPageList(start, batchSize);
if (CollectionUtils.isNotEmpty(dataList)) {
Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType))
.map(KeywordType::getType).collect(Collectors.toSet());
dataList.stream().filter(item -> SuggestionConstants.VALID_STATUS.equals(item.getStatus())).filter(item -> enabledKeywordTypes.contains(item.getType()))
.forEach(item -> wordMap.put(item.getKeyword(), item));
}
return wordMap;
}
@Override
Map<String, Object> getKeywordMap(int pageNo, int batchSize) {
int start = (pageNo - 1) * batchSize;
Map<String, Object> wordMap = new HashMap<>(batchSize);
List<SuggestWordDef> dataList = suggestWordDefService.selectPageList(start, batchSize);
if (CollectionUtils.isNotEmpty(dataList)) {
Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType))
.map(KeywordType::getType).collect(Collectors.toSet());
dataList.stream().filter(item -> SuggestionConstants.VALID_STATUS.equals(item.getStatus())).filter(item -> enabledKeywordTypes.contains(item.getType()))
.forEach(item -> wordMap.put(item.getKeyword(), item));
}
return wordMap;
}
@Override
boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap, Map<String, Integer> countMapForApp, Map<String, Integer> countMapForPC, Map<String, Integer> countMapForBlk) {
List<SuggestWordDef> batchList = new ArrayList<>(countMap.size());
countMap.forEach((keyword, count) -> {
SuggestWordDef suggestWordDef = (SuggestWordDef) keywordMap.get(keyword);
if (suggestWordDef != null) {
suggestWordDef.setCount(calCount(countMapForPC, keyword));
suggestWordDef.setCountForApp(calCount(countMapForApp, keyword));
suggestWordDef.setCountForBlk(calCount(countMapForBlk, keyword));
suggestWordDef.setWeight(KeywordType.getWeightValueByType(suggestWordDef.getType()));
batchList.add(suggestWordDef);
}
});
suggestWordDefService.updateBatch(batchList);
return true;
}
@Override
boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap) {
List<SuggestWordDef> batchList = new ArrayList<>(countMap.size());
countMap.forEach((keyword, count) -> {
SuggestWordDef suggestWordDef = (SuggestWordDef) keywordMap.get(keyword);
if (suggestWordDef != null) {
suggestWordDef.setCount(count);
suggestWordDef.setWeight(KeywordType.getWeightValueByType(suggestWordDef.getType()));
batchList.add(suggestWordDef);
}
});
suggestWordDefService.updateBatch(batchList);
return true;
}
Integer calCount(Map<String, Integer> countResultMap, String suggestWord) {
Integer count = countResultMap.get(suggestWord);
return count != null ? count : Integer.valueOf(0);
}
}
... ...
package com.yoho.search.consumer.suggests.counter;
import com.yoho.search.consumer.service.base.SuggestionKeywordsService;
import com.yoho.search.dal.model.SuggestionKeywords;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* Created by ginozhang on 2016/11/25.
*/
@Component
public class SuggestionKeywordCounter extends AbstractSuggestionCounter {
private static final List<String> invalidChractors = Arrays.asList(">", ":", "?", "+", "：", "？", "！", "@", "#", "￥", "$", "%", "……", "^", "&", "*", ";", "；", "<", "、", "/", "\\", "|", "【", "】", "[", "]", "(", ")", "（", "）");
private Map<String, Integer> notNormalWordMap = new ConcurrentHashMap<>();
@Autowired
private SuggestionKeywordsService suggestionKeywordsService;
@Override
public int getTotalCount() {
return suggestionKeywordsService.count();
}
@Override
Map<String, Object> getKeywordMap(int pageNo, int batchSize) {
int start = (pageNo - 1) * batchSize;
Map<String, Object> wordMap = new HashMap<>(batchSize);
List<SuggestionKeywords> dataList = suggestionKeywordsService.getPageLists(start, batchSize);
if (CollectionUtils.isNotEmpty(dataList)) {
dataList.forEach((item) -> wordMap.put(item.getKeyword(), item));
}
return wordMap;
}
@Override
boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap) {
List<SuggestionKeywords> batchList = new ArrayList<>(countMap.size());
countMap.forEach((keyword, count) -> {
SuggestionKeywords suggestionKeywords = (SuggestionKeywords) keywordMap.get(keyword);
if (suggestionKeywords != null) {
suggestionKeywords.setCount(count);
batchList.add(suggestionKeywords);
}
});
suggestionKeywordsService.updateBatch(batchList);
return true;
}
@Override
protected Map<String, Integer> filter(int pageNo, Map<String, Integer> countMap) {
Map<String, Integer> resultMap = super.filter(pageNo, countMap);
// 检查一些不友好的推荐词 输出到日志中
if (logger.isInfoEnabled()) {
resultMap.forEach((key, value) -> {
if (notNormalKeyword(key)) {
synchronized (this) {
notNormalWordMap.put(key, value);
}
}
});
}
return resultMap;
}
private boolean notNormalKeyword(String key) {
//1. 全数字的
key = key.trim();
if (NumberUtils.isNumber(key)) {
return true;
}
//2. 包含特殊字段
for (String invalidChar : invalidChractors) {
if (key.contains(invalidChar)) {
return true;
}
}
// 3. 长度太长或太短
if (key.length() <= 1 && key.length() >= 15) {
return true;
}
return false;
}
@Override
public void init() {
super.init();
this.notNormalWordMap.clear();
}
@Override
public void finish(boolean doBusinessResult, Exception exception) {
super.finish(doBusinessResult, exception);
if (!notNormalWordMap.isEmpty() && logger.isInfoEnabled()) {
logger.info("[{} check]found not normal words: {}", flowName(), notNormalWordMap.keySet().stream().map(item -> "'" + item + "'").collect(Collectors.joining(",")));
notNormalWordMap.clear();
}
}
}
package com.yoho.search.consumer.suggests.discover;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.consumer.index.common.IYohoIndexService;
import com.yoho.search.consumer.suggests.common.SuggestSearchParamBuilder;
import com.yoho.search.consumer.suggests.counter.CountUsage;
import com.yoho.search.consumer.suggests.counter.KeywordCounterService;
import com.yoho.search.core.es.agg.IAggregation;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.HashSet;
... ... @@ -18,12 +24,15 @@ public abstract class AbstractAggSuggestionDiscoverer extends AbstractSuggestion
@Autowired
protected KeywordCounterService keywordCounterService;
@Autowired
private IYohoIndexService yohoIndexService;
protected volatile List<String> tokenList = null;
@Override
public int count() {
// TODO: 对于聚合类的暂时都一次性地从ES获取 后续可以考虑通过skn分页来支持多次获取
Map<String, Integer> countMap = keywordCounterService.batchCountByAgg(getAggregation());
Map<String, Integer> countMap = findSuggestionTokenByAgg(getAggregation());
if (countMap == null || countMap.isEmpty()) {
return 0;
}
... ... @@ -43,6 +52,16 @@ public abstract class AbstractAggSuggestionDiscoverer extends AbstractSuggestion
return tokenList.size();
}
private Map<String, Integer> findSuggestionTokenByAgg(IAggregation aggregation) {
SearchParam searchParam = SuggestSearchParamBuilder.build("", CountUsage.PC);
searchParam.addAbstractAggregationBuilder(aggregation.getBuilder());
SearchResult searchResult = yohoIndexService.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
if (searchResult != null) {
return (Map<String, Integer>) aggregation.getAggregationResponseMap(searchResult.getAggMaps());
}
return null;
}
@Override
public Set<String> getSuggestWordSet(int pageNo, int batchSize) {
Set<String> sortNameSet = new HashSet<>(batchSize);
... ...
... ... @@ -4,50 +4,69 @@ import java.io.Serializable;
public class SuggestIndexBO implements Serializable {
private static final long serialVersionUID = 7154651415633074270L;
private String keyword;
private int type;
private int weight;
private int count;
public SuggestIndexBO(String keyword, int type, int weight, int count) {
super();
this.keyword = keyword;
this.type = type;
this.weight = weight;
this.count = count;
}
public String getKeyword() {
return keyword;
}
public void setKeyword(String keyword) {
this.keyword = keyword;
}
public int getType() {
return type;
}
public void setType(int type) {
this.type = type;
}
public int getWeight() {
return weight;
}
public void setWeight(int weight) {
this.weight = weight;
}
public int getCount() {
return count;
}
public void setCount(int count) {
this.count = count;
}
private static final long serialVersionUID = 7154651415633074270L;
private String keyword;
private int type;
private int weight;
private int count;
private int countForApp;
private int countForBlk;
public SuggestIndexBO(String keyword, int type, int weight, Integer count, Integer countForApp, Integer countForBlk) {
super();
this.keyword = keyword;
this.type = type;
this.weight = weight;
this.count = count != null ? count.intValue() : 0;
this.countForApp = countForApp != null ? countForApp.intValue() : 0;
this.countForBlk = countForBlk != null ? countForBlk.intValue() : 0;
}
public String getKeyword() {
return keyword;
}
public void setKeyword(String keyword) {
this.keyword = keyword;
}
public int getType() {
return type;
}
public void setType(int type) {
this.type = type;
}
public int getWeight() {
return weight;
}
public void setWeight(int weight) {
this.weight = weight;
}
public int getCount() {
return count;
}
public void setCount(int count) {
this.count = count;
}
public int getCountForApp() {
return countForApp;
}
public void setCountForApp(int countForApp) {
this.countForApp = countForApp;
}
public int getCountForBlk() {
return countForBlk;
}
public void setCountForBlk(int countForBlk) {
this.countForBlk = countForBlk;
}
}
... ...
... ... @@ -87,7 +87,7 @@
<property key="refresh_interval" value="1s"/>
<property key="translog.flush_threshold_ops" value="5000"/>
</properties>
<builderClass>com.yoho.search.consumer.index.fullbuild.SuggestExtendedIndexBuilder,com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder</builderClass>
<builderClass>com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder</builderClass>
<mappingFile>esmapping/suggest.json</mappingFile>
</index>
... ...
... ... @@ -87,7 +87,7 @@
<property key="refresh_interval" value="${search.index.refresh_interval}"/>
<property key="translog.flush_threshold_ops" value="${search.index.translog.flush_threshold_ops}"/>
</properties>
<builderClass>com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder,com.yoho.search.consumer.index.fullbuild.SuggestExtendedIndexBuilder</builderClass>
<builderClass>com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder</builderClass>
<mappingFile>esmapping/suggest.json</mappingFile>
</index>
... ...