suggest优化: 1.去除支持老的suggestion_keywords表; 2.suggest的计数支持blk和app;
Showing
15 changed files
with
283 additions
and
359 deletions
@@ -11,6 +11,10 @@ public class SuggestWordDef { | @@ -11,6 +11,10 @@ public class SuggestWordDef { | ||
11 | 11 | ||
12 | private Integer count; | 12 | private Integer count; |
13 | 13 | ||
14 | + private Integer countForBlk; | ||
15 | + | ||
16 | + private Integer countForApp; | ||
17 | + | ||
14 | private Integer weight; | 18 | private Integer weight; |
15 | 19 | ||
16 | private Integer type; | 20 | private Integer type; |
@@ -49,6 +53,22 @@ public class SuggestWordDef { | @@ -49,6 +53,22 @@ public class SuggestWordDef { | ||
49 | this.count = count; | 53 | this.count = count; |
50 | } | 54 | } |
51 | 55 | ||
56 | + public Integer getCountForBlk() { | ||
57 | + return countForBlk; | ||
58 | + } | ||
59 | + | ||
60 | + public void setCountForBlk(Integer countForBlk) { | ||
61 | + this.countForBlk = countForBlk; | ||
62 | + } | ||
63 | + | ||
64 | + public Integer getCountForApp() { | ||
65 | + return countForApp; | ||
66 | + } | ||
67 | + | ||
68 | + public void setCountForApp(Integer countForApp) { | ||
69 | + this.countForApp = countForApp; | ||
70 | + } | ||
71 | + | ||
52 | public Integer getWeight() { | 72 | public Integer getWeight() { |
53 | return weight; | 73 | return weight; |
54 | } | 74 | } |
@@ -6,21 +6,25 @@ | @@ -6,21 +6,25 @@ | ||
6 | <result column="keyword" property="keyword" jdbcType="VARCHAR" /> | 6 | <result column="keyword" property="keyword" jdbcType="VARCHAR" /> |
7 | <result column="weight" property="weight" jdbcType="INTEGER" /> | 7 | <result column="weight" property="weight" jdbcType="INTEGER" /> |
8 | <result column="count" property="count" jdbcType="INTEGER" /> | 8 | <result column="count" property="count" jdbcType="INTEGER" /> |
9 | + <result column="count_for_blk" property="countForBlk" jdbcType="INTEGER" /> | ||
10 | + <result column="count_for_global" property="countForGlobal" jdbcType="INTEGER" /> | ||
9 | <result column="type" property="type" jdbcType="INTEGER" /> | 11 | <result column="type" property="type" jdbcType="INTEGER" /> |
10 | <result column="status" property="status" jdbcType="INTEGER" /> | 12 | <result column="status" property="status" jdbcType="INTEGER" /> |
11 | </resultMap> | 13 | </resultMap> |
12 | <sql id="Base_Column_List"> | 14 | <sql id="Base_Column_List"> |
13 | - id, keyword, weight, count, type, status | 15 | + id, keyword, weight, count, count_for_blk, count_for_global, type, status |
14 | </sql> | 16 | </sql> |
15 | 17 | ||
16 | <insert id="insertBatch" parameterType="java.util.List" timeout="20000"> | 18 | <insert id="insertBatch" parameterType="java.util.List" timeout="20000"> |
17 | - insert ignore into suggest_word_def (keyword, weight, count, type) | 19 | + insert ignore into suggest_word_def (keyword, weight, count, count_for_blk, count_for_global, type) |
18 | values | 20 | values |
19 | <foreach collection="list" item="item" index="index" | 21 | <foreach collection="list" item="item" index="index" |
20 | separator=","> | 22 | separator=","> |
21 | (#{item.keyword, jdbcType=VARCHAR}, | 23 | (#{item.keyword, jdbcType=VARCHAR}, |
22 | #{item.weight, jdbcType=INTEGER}, | 24 | #{item.weight, jdbcType=INTEGER}, |
23 | #{item.count, jdbcType=INTEGER}, | 25 | #{item.count, jdbcType=INTEGER}, |
26 | + #{item.countForBlk, jdbcType=INTEGER}, | ||
27 | + #{item.countForGlobal, jdbcType=INTEGER}, | ||
24 | #{item.type, jdbcType=INTEGER}) | 28 | #{item.type, jdbcType=INTEGER}) |
25 | </foreach> | 29 | </foreach> |
26 | </insert> | 30 | </insert> |
@@ -33,6 +37,16 @@ | @@ -33,6 +37,16 @@ | ||
33 | when id = #{item.id,jdbcType=INTEGER} then #{item.count,jdbcType=INTEGER} | 37 | when id = #{item.id,jdbcType=INTEGER} then #{item.count,jdbcType=INTEGER} |
34 | </foreach> | 38 | </foreach> |
35 | </trim> | 39 | </trim> |
40 | + <trim prefix="count_for_blk =case" suffix="end,"> | ||
41 | + <foreach collection="suggestWordDefList" item="item" index="index"> | ||
42 | + when id = #{item.id,jdbcType=INTEGER} then #{item.countForBlk,jdbcType=INTEGER} | ||
43 | + </foreach> | ||
44 | + </trim> | ||
45 | + <trim prefix="count_for_global =case" suffix="end,"> | ||
46 | + <foreach collection="suggestWordDefList" item="item" index="index"> | ||
47 | + when id = #{item.id,jdbcType=INTEGER} then #{item.countForGlobal,jdbcType=INTEGER} | ||
48 | + </foreach> | ||
49 | + </trim> | ||
36 | <trim prefix="weight =case" suffix="end,"> | 50 | <trim prefix="weight =case" suffix="end,"> |
37 | <foreach collection="suggestWordDefList" item="item" index="index"> | 51 | <foreach collection="suggestWordDefList" item="item" index="index"> |
38 | when id = #{item.id,jdbcType=INTEGER} then #{item.weight,jdbcType=DECIMAL} | 52 | when id = #{item.id,jdbcType=INTEGER} then #{item.weight,jdbcType=DECIMAL} |
1 | -package com.yoho.search.consumer.index.fullbuild; | ||
2 | - | ||
3 | -import java.util.ArrayList; | ||
4 | -import java.util.List; | ||
5 | -import java.util.Set; | ||
6 | -import java.util.stream.Collectors; | ||
7 | -import java.util.stream.Stream; | ||
8 | - | ||
9 | -import org.springframework.beans.factory.annotation.Autowired; | ||
10 | -import org.springframework.stereotype.Component; | ||
11 | - | ||
12 | -import com.yoho.search.base.utils.MD5Util; | ||
13 | -import com.yoho.search.consumer.common.DynamicConfigService; | ||
14 | -import com.yoho.search.consumer.index.common.IIndexBuilder; | ||
15 | -import com.yoho.search.consumer.service.base.SuggestWordDefService; | ||
16 | -import com.yoho.search.consumer.service.bo.SuggestIndexBO; | ||
17 | -import com.yoho.search.consumer.suggests.common.KeywordType; | ||
18 | -import com.yoho.search.consumer.suggests.common.SuggestionConstants; | ||
19 | -import com.yoho.search.dal.model.SuggestWordDef; | ||
20 | - | ||
21 | -@Component | ||
22 | -public class SuggestExtendedIndexBuilder extends IIndexBuilder { | ||
23 | - | ||
24 | - @Autowired | ||
25 | - private SuggestWordDefService suggestWordDefService; | ||
26 | - @Autowired | ||
27 | - private DynamicConfigService dynamicConfigService; | ||
28 | - | ||
29 | - @Override | ||
30 | - public int getTotalCount() throws Exception { | ||
31 | - return suggestWordDefService.selectTotalCount(); | ||
32 | - } | ||
33 | - | ||
34 | - @Override | ||
35 | - public List<?> getPageLists(int offset, int limit) throws Exception { | ||
36 | - Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType)) | ||
37 | - .map(KeywordType::getType).collect(Collectors.toSet()); | ||
38 | - // 获取分页列表 | ||
39 | - List<SuggestWordDef> list = suggestWordDefService.selectPageList(offset, limit); | ||
40 | - // 构建结果 | ||
41 | - List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>(); | ||
42 | - for (SuggestWordDef suggestWordDef : list) { | ||
43 | - if (suggestWordDef.getCount() == null || suggestWordDef.getCount() == 0) { | ||
44 | - continue; | ||
45 | - } | ||
46 | - if (!suggestWordDef.getStatus().equals(SuggestionConstants.VALID_STATUS) || !enabledKeywordTypes.contains(suggestWordDef.getType())) { | ||
47 | - continue; | ||
48 | - } | ||
49 | - // 根据Type取权重 | ||
50 | - int weight = KeywordType.getWeightValueByType(suggestWordDef.getType()); | ||
51 | - results.add(new SuggestIndexBO(suggestWordDef.getKeyword(), suggestWordDef.getType(), weight, suggestWordDef.getCount())); | ||
52 | - } | ||
53 | - return results; | ||
54 | - } | ||
55 | - | ||
56 | - @Override | ||
57 | - public String getId(Object object) { | ||
58 | - return MD5Util.string2MD5(((SuggestIndexBO) object).getKeyword().trim().toLowerCase()); | ||
59 | - } | ||
60 | - | ||
61 | -} |
1 | package com.yoho.search.consumer.index.fullbuild; | 1 | package com.yoho.search.consumer.index.fullbuild; |
2 | 2 | ||
3 | -import java.util.ArrayList; | ||
4 | -import java.util.List; | ||
5 | - | ||
6 | -import org.springframework.beans.factory.annotation.Autowired; | ||
7 | -import org.springframework.stereotype.Component; | ||
8 | - | ||
9 | import com.yoho.search.base.utils.MD5Util; | 3 | import com.yoho.search.base.utils.MD5Util; |
4 | +import com.yoho.search.consumer.common.DynamicConfigService; | ||
10 | import com.yoho.search.consumer.index.common.IIndexBuilder; | 5 | import com.yoho.search.consumer.index.common.IIndexBuilder; |
11 | -import com.yoho.search.consumer.service.base.SuggestionKeywordsService; | 6 | +import com.yoho.search.consumer.service.base.SuggestWordDefService; |
12 | import com.yoho.search.consumer.service.bo.SuggestIndexBO; | 7 | import com.yoho.search.consumer.service.bo.SuggestIndexBO; |
8 | +import com.yoho.search.consumer.suggests.common.KeywordType; | ||
9 | +import com.yoho.search.consumer.suggests.common.SuggestionConstants; | ||
10 | +import com.yoho.search.dal.model.SuggestWordDef; | ||
11 | +import org.springframework.beans.factory.annotation.Autowired; | ||
12 | +import org.springframework.stereotype.Component; | ||
13 | + | ||
14 | +import java.util.ArrayList; | ||
15 | +import java.util.List; | ||
16 | +import java.util.Set; | ||
17 | +import java.util.stream.Collectors; | ||
18 | +import java.util.stream.Stream; | ||
13 | 19 | ||
14 | @Component | 20 | @Component |
15 | public class SuggestIndexBuilder extends IIndexBuilder { | 21 | public class SuggestIndexBuilder extends IIndexBuilder { |
16 | 22 | ||
17 | @Autowired | 23 | @Autowired |
18 | - private SuggestionKeywordsService suggestionKeywordsService; | 24 | + private SuggestWordDefService suggestWordDefService; |
25 | + | ||
26 | + @Autowired | ||
27 | + private DynamicConfigService dynamicConfigService; | ||
19 | 28 | ||
20 | @Override | 29 | @Override |
21 | public int getTotalCount() throws Exception { | 30 | public int getTotalCount() throws Exception { |
22 | - // return suggestionKeywordsService.count(); | ||
23 | - return 0; | 31 | + return suggestWordDefService.selectTotalCount(); |
24 | } | 32 | } |
25 | 33 | ||
26 | @Override | 34 | @Override |
27 | public List<?> getPageLists(int offset, int limit) throws Exception { | 35 | public List<?> getPageLists(int offset, int limit) throws Exception { |
28 | - // List<SuggestionKeywords> guggestionKeywordList = | ||
29 | - // suggestionKeywordsService.getPageLists(offset, limit); | ||
30 | - // List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>(); | ||
31 | - // for (SuggestionKeywords kw : guggestionKeywordList) { | ||
32 | - // if (kw.getCount() == null || kw.getCount() == 0) { | ||
33 | - // continue; | ||
34 | - // } | ||
35 | - // int type = KeywordType.Customized.getType(); | ||
36 | - // int weight = KeywordType.Customized.getWeightValue(); | ||
37 | - // results.add(new SuggestIndexBO(kw.getKeyword(), type, weight, | ||
38 | - // kw.getCount())); | ||
39 | - // } | ||
40 | - return new ArrayList<SuggestIndexBO>(); | 36 | + Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType)) |
37 | + .map(KeywordType::getType).collect(Collectors.toSet()); | ||
38 | + // 获取分页列表 | ||
39 | + List<SuggestWordDef> list = suggestWordDefService.selectPageList(offset, limit); | ||
40 | + // 构建结果 | ||
41 | + List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>(); | ||
42 | + for (SuggestWordDef suggestWordDef : list) { | ||
43 | + if (!suggestWordDef.getStatus().equals(SuggestionConstants.VALID_STATUS) || !enabledKeywordTypes.contains(suggestWordDef.getType())) { | ||
44 | + continue; | ||
45 | + } | ||
46 | + | ||
47 | + // 根据Type取权重 | ||
48 | + int weight = KeywordType.getWeightValueByType(suggestWordDef.getType()); | ||
49 | + results.add(new SuggestIndexBO(suggestWordDef.getKeyword(), suggestWordDef.getType(), weight, suggestWordDef.getCount(), suggestWordDef.getCountForApp(), suggestWordDef.getCountForBlk())); | ||
50 | + } | ||
51 | + return results; | ||
41 | } | 52 | } |
42 | 53 | ||
43 | @Override | 54 | @Override |
@@ -284,25 +284,6 @@ public class IndexController implements ApplicationEventPublisherAware { | @@ -284,25 +284,6 @@ public class IndexController implements ApplicationEventPublisherAware { | ||
284 | } | 284 | } |
285 | } | 285 | } |
286 | 286 | ||
287 | - @RequestMapping(value = "/index/suggestion/count") | ||
288 | - @ResponseBody | ||
289 | - public Map<String, Object> suggestionDetail(@RequestParam String keyword) { | ||
290 | - try { | ||
291 | - Integer countInES = keywordCounterService.countInES(keyword); | ||
292 | - Map<String, Object> rtnMap = getResultMap(200, "success"); | ||
293 | - Map<String, Integer> data = new HashMap<>(10); | ||
294 | - data.put("countInES", countInES); | ||
295 | - rtnMap.put("data", data); | ||
296 | - return rtnMap; | ||
297 | - } catch (Exception e) { | ||
298 | - logger.error(e.getMessage(), e); | ||
299 | - Map<String, Object> rtnMap = new HashMap<String, Object>(); | ||
300 | - rtnMap.put("code", 400); | ||
301 | - rtnMap.put("msg", e.getMessage()); | ||
302 | - return rtnMap; | ||
303 | - } | ||
304 | - } | ||
305 | - | ||
306 | @RequestMapping(value = "/index/generateRule") | 287 | @RequestMapping(value = "/index/generateRule") |
307 | @ResponseBody | 288 | @ResponseBody |
308 | public Map<String, Object> generateRule() { | 289 | public Map<String, Object> generateRule() { |
index/src/main/java/com/yoho/search/consumer/suggests/common/SuggestSearchParamBuilder.java
0 → 100644
1 | +package com.yoho.search.consumer.suggests.common; | ||
2 | + | ||
3 | +import com.yoho.search.base.utils.ConvertUtils; | ||
4 | +import com.yoho.search.base.utils.ISearchConstants; | ||
5 | +import com.yoho.search.consumer.suggests.counter.CountUsage; | ||
6 | +import com.yoho.search.core.es.model.SearchParam; | ||
7 | +import org.apache.commons.lang3.StringUtils; | ||
8 | +import org.elasticsearch.index.query.*; | ||
9 | + | ||
10 | +import java.util.List; | ||
11 | + | ||
12 | +/** | ||
13 | + * Created by ginozhang on 2017/1/3. | ||
14 | + */ | ||
15 | +public final class SuggestSearchParamBuilder { | ||
16 | + | ||
17 | + public static SearchParam build(String keyword, CountUsage countUsage) { | ||
18 | + // TODO: 暂时先从service那边拷贝相关代码 后续考虑抽取到core | ||
19 | + SearchParam searchParam = new SearchParam(); | ||
20 | + searchParam.setQuery(getMultiMatchQueryBuilder(keyword)); | ||
21 | + searchParam.setFiter(getBoolQueryBuilder(keyword, countUsage)); | ||
22 | + searchParam.setSize(0); | ||
23 | + return searchParam; | ||
24 | + } | ||
25 | + | ||
26 | + private static BoolQueryBuilder getBoolQueryBuilder(String keyword, CountUsage countUsage) { | ||
27 | + BoolQueryBuilder boolFilter = QueryBuilders.boolQuery(); | ||
28 | + // 增加默认的过滤条件 | ||
29 | + switch (countUsage) { | ||
30 | + case BLK: | ||
31 | + boolFilter.must(QueryBuilders.termQuery("appType", 1)); //BLK商品 | ||
32 | + case APP: | ||
33 | + break; | ||
34 | + case PC: | ||
35 | + boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y")); // 非全球购 | ||
36 | + break; | ||
37 | + default: | ||
38 | + break; | ||
39 | + } | ||
40 | + | ||
41 | + boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y")); // 非秒杀 | ||
42 | + boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(1)); // 有库存 | ||
43 | + boolFilter.mustNot(QueryBuilders.termQuery("attribute", "2")); // 非赠品 | ||
44 | + boolFilter.must(QueryBuilders.termQuery("status", "1")); // 上架商品 | ||
45 | + // 1对男女关键字做特殊处理 | ||
46 | + boolean cotainBoy = keyword.contains("男"); | ||
47 | + boolean cotainGirl = keyword.contains("女"); | ||
48 | + // 对性别做特殊处理 | ||
49 | + String gender = null; | ||
50 | + if (cotainBoy && !cotainGirl) { | ||
51 | + gender = "1,3"; | ||
52 | + } else if (!cotainBoy && cotainGirl) { | ||
53 | + gender = "2,3"; | ||
54 | + } else if (cotainBoy && cotainGirl) { | ||
55 | + gender = "1,2,3"; | ||
56 | + } | ||
57 | + if (gender != null) { | ||
58 | + int[] genders = ConvertUtils.stringToIntArray(gender, ","); | ||
59 | + boolFilter.must(QueryBuilders.termsQuery("gender", genders)); | ||
60 | + } | ||
61 | + return boolFilter; | ||
62 | + } | ||
63 | + | ||
64 | + private static QueryBuilder getMultiMatchQueryBuilder(String keyword) { | ||
65 | + if (StringUtils.isEmpty(keyword)) { | ||
66 | + return QueryBuilders.matchAllQuery(); | ||
67 | + } | ||
68 | + | ||
69 | + MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keyword); | ||
70 | + MultiMatchQueryBuilder.Type multiMatchQueryBuilderType = getMultiMatchQueryBuilderType(); | ||
71 | + if (multiMatchQueryBuilderType != null) { | ||
72 | + queryBuilder.type(multiMatchQueryBuilderType); | ||
73 | + } | ||
74 | + | ||
75 | + setDefaultSearchField(queryBuilder); | ||
76 | + if (ISearchConstants.SEARCH_OPERATOR.equalsIgnoreCase("or")) { | ||
77 | + queryBuilder.operator(MatchQueryBuilder.Operator.OR); | ||
78 | + queryBuilder.minimumShouldMatch(ISearchConstants.SEARCH_MINIMUM_SHOULD_MATCH); | ||
79 | + } else { | ||
80 | + queryBuilder.operator(MatchQueryBuilder.Operator.AND); | ||
81 | + } | ||
82 | + return queryBuilder; | ||
83 | + } | ||
84 | + | ||
85 | + private static MultiMatchQueryBuilder.Type getMultiMatchQueryBuilderType() { | ||
86 | + String configMultiMatchQueryType = ISearchConstants.SEARCH_MULTIMATCHQUERY_TYPE; | ||
87 | + if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.BEST_FIELDS.name())) { | ||
88 | + return MultiMatchQueryBuilder.Type.BEST_FIELDS; | ||
89 | + } | ||
90 | + if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.MOST_FIELDS.name())) { | ||
91 | + return MultiMatchQueryBuilder.Type.MOST_FIELDS; | ||
92 | + } | ||
93 | + if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.CROSS_FIELDS.name())) { | ||
94 | + return MultiMatchQueryBuilder.Type.CROSS_FIELDS; | ||
95 | + } | ||
96 | + if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE.name())) { | ||
97 | + return MultiMatchQueryBuilder.Type.PHRASE; | ||
98 | + } | ||
99 | + if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE_PREFIX.name())) { | ||
100 | + return MultiMatchQueryBuilder.Type.PHRASE_PREFIX; | ||
101 | + } | ||
102 | + return null; | ||
103 | + } | ||
104 | + | ||
105 | + private static void setDefaultSearchField(MultiMatchQueryBuilder queryBuilder) { | ||
106 | + List<String> fields = ISearchConstants.SEARCH_DEFAULT_FIELD; | ||
107 | + for (String field : fields) { | ||
108 | + String[] fieldBoost = field.split("^"); | ||
109 | + if (fieldBoost.length == 2) { | ||
110 | + queryBuilder.field(fieldBoost[0], Float.parseFloat(fieldBoost[1])); | ||
111 | + } else if (fieldBoost.length == 1) { | ||
112 | + queryBuilder.field(fieldBoost[0]); | ||
113 | + } | ||
114 | + } | ||
115 | + } | ||
116 | +} |
@@ -52,17 +52,32 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli | @@ -52,17 +52,32 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli | ||
52 | return true; | 52 | return true; |
53 | } | 53 | } |
54 | 54 | ||
55 | - Map<String, Integer> countMap = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList())); | ||
56 | - int countMapSize = countMap != null ? countMap.size() : 0; | ||
57 | - if (countMapSize == 0) { | ||
58 | - logger.info("[{} business][pageNo={}][countMapSize={}]", flowName(), pageNo, countMapSize); | 55 | + Map<String, Integer> countMapForApp = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()), CountUsage.APP); |
56 | + int countMapSizeForApp = countMapForApp != null ? countMapForApp.size() : 0; | ||
57 | + if (countMapSizeForApp == 0) { | ||
58 | + logger.info("[{} business][pageNo={}][countMapSizeForApp={}]", flowName(), pageNo, countMapSizeForApp); | ||
59 | return false; | 59 | return false; |
60 | } | 60 | } |
61 | 61 | ||
62 | - Map<String, Integer> filterMap = filter(pageNo, countMap); | ||
63 | - logger.info("[{} business][pageNo={}][fetchWordSize={}][countMapSize={}][filterMapSize={}]", flowName(), pageNo, fetchSize, countMapSize, filterMap.size()); | ||
64 | - logger.trace("[{} business][pageNo={}][keywordMap={}][countMap={}][filterMap={}]", flowName(), pageNo, keywordMap, countMap, filterMap); | ||
65 | - return persistence(keywordMap, filterMap); | 62 | + Map<String, Integer> countMapForPC = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()), CountUsage.PC); |
63 | + int countMapSizeForPC = countMapForPC != null ? countMapForPC.size() : 0; | ||
64 | + if (countMapSizeForPC == 0) { | ||
65 | + logger.info("[{} business][pageNo={}][countMapForPC={}]", flowName(), pageNo, countMapSizeForPC); | ||
66 | + return false; | ||
67 | + } | ||
68 | + | ||
69 | + Map<String, Integer> countMapForBlk = keywordCounterService.batchCount(keywordMap.keySet().stream().collect(Collectors.toList()), CountUsage.BLK); | ||
70 | + int countMapSizeForBlk = countMapForBlk != null ? countMapForBlk.size() : 0; | ||
71 | + if (countMapSizeForBlk == 0) { | ||
72 | + logger.info("[{} business][pageNo={}][countMapSizeForBlk={}]", flowName(), pageNo, countMapSizeForBlk); | ||
73 | + return false; | ||
74 | + } | ||
75 | + | ||
76 | + // countMapForApp 的过滤条件最宽松,使用该Map来进行filter | ||
77 | + Map<String, Integer> filterMap = filter(pageNo, countMapForApp); | ||
78 | + logger.info("[{} business][pageNo={}][fetchWordSize={}][countMapForApp={}][countMapForPC={}][countMapForBlk={}][filterMapSize={}]", flowName(), pageNo, fetchSize, countMapForApp, countMapForPC, countMapForBlk, filterMap.size()); | ||
79 | + logger.trace("[{} business][pageNo={}][keywordMap={}][countMapForApp={}][filterMap={}]", flowName(), pageNo, keywordMap, countMapForApp, filterMap); | ||
80 | + return persistence(keywordMap, filterMap, countMapForApp, countMapForPC, countMapForBlk); | ||
66 | } | 81 | } |
67 | 82 | ||
68 | protected Map<String, Integer> filter(int pageNo, Map<String, Integer> countMap) { | 83 | protected Map<String, Integer> filter(int pageNo, Map<String, Integer> countMap) { |
@@ -88,5 +103,5 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli | @@ -88,5 +103,5 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli | ||
88 | 103 | ||
89 | abstract Map<String, Object> getKeywordMap(int pageNo, int batchSize); | 104 | abstract Map<String, Object> getKeywordMap(int pageNo, int batchSize); |
90 | 105 | ||
91 | - abstract boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap); | 106 | + abstract boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap, Map<String, Integer> countMapForApp, Map<String, Integer> countMapForPC, Map<String, Integer> countMapForBlk); |
92 | } | 107 | } |
1 | package com.yoho.search.consumer.suggests.counter; | 1 | package com.yoho.search.consumer.suggests.counter; |
2 | 2 | ||
3 | -import com.yoho.search.base.utils.ConvertUtils; | ||
4 | import com.yoho.search.base.utils.ISearchConstants; | 3 | import com.yoho.search.base.utils.ISearchConstants; |
5 | import com.yoho.search.consumer.index.common.IYohoIndexService; | 4 | import com.yoho.search.consumer.index.common.IYohoIndexService; |
6 | -import com.yoho.search.core.es.agg.IAggregation; | 5 | +import com.yoho.search.consumer.suggests.common.SuggestSearchParamBuilder; |
7 | import com.yoho.search.core.es.model.SearchParam; | 6 | import com.yoho.search.core.es.model.SearchParam; |
8 | import com.yoho.search.core.es.model.SearchResult; | 7 | import com.yoho.search.core.es.model.SearchResult; |
9 | - | ||
10 | -import org.apache.commons.lang3.StringUtils; | ||
11 | -import org.elasticsearch.index.query.*; | ||
12 | import org.springframework.beans.factory.annotation.Autowired; | 8 | import org.springframework.beans.factory.annotation.Autowired; |
13 | import org.springframework.stereotype.Component; | 9 | import org.springframework.stereotype.Component; |
14 | import org.springframework.util.Assert; | 10 | import org.springframework.util.Assert; |
@@ -27,123 +23,17 @@ public class KeywordCounterService { | @@ -27,123 +23,17 @@ public class KeywordCounterService { | ||
27 | @Autowired | 23 | @Autowired |
28 | private IYohoIndexService yohoIndexService; | 24 | private IYohoIndexService yohoIndexService; |
29 | 25 | ||
30 | - public Integer countInES(String keyword) { | ||
31 | - Assert.notNull(keyword); | ||
32 | - SearchParam searchParam = buildSearchParam(keyword); | ||
33 | - SearchResult result = yohoIndexService.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam); | ||
34 | - return result != null ? Integer.valueOf(String.valueOf(result.getTotal())) : null; | ||
35 | - } | ||
36 | - | ||
37 | - @SuppressWarnings("unchecked") | ||
38 | - public Map<String, Integer> batchCountByAgg(IAggregation aggregation) { | ||
39 | - SearchParam searchParam = buildSearchParam(""); | ||
40 | - searchParam.addAbstractAggregationBuilder(aggregation.getBuilder()); | ||
41 | - SearchResult searchResult = yohoIndexService.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam); | ||
42 | - if (searchResult != null) { | ||
43 | - return (Map<String, Integer>) aggregation.getAggregationResponseMap(searchResult.getAggMaps()); | ||
44 | - } | ||
45 | - return null; | ||
46 | - } | ||
47 | - | ||
48 | - public Map<String, Integer> batchCount(List<String> keywords) { | 26 | + public Map<String, Integer> batchCount(List<String> keywords, CountUsage countUsage) { |
49 | Assert.notNull(keywords); | 27 | Assert.notNull(keywords); |
50 | Map<String, Integer> countResultMap = new LinkedHashMap<>(); | 28 | Map<String, Integer> countResultMap = new LinkedHashMap<>(); |
51 | - List<SearchParam> searchParams = keywords.stream().map(keyword -> buildSearchParam(keyword)).collect(Collectors.toList()); | 29 | + List<SearchParam> searchParams = keywords.stream().map(keyword -> SuggestSearchParamBuilder.build(keyword, countUsage)).collect(Collectors.toList()); |
52 | List<SearchResult> results = yohoIndexService.multiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams); | 30 | List<SearchResult> results = yohoIndexService.multiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams); |
53 | Assert.notNull(results); | 31 | Assert.notNull(results); |
54 | Assert.isTrue(results.size() == keywords.size()); | 32 | Assert.isTrue(results.size() == keywords.size()); |
55 | for (int i = 0; i < keywords.size(); i++) { | 33 | for (int i = 0; i < keywords.size(); i++) { |
56 | countResultMap.put(keywords.get(i), Integer.valueOf(String.valueOf(results.get(i).getTotal()))); | 34 | countResultMap.put(keywords.get(i), Integer.valueOf(String.valueOf(results.get(i).getTotal()))); |
57 | } | 35 | } |
58 | - return countResultMap; | ||
59 | - } | ||
60 | - | ||
61 | - private SearchParam buildSearchParam(String keyword) { | ||
62 | - // TODO: 暂时先从service那边拷贝相关代码 后续考虑抽取到core | ||
63 | - SearchParam searchParam = new SearchParam(); | ||
64 | - searchParam.setQuery(getMultiMatchQueryBuilder(keyword)); | ||
65 | - searchParam.setFiter(getBoolQueryBuilder(keyword)); | ||
66 | - searchParam.setSize(0); | ||
67 | - return searchParam; | ||
68 | - } | ||
69 | - | ||
70 | - private BoolQueryBuilder getBoolQueryBuilder(String keyword) { | ||
71 | - BoolQueryBuilder boolFilter = QueryBuilders.boolQuery(); | ||
72 | - // 增加默认的过滤条件 TODO: 支持可配置 | ||
73 | - boolFilter.mustNot(QueryBuilders.termsQuery("isGlobal", "Y")); // 非全球购 | ||
74 | - boolFilter.mustNot(QueryBuilders.termsQuery("isSeckill", "Y")); // 非秒杀 | ||
75 | - boolFilter.must(QueryBuilders.rangeQuery("storageNum").gte(1)); // 有库存 | ||
76 | - boolFilter.mustNot(QueryBuilders.termQuery("attribute", "2")); // 非赠品 | ||
77 | - boolFilter.must(QueryBuilders.termQuery("status", "1")); // 上架商品 | ||
78 | - // 1对男女关键字做特殊处理 | ||
79 | - boolean cotainBoy = keyword.contains("男"); | ||
80 | - boolean cotainGirl = keyword.contains("女"); | ||
81 | - // 对性别做特殊处理 | ||
82 | - String gender = null; | ||
83 | - if (cotainBoy && !cotainGirl) { | ||
84 | - gender = "1,3"; | ||
85 | - } else if (!cotainBoy && cotainGirl) { | ||
86 | - gender = "2,3"; | ||
87 | - } else if (cotainBoy && cotainGirl) { | ||
88 | - gender = "1,2,3"; | ||
89 | - } | ||
90 | - if (gender != null) { | ||
91 | - int[] genders = ConvertUtils.stringToIntArray(gender, ","); | ||
92 | - boolFilter.must(QueryBuilders.termsQuery("gender", genders)); | ||
93 | - } | ||
94 | - return boolFilter; | ||
95 | - } | ||
96 | - | ||
97 | - private QueryBuilder getMultiMatchQueryBuilder(String keyword) { | ||
98 | - if (StringUtils.isEmpty(keyword)) { | ||
99 | - return QueryBuilders.matchAllQuery(); | ||
100 | - } | ||
101 | 36 | ||
102 | - MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(keyword); | ||
103 | - MultiMatchQueryBuilder.Type multiMatchQueryBuilderType = this.getMultiMatchQueryBuilderType(); | ||
104 | - if (multiMatchQueryBuilderType != null) { | ||
105 | - queryBuilder.type(multiMatchQueryBuilderType); | ||
106 | - } | ||
107 | - | ||
108 | - setDefaultSearchField(queryBuilder); | ||
109 | - if (ISearchConstants.SEARCH_OPERATOR.equalsIgnoreCase("or")) { | ||
110 | - queryBuilder.operator(MatchQueryBuilder.Operator.OR); | ||
111 | - queryBuilder.minimumShouldMatch(ISearchConstants.SEARCH_MINIMUM_SHOULD_MATCH); | ||
112 | - } else { | ||
113 | - queryBuilder.operator(MatchQueryBuilder.Operator.AND); | ||
114 | - } | ||
115 | - return queryBuilder; | ||
116 | - } | ||
117 | - | ||
118 | - private MultiMatchQueryBuilder.Type getMultiMatchQueryBuilderType() { | ||
119 | - String configMultiMatchQueryType = ISearchConstants.SEARCH_MULTIMATCHQUERY_TYPE; | ||
120 | - if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.BEST_FIELDS.name())) { | ||
121 | - return MultiMatchQueryBuilder.Type.BEST_FIELDS; | ||
122 | - } | ||
123 | - if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.MOST_FIELDS.name())) { | ||
124 | - return MultiMatchQueryBuilder.Type.MOST_FIELDS; | ||
125 | - } | ||
126 | - if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.CROSS_FIELDS.name())) { | ||
127 | - return MultiMatchQueryBuilder.Type.CROSS_FIELDS; | ||
128 | - } | ||
129 | - if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE.name())) { | ||
130 | - return MultiMatchQueryBuilder.Type.PHRASE; | ||
131 | - } | ||
132 | - if (configMultiMatchQueryType.equalsIgnoreCase(MultiMatchQueryBuilder.Type.PHRASE_PREFIX.name())) { | ||
133 | - return MultiMatchQueryBuilder.Type.PHRASE_PREFIX; | ||
134 | - } | ||
135 | - return null; | ||
136 | - } | ||
137 | - | ||
138 | - private void setDefaultSearchField(MultiMatchQueryBuilder queryBuilder) { | ||
139 | - List<String> fields = ISearchConstants.SEARCH_DEFAULT_FIELD; | ||
140 | - for (String field : fields) { | ||
141 | - String[] fieldBoost = field.split("^"); | ||
142 | - if (fieldBoost.length == 2) { | ||
143 | - queryBuilder.field(fieldBoost[0], Float.parseFloat(fieldBoost[1])); | ||
144 | - } else if (fieldBoost.length == 1) { | ||
145 | - queryBuilder.field(fieldBoost[0]); | ||
146 | - } | ||
147 | - } | 37 | + return countResultMap; |
148 | } | 38 | } |
149 | } | 39 | } |
1 | package com.yoho.search.consumer.suggests.counter; | 1 | package com.yoho.search.consumer.suggests.counter; |
2 | 2 | ||
3 | -import java.util.ArrayList; | ||
4 | -import java.util.HashMap; | ||
5 | -import java.util.List; | ||
6 | -import java.util.Map; | ||
7 | -import java.util.Set; | ||
8 | -import java.util.stream.Collectors; | ||
9 | -import java.util.stream.Stream; | ||
10 | - | ||
11 | -import org.apache.commons.collections.CollectionUtils; | ||
12 | -import org.springframework.beans.factory.annotation.Autowired; | ||
13 | -import org.springframework.stereotype.Component; | ||
14 | - | ||
15 | import com.yoho.search.consumer.common.DynamicConfigService; | 3 | import com.yoho.search.consumer.common.DynamicConfigService; |
16 | import com.yoho.search.consumer.service.base.SuggestWordDefService; | 4 | import com.yoho.search.consumer.service.base.SuggestWordDefService; |
17 | import com.yoho.search.consumer.suggests.common.KeywordType; | 5 | import com.yoho.search.consumer.suggests.common.KeywordType; |
18 | import com.yoho.search.consumer.suggests.common.SuggestionConstants; | 6 | import com.yoho.search.consumer.suggests.common.SuggestionConstants; |
19 | import com.yoho.search.dal.model.SuggestWordDef; | 7 | import com.yoho.search.dal.model.SuggestWordDef; |
8 | +import org.apache.commons.collections.CollectionUtils; | ||
9 | +import org.springframework.beans.factory.annotation.Autowired; | ||
10 | +import org.springframework.stereotype.Component; | ||
11 | + | ||
12 | +import java.util.*; | ||
13 | +import java.util.stream.Collectors; | ||
14 | +import java.util.stream.Stream; | ||
20 | 15 | ||
21 | /** | 16 | /** |
22 | * Created by ginozhang on 2016/11/25. | 17 | * Created by ginozhang on 2016/11/25. |
@@ -49,12 +44,14 @@ public class SuggestWordDefCounter extends AbstractSuggestionCounter { | @@ -49,12 +44,14 @@ public class SuggestWordDefCounter extends AbstractSuggestionCounter { | ||
49 | } | 44 | } |
50 | 45 | ||
51 | @Override | 46 | @Override |
52 | - boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap) { | 47 | + boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap, Map<String, Integer> countMapForApp, Map<String, Integer> countMapForPC, Map<String, Integer> countMapForBlk) { |
53 | List<SuggestWordDef> batchList = new ArrayList<>(countMap.size()); | 48 | List<SuggestWordDef> batchList = new ArrayList<>(countMap.size()); |
54 | countMap.forEach((keyword, count) -> { | 49 | countMap.forEach((keyword, count) -> { |
55 | SuggestWordDef suggestWordDef = (SuggestWordDef) keywordMap.get(keyword); | 50 | SuggestWordDef suggestWordDef = (SuggestWordDef) keywordMap.get(keyword); |
56 | if (suggestWordDef != null) { | 51 | if (suggestWordDef != null) { |
57 | - suggestWordDef.setCount(count); | 52 | + suggestWordDef.setCount(calCount(countMapForPC, keyword)); |
53 | + suggestWordDef.setCountForApp(calCount(countMapForApp, keyword)); | ||
54 | + suggestWordDef.setCountForBlk(calCount(countMapForBlk, keyword)); | ||
58 | suggestWordDef.setWeight(KeywordType.getWeightValueByType(suggestWordDef.getType())); | 55 | suggestWordDef.setWeight(KeywordType.getWeightValueByType(suggestWordDef.getType())); |
59 | batchList.add(suggestWordDef); | 56 | batchList.add(suggestWordDef); |
60 | } | 57 | } |
@@ -62,4 +59,9 @@ public class SuggestWordDefCounter extends AbstractSuggestionCounter { | @@ -62,4 +59,9 @@ public class SuggestWordDefCounter extends AbstractSuggestionCounter { | ||
62 | suggestWordDefService.updateBatch(batchList); | 59 | suggestWordDefService.updateBatch(batchList); |
63 | return true; | 60 | return true; |
64 | } | 61 | } |
62 | + | ||
63 | + Integer calCount(Map<String, Integer> countResultMap, String suggestWord) { | ||
64 | + Integer count = countResultMap.get(suggestWord); | ||
65 | + return count != null ? count : Integer.valueOf(0); | ||
66 | + } | ||
65 | } | 67 | } |
index/src/main/java/com/yoho/search/consumer/suggests/counter/SuggestionKeywordCounter.java
deleted
100644 → 0
1 | -package com.yoho.search.consumer.suggests.counter; | ||
2 | - | ||
3 | -import com.yoho.search.consumer.service.base.SuggestionKeywordsService; | ||
4 | -import com.yoho.search.dal.model.SuggestionKeywords; | ||
5 | -import org.apache.commons.collections.CollectionUtils; | ||
6 | -import org.apache.commons.lang3.math.NumberUtils; | ||
7 | -import org.springframework.beans.factory.annotation.Autowired; | ||
8 | -import org.springframework.stereotype.Component; | ||
9 | - | ||
10 | -import java.util.*; | ||
11 | -import java.util.concurrent.ConcurrentHashMap; | ||
12 | -import java.util.stream.Collectors; | ||
13 | - | ||
14 | -/** | ||
15 | - * Created by ginozhang on 2016/11/25. | ||
16 | - */ | ||
17 | -@Component | ||
18 | -public class SuggestionKeywordCounter extends AbstractSuggestionCounter { | ||
19 | - | ||
20 | - private static final List<String> invalidChractors = Arrays.asList(">", ":", "?", "+", ":", "?", "!", "@", "#", "¥", "$", "%", "……", "^", "&", "*", ";", ";", "<", "、", "/", "\\", "|", "【", "】", "[", "]", "(", ")", "(", ")"); | ||
21 | - | ||
22 | - private Map<String, Integer> notNormalWordMap = new ConcurrentHashMap<>(); | ||
23 | - | ||
24 | - @Autowired | ||
25 | - private SuggestionKeywordsService suggestionKeywordsService; | ||
26 | - | ||
27 | - @Override | ||
28 | - public int getTotalCount() { | ||
29 | - return suggestionKeywordsService.count(); | ||
30 | - } | ||
31 | - | ||
32 | - @Override | ||
33 | - Map<String, Object> getKeywordMap(int pageNo, int batchSize) { | ||
34 | - int start = (pageNo - 1) * batchSize; | ||
35 | - Map<String, Object> wordMap = new HashMap<>(batchSize); | ||
36 | - List<SuggestionKeywords> dataList = suggestionKeywordsService.getPageLists(start, batchSize); | ||
37 | - if (CollectionUtils.isNotEmpty(dataList)) { | ||
38 | - dataList.forEach((item) -> wordMap.put(item.getKeyword(), item)); | ||
39 | - } | ||
40 | - | ||
41 | - return wordMap; | ||
42 | - } | ||
43 | - | ||
44 | - @Override | ||
45 | - boolean persistence(Map<String, Object> keywordMap, Map<String, Integer> countMap) { | ||
46 | - List<SuggestionKeywords> batchList = new ArrayList<>(countMap.size()); | ||
47 | - countMap.forEach((keyword, count) -> { | ||
48 | - SuggestionKeywords suggestionKeywords = (SuggestionKeywords) keywordMap.get(keyword); | ||
49 | - if (suggestionKeywords != null) { | ||
50 | - suggestionKeywords.setCount(count); | ||
51 | - batchList.add(suggestionKeywords); | ||
52 | - } | ||
53 | - }); | ||
54 | - suggestionKeywordsService.updateBatch(batchList); | ||
55 | - return true; | ||
56 | - } | ||
57 | - | ||
58 | - @Override | ||
59 | - protected Map<String, Integer> filter(int pageNo, Map<String, Integer> countMap) { | ||
60 | - Map<String, Integer> resultMap = super.filter(pageNo, countMap); | ||
61 | - | ||
62 | - // 检查一些不友好的推荐词 输出到日志中 | ||
63 | - if (logger.isInfoEnabled()) { | ||
64 | - resultMap.forEach((key, value) -> { | ||
65 | - if (notNormalKeyword(key)) { | ||
66 | - synchronized (this) { | ||
67 | - notNormalWordMap.put(key, value); | ||
68 | - } | ||
69 | - } | ||
70 | - }); | ||
71 | - } | ||
72 | - | ||
73 | - return resultMap; | ||
74 | - } | ||
75 | - | ||
76 | - private boolean notNormalKeyword(String key) { | ||
77 | - //1. 全数字的 | ||
78 | - key = key.trim(); | ||
79 | - if (NumberUtils.isNumber(key)) { | ||
80 | - return true; | ||
81 | - } | ||
82 | - | ||
83 | - //2. 包含特殊字段 | ||
84 | - for (String invalidChar : invalidChractors) { | ||
85 | - if (key.contains(invalidChar)) { | ||
86 | - return true; | ||
87 | - } | ||
88 | - } | ||
89 | - | ||
90 | - // 3. 长度太长或太短 | ||
91 | - if (key.length() <= 1 && key.length() >= 15) { | ||
92 | - return true; | ||
93 | - } | ||
94 | - | ||
95 | - return false; | ||
96 | - } | ||
97 | - | ||
98 | - @Override | ||
99 | - public void init() { | ||
100 | - super.init(); | ||
101 | - this.notNormalWordMap.clear(); | ||
102 | - } | ||
103 | - | ||
104 | - @Override | ||
105 | - public void finish(boolean doBusinessResult, Exception exception) { | ||
106 | - super.finish(doBusinessResult, exception); | ||
107 | - if (!notNormalWordMap.isEmpty() && logger.isInfoEnabled()) { | ||
108 | - logger.info("[{} check]found not normal words: {}", flowName(), notNormalWordMap.keySet().stream().map(item -> "'" + item + "'").collect(Collectors.joining(","))); | ||
109 | - notNormalWordMap.clear(); | ||
110 | - } | ||
111 | - } | ||
112 | -} |
1 | package com.yoho.search.consumer.suggests.discover; | 1 | package com.yoho.search.consumer.suggests.discover; |
2 | 2 | ||
3 | +import com.yoho.search.base.utils.ISearchConstants; | ||
4 | +import com.yoho.search.consumer.index.common.IYohoIndexService; | ||
5 | +import com.yoho.search.consumer.suggests.common.SuggestSearchParamBuilder; | ||
6 | +import com.yoho.search.consumer.suggests.counter.CountUsage; | ||
3 | import com.yoho.search.consumer.suggests.counter.KeywordCounterService; | 7 | import com.yoho.search.consumer.suggests.counter.KeywordCounterService; |
4 | import com.yoho.search.core.es.agg.IAggregation; | 8 | import com.yoho.search.core.es.agg.IAggregation; |
9 | +import com.yoho.search.core.es.model.SearchParam; | ||
10 | +import com.yoho.search.core.es.model.SearchResult; | ||
5 | import org.springframework.beans.factory.annotation.Autowired; | 11 | import org.springframework.beans.factory.annotation.Autowired; |
6 | 12 | ||
7 | import java.util.HashSet; | 13 | import java.util.HashSet; |
@@ -18,12 +24,15 @@ public abstract class AbstractAggSuggestionDiscoverer extends AbstractSuggestion | @@ -18,12 +24,15 @@ public abstract class AbstractAggSuggestionDiscoverer extends AbstractSuggestion | ||
18 | @Autowired | 24 | @Autowired |
19 | protected KeywordCounterService keywordCounterService; | 25 | protected KeywordCounterService keywordCounterService; |
20 | 26 | ||
27 | + @Autowired | ||
28 | + private IYohoIndexService yohoIndexService; | ||
29 | + | ||
21 | protected volatile List<String> tokenList = null; | 30 | protected volatile List<String> tokenList = null; |
22 | 31 | ||
23 | @Override | 32 | @Override |
24 | public int count() { | 33 | public int count() { |
25 | // TODO: 对于聚合类的暂时都一次性地从ES获取 后续可以考虑通过skn分页来支持多次获取 | 34 | // TODO: 对于聚合类的暂时都一次性地从ES获取 后续可以考虑通过skn分页来支持多次获取 |
26 | - Map<String, Integer> countMap = keywordCounterService.batchCountByAgg(getAggregation()); | 35 | + Map<String, Integer> countMap = findSuggestionTokenByAgg(getAggregation()); |
27 | if (countMap == null || countMap.isEmpty()) { | 36 | if (countMap == null || countMap.isEmpty()) { |
28 | return 0; | 37 | return 0; |
29 | } | 38 | } |
@@ -43,6 +52,16 @@ public abstract class AbstractAggSuggestionDiscoverer extends AbstractSuggestion | @@ -43,6 +52,16 @@ public abstract class AbstractAggSuggestionDiscoverer extends AbstractSuggestion | ||
43 | return tokenList.size(); | 52 | return tokenList.size(); |
44 | } | 53 | } |
45 | 54 | ||
55 | + private Map<String, Integer> findSuggestionTokenByAgg(IAggregation aggregation) { | ||
56 | + SearchParam searchParam = SuggestSearchParamBuilder.build("", CountUsage.PC); | ||
57 | + searchParam.addAbstractAggregationBuilder(aggregation.getBuilder()); | ||
58 | + SearchResult searchResult = yohoIndexService.search(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam); | ||
59 | + if (searchResult != null) { | ||
60 | + return (Map<String, Integer>) aggregation.getAggregationResponseMap(searchResult.getAggMaps()); | ||
61 | + } | ||
62 | + return null; | ||
63 | + } | ||
64 | + | ||
46 | @Override | 65 | @Override |
47 | public Set<String> getSuggestWordSet(int pageNo, int batchSize) { | 66 | public Set<String> getSuggestWordSet(int pageNo, int batchSize) { |
48 | Set<String> sortNameSet = new HashSet<>(batchSize); | 67 | Set<String> sortNameSet = new HashSet<>(batchSize); |
@@ -9,13 +9,17 @@ public class SuggestIndexBO implements Serializable { | @@ -9,13 +9,17 @@ public class SuggestIndexBO implements Serializable { | ||
9 | private int type; | 9 | private int type; |
10 | private int weight; | 10 | private int weight; |
11 | private int count; | 11 | private int count; |
12 | + private int countForApp; | ||
13 | + private int countForBlk; | ||
12 | 14 | ||
13 | - public SuggestIndexBO(String keyword, int type, int weight, int count) { | 15 | + public SuggestIndexBO(String keyword, int type, int weight, Integer count, Integer countForApp, Integer countForBlk) { |
14 | super(); | 16 | super(); |
15 | this.keyword = keyword; | 17 | this.keyword = keyword; |
16 | this.type = type; | 18 | this.type = type; |
17 | this.weight = weight; | 19 | this.weight = weight; |
18 | - this.count = count; | 20 | + this.count = count != null ? count.intValue() : 0; |
21 | + this.countForApp = countForApp != null ? countForApp.intValue() : 0; | ||
22 | + this.countForBlk = countForBlk != null ? countForBlk.intValue() : 0; | ||
19 | } | 23 | } |
20 | 24 | ||
21 | public String getKeyword() { | 25 | public String getKeyword() { |
@@ -50,4 +54,19 @@ public class SuggestIndexBO implements Serializable { | @@ -50,4 +54,19 @@ public class SuggestIndexBO implements Serializable { | ||
50 | this.count = count; | 54 | this.count = count; |
51 | } | 55 | } |
52 | 56 | ||
57 | + public int getCountForApp() { | ||
58 | + return countForApp; | ||
59 | + } | ||
60 | + | ||
61 | + public void setCountForApp(int countForApp) { | ||
62 | + this.countForApp = countForApp; | ||
63 | + } | ||
64 | + | ||
65 | + public int getCountForBlk() { | ||
66 | + return countForBlk; | ||
67 | + } | ||
68 | + | ||
69 | + public void setCountForBlk(int countForBlk) { | ||
70 | + this.countForBlk = countForBlk; | ||
71 | + } | ||
53 | } | 72 | } |
@@ -87,7 +87,7 @@ | @@ -87,7 +87,7 @@ | ||
87 | <property key="refresh_interval" value="1s"/> | 87 | <property key="refresh_interval" value="1s"/> |
88 | <property key="translog.flush_threshold_ops" value="5000"/> | 88 | <property key="translog.flush_threshold_ops" value="5000"/> |
89 | </properties> | 89 | </properties> |
90 | - <builderClass>com.yoho.search.consumer.index.fullbuild.SuggestExtendedIndexBuilder,com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder</builderClass> | 90 | + <builderClass>com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder</builderClass> |
91 | <mappingFile>esmapping/suggest.json</mappingFile> | 91 | <mappingFile>esmapping/suggest.json</mappingFile> |
92 | </index> | 92 | </index> |
93 | 93 |
@@ -87,7 +87,7 @@ | @@ -87,7 +87,7 @@ | ||
87 | <property key="refresh_interval" value="${search.index.refresh_interval}"/> | 87 | <property key="refresh_interval" value="${search.index.refresh_interval}"/> |
88 | <property key="translog.flush_threshold_ops" value="${search.index.translog.flush_threshold_ops}"/> | 88 | <property key="translog.flush_threshold_ops" value="${search.index.translog.flush_threshold_ops}"/> |
89 | </properties> | 89 | </properties> |
90 | - <builderClass>com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder,com.yoho.search.consumer.index.fullbuild.SuggestExtendedIndexBuilder</builderClass> | 90 | + <builderClass>com.yoho.search.consumer.index.fullbuild.SuggestIndexBuilder</builderClass> |
91 | <mappingFile>esmapping/suggest.json</mappingFile> | 91 | <mappingFile>esmapping/suggest.json</mappingFile> |
92 | </index> | 92 | </index> |
93 | 93 |
-
Please register or login to post a comment