yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by Gino Zhang
Commit fb270d0806dd52b02cbb0a2e5928fbbe4b11ccc7 2 parents 1da75bb6 6cc2fdf8

Merge branch 'zf_suggest_parametermake' into zf_smart_search2 

# Conflicts:
#	service/src/main/java/com/yoho/search/consumer/service/base/ParameterMakeService.java
Inline Side-by-side
Showing 9 changed files with 125 additions and 49 deletions
1 package com.yoho.search.dal; 1 package com.yoho.search.dal;
2 2
3 import com.yoho.search.dal.model.ParameterMake; 3 import com.yoho.search.dal.model.ParameterMake;
  4 +import org.apache.ibatis.annotations.Param;
4 5
5 import java.util.List; 6 import java.util.List;
6 7
@@ -18,4 +19,8 @@ public interface ParameterMakeMapper { @@ -18,4 +19,8 @@ public interface ParameterMakeMapper {
18 int updateByPrimaryKey(ParameterMake record); 19 int updateByPrimaryKey(ParameterMake record);
19 20
20 List<ParameterMake> getAll(); 21 List<ParameterMake> getAll();
  22 +
  23 + int selectTotalCount();
  24 +
  25 + List<ParameterMake> selectPageLists(@Param(value="offset")Integer offset, @Param(value="pageSize")Integer pageSize);
21 } 26 }
@@ -92,4 +92,12 @@ @@ -92,4 +92,12 @@
92 <include refid="Base_Column_List" /> 92 <include refid="Base_Column_List" />
93 from parameter_make 93 from parameter_make
94 </select> 94 </select>
  95 + <select id="selectTotalCount" resultType="java.lang.Integer" timeout="20000">
  96 + SELECT count(1) FROM parameter_make
  97 + </select>
  98 + <select id="selectPageLists" resultMap="BaseResultMap" timeout="20000">
  99 + select
  100 + <include refid="Base_Column_List" />
  101 + from parameter_make limit #{offset},#{pageSize}
  102 + </select>
95 </mapper> 103 </mapper>
@@ -16,7 +16,9 @@ public enum KeywordType { @@ -16,7 +16,9 @@ public enum KeywordType {
16 ProductNameToken(4, 6, SuggestionConstants.PRODUCTNAME_TOKEN_ENABLED_KEY), 16 ProductNameToken(4, 6, SuggestionConstants.PRODUCTNAME_TOKEN_ENABLED_KEY),
17 ProductKeywordToken(5, 6,SuggestionConstants.PRODUCTKEYWORD_TOKEN_ENABLED_KEY), 17 ProductKeywordToken(5, 6,SuggestionConstants.PRODUCTKEYWORD_TOKEN_ENABLED_KEY),
18 BrandKeywordToken(6, 6, SuggestionConstants.BRANDKEYWORD_TOKEN_ENABLED_KEY), 18 BrandKeywordToken(6, 6, SuggestionConstants.BRANDKEYWORD_TOKEN_ENABLED_KEY),
19 - 19 +
  20 + ParameterMake(12, 3, SuggestionConstants.PARAMETERMAKE_ENABLED_KEY),
  21 +
20 ProductName(8, 1, SuggestionConstants.PRODUCTNAME_ENABLED_KEY), 22 ProductName(8, 1, SuggestionConstants.PRODUCTNAME_ENABLED_KEY),
21 Customized(0, 1), 23 Customized(0, 1),
22 HotKeyword(3, 0,SuggestionConstants.HOTKEYWORD_ENABLED_KEY); 24 HotKeyword(3, 0,SuggestionConstants.HOTKEYWORD_ENABLED_KEY);
@@ -2,6 +2,9 @@ package com.yoho.search.consumer.suggests.common; @@ -2,6 +2,9 @@ package com.yoho.search.consumer.suggests.common;
2 2
3 import com.yoho.search.base.utils.Configuration; 3 import com.yoho.search.base.utils.Configuration;
4 4
  5 +import java.util.Arrays;
  6 +import java.util.List;
  7 +
5 /** 8 /**
6 * Created by ginozhang on 2016/11/25. 9 * Created by ginozhang on 2016/11/25.
7 */ 10 */
@@ -13,6 +16,8 @@ public class SuggestionConstants { @@ -13,6 +16,8 @@ public class SuggestionConstants {
13 16
14 public static final Integer VALID_STATUS = Integer.valueOf(1); 17 public static final Integer VALID_STATUS = Integer.valueOf(1);
15 18
  19 + public static final List<String> IGNORE_KEYWORDS = Arrays.asList("其他","正常","中文","中国","普通","2%");
  20 +
16 public static String YOHO_SEARCH_KEYWORDS_HOT = "YOHO.SEARCH.KEYWORDS.HOT"; 21 public static String YOHO_SEARCH_KEYWORDS_HOT = "YOHO.SEARCH.KEYWORDS.HOT";
17 22
18 23
@@ -27,6 +32,7 @@ public class SuggestionConstants { @@ -27,6 +32,7 @@ public class SuggestionConstants {
27 32
28 public static String SHOPNAME_ENABLED_KEY = "search.suggestion.consumer.shopname.enable"; 33 public static String SHOPNAME_ENABLED_KEY = "search.suggestion.consumer.shopname.enable";
29 public static String PRODUCTNAME_ENABLED_KEY = "search.suggestion.consumer.productname.enable"; 34 public static String PRODUCTNAME_ENABLED_KEY = "search.suggestion.consumer.productname.enable";
  35 + public static String PARAMETERMAKE_ENABLED_KEY = "search.suggestion.consumer.parametermake.enable";
30 36
31 public static int FETCH_HOTKEYWORD_COUNT = 10000; 37 public static int FETCH_HOTKEYWORD_COUNT = 10000;
32 38
  1 +package com.yoho.search.consumer.suggests.discover;
  2 +
  3 +import com.yoho.search.consumer.service.base.ParameterMakeService;
  4 +import com.yoho.search.consumer.suggests.common.KeywordType;
  5 +import com.yoho.search.consumer.suggests.common.SuggestionConstants;
  6 +import com.yoho.search.dal.model.ParameterMake;
  7 +import org.apache.commons.collections.CollectionUtils;
  8 +import org.springframework.beans.factory.annotation.Autowired;
  9 +import org.springframework.stereotype.Component;
  10 +
  11 +import java.util.Arrays;
  12 +import java.util.HashSet;
  13 +import java.util.List;
  14 +import java.util.Set;
  15 +
  16 +/**
  17 + * Created by ginozhang on 2016/11/25.
  18 + */
  19 +@Component
  20 +public class ParameterMakeSuggestionDiscoverer extends AbstractSuggestionDiscoverer {
  21 +
  22 + @Autowired
  23 + private ParameterMakeService parameterMakeService;
  24 +
  25 + @Override
  26 + public int count() {
  27 + return parameterMakeService.selectTotalCount();
  28 + }
  29 +
  30 + @Override
  31 + public Set<String> getSuggestWordSet(int pageNo, int batchSize) {
  32 + Set<String> parameterValueSet = new HashSet<>(batchSize);
  33 + int start = (pageNo - 1) * batchSize;
  34 + List<ParameterMake> parameterMakeList = parameterMakeService.selectPageLists(start, batchSize);
  35 + if (CollectionUtils.isNotEmpty(parameterMakeList)) {
  36 + for (ParameterMake pm : parameterMakeList) {
  37 + if (pm.getParameterValue() != null && pm.getParameterValue().length() > 1) {
  38 + for (String item : pm.getParameterValue().split("\\/")) {
  39 + if (!SuggestionConstants.IGNORE_KEYWORDS.contains(pm.getParameterValue())) {
  40 + parameterValueSet.add(item);
  41 + }
  42 + }
  43 + }
  44 + }
  45 + }
  46 +
  47 + return parameterValueSet;
  48 + }
  49 +
  50 + @Override
  51 + public KeywordType getKeywordType() {
  52 + return KeywordType.ParameterMake;
  53 + }
  54 +
  55 + public static void main(String[] args) {
  56 + System.out.println(Arrays.asList("羊毛/羊绒".split("\\/")));
  57 + }
  58 +}
1 package com.yoho.search.consumer.service.base; 1 package com.yoho.search.consumer.service.base;
2 2
  3 +import com.yoho.search.dal.ParameterMakeMapper;
3 import com.yoho.search.dal.model.ParameterMake; 4 import com.yoho.search.dal.model.ParameterMake;
4 import org.springframework.beans.factory.annotation.Autowired; 5 import org.springframework.beans.factory.annotation.Autowired;
5 import org.springframework.stereotype.Service; 6 import org.springframework.stereotype.Service;
6 7
7 -import com.yoho.search.dal.ParameterMakeMapper;  
8 -  
9 import java.util.List; 8 import java.util.List;
10 9
11 @Service 10 @Service
12 public class ParameterMakeService { 11 public class ParameterMakeService {
13 -  
14 - @Autowired  
15 - private ParameterMakeMapper parameterMakeMapper;  
16 -  
17 - public ParameterMake getById(Integer id) {  
18 - return parameterMakeMapper.selectByPrimaryKey(id);  
19 - }  
20 -  
21 - public int insert(ParameterMake parameterMake) {  
22 - return parameterMakeMapper.insert(parameterMake);  
23 - }  
24 -  
25 - public int update(ParameterMake parameterMake) {  
26 - return parameterMakeMapper.updateByPrimaryKeySelective(parameterMake);  
27 - }  
28 -  
29 - public int saveOrUpdate(ParameterMake parameterMake) {  
30 - if (parameterMake.getId() == null || parameterMakeMapper.selectByPrimaryKey(parameterMake.getId()) == null) {  
31 - return parameterMakeMapper.insert(parameterMake);  
32 - } else {  
33 - return parameterMakeMapper.updateByPrimaryKeySelective(parameterMake);  
34 - }  
35 - }  
36 -  
37 - public int delete(Integer id) {  
38 - return parameterMakeMapper.deleteByPrimaryKey(id);  
39 - }  
40 -  
41 - public List<ParameterMake> getAll(){  
42 - return parameterMakeMapper.getAll();  
43 - } 12 +
  13 + @Autowired
  14 + private ParameterMakeMapper parameterMakeMapper;
  15 +
  16 + public ParameterMake getById(Integer id) {
  17 + return parameterMakeMapper.selectByPrimaryKey(id);
  18 + }
  19 +
  20 + public int insert(ParameterMake parameterMake) {
  21 + return parameterMakeMapper.insert(parameterMake);
  22 + }
  23 +
  24 + public int update(ParameterMake parameterMake) {
  25 + return parameterMakeMapper.updateByPrimaryKeySelective(parameterMake);
  26 + }
  27 +
  28 + public int saveOrUpdate(ParameterMake parameterMake) {
  29 + if (parameterMake.getId() == null || parameterMakeMapper.selectByPrimaryKey(parameterMake.getId()) == null) {
  30 + return parameterMakeMapper.insert(parameterMake);
  31 + } else {
  32 + return parameterMakeMapper.updateByPrimaryKeySelective(parameterMake);
  33 + }
  34 + }
  35 +
  36 + public int delete(Integer id) {
  37 + return parameterMakeMapper.deleteByPrimaryKey(id);
  38 + }
  39 +
  40 + public List<ParameterMake> getAll() {
  41 + return parameterMakeMapper.getAll();
  42 + }
  43 +
  44 + public int selectTotalCount() {
  45 + return parameterMakeMapper.selectTotalCount();
  46 + }
  47 +
  48 + public List<ParameterMake> selectPageLists(int start, int size) {
  49 + return parameterMakeMapper.selectPageLists(start, size);
  50 + }
44 } 51 }
@@ -2,6 +2,7 @@ package com.yoho.search.spider.conversation; @@ -2,6 +2,7 @@ package com.yoho.search.spider.conversation;
2 2
3 import com.yoho.search.consumer.index.common.AnalyzerHelper; 3 import com.yoho.search.consumer.index.common.AnalyzerHelper;
4 import com.yoho.search.consumer.service.base.*; 4 import com.yoho.search.consumer.service.base.*;
  5 +import com.yoho.search.consumer.suggests.common.SuggestionConstants;
5 import com.yoho.search.dal.model.*; 6 import com.yoho.search.dal.model.*;
6 import org.apache.commons.collections.CollectionUtils; 7 import org.apache.commons.collections.CollectionUtils;
7 import org.apache.commons.collections.map.HashedMap; 8 import org.apache.commons.collections.map.HashedMap;
@@ -26,9 +27,6 @@ public class SuggestConvertorService { @@ -26,9 +27,6 @@ public class SuggestConvertorService {
26 27
27 private static final int MAX_KEYWORDS = 5; 28 private static final int MAX_KEYWORDS = 5;
28 29
29 - // TODO:  
30 - private static final List<String> IGNORE_KEYWORDS = Arrays.asList("其他","正常");  
31 -  
32 @Autowired 30 @Autowired
33 private AnalyzerHelper analyzerHelper; 31 private AnalyzerHelper analyzerHelper;
34 32
@@ -58,7 +56,7 @@ public class SuggestConvertorService { @@ -58,7 +56,7 @@ public class SuggestConvertorService {
58 Map<String, Set<String>> yohoKeywords = new HashMap<>(keywordSet.size()); 56 Map<String, Set<String>> yohoKeywords = new HashMap<>(keywordSet.size());
59 Set<String> tokens; 57 Set<String> tokens;
60 for (String keyword : keywordSet) { 58 for (String keyword : keywordSet) {
61 - if (keyword.length() > 1 && !IGNORE_KEYWORDS.contains(keyword) && CollectionUtils.isNotEmpty(tokens = analyzerHelper.getTokens(keyword, ANALYZER))) { 59 + if (keyword.length() > 1 && !SuggestionConstants.IGNORE_KEYWORDS.contains(keyword) && CollectionUtils.isNotEmpty(tokens = analyzerHelper.getTokens(keyword, ANALYZER))) {
62 yohoKeywords.put(keyword, tokens); 60 yohoKeywords.put(keyword, tokens);
63 } 61 }
64 } 62 }
@@ -6,6 +6,7 @@ import com.yoho.search.base.utils.RedisKeys; @@ -6,6 +6,7 @@ import com.yoho.search.base.utils.RedisKeys;
6 import com.yoho.search.consumer.index.common.AnalyzerHelper; 6 import com.yoho.search.consumer.index.common.AnalyzerHelper;
7 import com.yoho.search.consumer.service.base.SpiderContentService; 7 import com.yoho.search.consumer.service.base.SpiderContentService;
8 import com.yoho.search.consumer.suggests.common.RetryBusinessFlow; 8 import com.yoho.search.consumer.suggests.common.RetryBusinessFlow;
  9 +import com.yoho.search.consumer.suggests.common.SuggestionConstants;
9 import com.yoho.search.dal.model.SpiderContent; 10 import com.yoho.search.dal.model.SpiderContent;
10 import org.apache.commons.collections.CollectionUtils; 11 import org.apache.commons.collections.CollectionUtils;
11 import org.apache.commons.lang.StringUtils; 12 import org.apache.commons.lang.StringUtils;
@@ -33,8 +34,6 @@ public class KeywordCrawlerFlow implements RetryBusinessFlow { @@ -33,8 +34,6 @@ public class KeywordCrawlerFlow implements RetryBusinessFlow {
33 34
34 private static final Logger logger = LoggerFactory.getLogger("FLOW_EXECUTOR"); 35 private static final Logger logger = LoggerFactory.getLogger("FLOW_EXECUTOR");
35 36
36 - private static final Set<String> IGNORE_KEYWORDS = new HashSet<>();  
37 -  
38 private static final int KEYWORD_COUNT = 1000; 37 private static final int KEYWORD_COUNT = 1000;
39 38
40 @Resource(name = "yhNoSyncZSetOperations") 39 @Resource(name = "yhNoSyncZSetOperations")
@@ -61,13 +60,6 @@ public class KeywordCrawlerFlow implements RetryBusinessFlow { @@ -61,13 +60,6 @@ public class KeywordCrawlerFlow implements RetryBusinessFlow {
61 60
62 @Override 61 @Override
63 public void init() { 62 public void init() {
64 - //TODO： 增加忽略的关键词 需要能自行添加  
65 - IGNORE_KEYWORDS.add("其他");  
66 - IGNORE_KEYWORDS.add("中文");  
67 - IGNORE_KEYWORDS.add("中国");  
68 - IGNORE_KEYWORDS.add("普通");  
69 - IGNORE_KEYWORDS.add("2%");  
70 -  
71 Set<String> keywordSet = new HashSet<>(1000); 63 Set<String> keywordSet = new HashSet<>(1000);
72 Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(RedisKeys.getRedisKey4Yesterday(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY), 0, KEYWORD_COUNT); 64 Set<ZSetOperations.TypedTuple<String>> redisResults = yhNoSyncZSetOperations.reverseRangeWithScores(RedisKeys.getRedisKey4Yesterday(RedisKeys.YOHO_SEARCH_KEYWORDS_EMPTY), 0, KEYWORD_COUNT);
73 for (ZSetOperations.TypedTuple<String> typedTuple : redisResults) { 65 for (ZSetOperations.TypedTuple<String> typedTuple : redisResults) {
@@ -106,7 +98,7 @@ public class KeywordCrawlerFlow implements RetryBusinessFlow { @@ -106,7 +98,7 @@ public class KeywordCrawlerFlow implements RetryBusinessFlow {
106 } 98 }
107 99
108 // 在忽略列表的不要 100 // 在忽略列表的不要
109 - if (IGNORE_KEYWORDS.contains(keyword)) { 101 + if (SuggestionConstants.IGNORE_KEYWORDS.contains(keyword)) {
110 return false; 102 return false;
111 } 103 }
112 104
@@ -6,7 +6,7 @@ search.suggestion.consumer.sortname.enable=true @@ -6,7 +6,7 @@ search.suggestion.consumer.sortname.enable=true
6 search.suggestion.consumer.productname.token.enable=true 6 search.suggestion.consumer.productname.token.enable=true
7 search.suggestion.consumer.productkeyword.token.enable=true 7 search.suggestion.consumer.productkeyword.token.enable=true
8 search.suggestion.consumer.brandkeyword.token.enable=true 8 search.suggestion.consumer.brandkeyword.token.enable=true
9 - 9 +search.suggestion.consumer.parametermake.enable=true
10 search.suggestion.consumer.shopname.enable=true 10 search.suggestion.consumer.shopname.enable=true
11 search.suggestion.consumer.productname.enable=true 11 search.suggestion.consumer.productname.enable=true
12 search.suggestion.consumer.brandwithsortname.enable=true 12 search.suggestion.consumer.brandwithsortname.enable=true