yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by wangnan
Commit e532dbfbef0e163684fd21340e15982102849db6 2 parents d7fd4cad b3e831f1

Merge branch '1215' of http://git.yoho.cn/yoho-search/yoho-search-consumer into 1215

Inline Side-by-side
Showing 7 changed files with 16 additions and 6 deletions
@@ -46,7 +46,6 @@ public class IndexRebuildJob implements ApplicationEventPublisherAware { @@ -46,7 +46,6 @@ public class IndexRebuildJob implements ApplicationEventPublisherAware {
46 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_ROBOTQUESTION); 46 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_ROBOTQUESTION);
47 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_PRODUCT_PRICE_PLAN); 47 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_PRODUCT_PRICE_PLAN);
48 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_HELPER); 48 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_HELPER);
49 - this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_SUGGEST);  
50 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_SHOPS); 49 this.rebuildIndexWithlog(ISearchConstants.INDEX_NAME_SHOPS);
51 50
52 try { 51 try {
@@ -189,6 +189,7 @@ public class IndexController implements ApplicationEventPublisherAware { @@ -189,6 +189,7 @@ public class IndexController implements ApplicationEventPublisherAware {
189 } 189 }
190 indexRebuildJob.rebuildTblProductIndex(); 190 indexRebuildJob.rebuildTblProductIndex();
191 indexRebuildJob.execute(); 191 indexRebuildJob.execute();
  192 + suggestionDiscoveryJob.execute();
192 suggestionCounterJob.execute(); 193 suggestionCounterJob.execute();
193 indexRebuildJob.rebuildSuggestIndex(); 194 indexRebuildJob.rebuildSuggestIndex();
194 return getResultMap(200, "rebuildAll success"); 195 return getResultMap(200, "rebuildAll success");
@@ -26,9 +26,9 @@ public class SuggestionConstants { @@ -26,9 +26,9 @@ public class SuggestionConstants {
26 26
27 public static int SUGGESTION_COUNT_BATCH_LIMIT = Configuration.getInt("suggestion.count.batch.limit", 300); 27 public static int SUGGESTION_COUNT_BATCH_LIMIT = Configuration.getInt("suggestion.count.batch.limit", 300);
28 28
29 - public static int SUGGESTION_COUNT_BATCH_MAX_THREAD_SIZE = Configuration.getInt("suggestion.count.batch.max.thread.size", 3); 29 + public static int SUGGESTION_COUNT_BATCH_MAX_THREAD_SIZE = Configuration.getInt("suggestion.count.batch.max.thread.size", 1);
30 30
31 - public static int SUGGESTION_DISCOVER_BATCH_LIMIT = Configuration.getInt("suggestion.discover.batch.limit", 300); 31 + public static int SUGGESTION_DISCOVER_BATCH_LIMIT = Configuration.getInt("suggestion.discover.batch.limit", 500);
32 32
33 public static int SUGGESTION_DISCOVER_BATCH_MAX_THREAD_SIZE = Configuration.getInt("suggestion.discover.batch.max.thread.size", 3); 33 public static int SUGGESTION_DISCOVER_BATCH_MAX_THREAD_SIZE = Configuration.getInt("suggestion.discover.batch.max.thread.size", 3);
34 34
@@ -60,7 +60,7 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli @@ -60,7 +60,7 @@ public abstract class AbstractSuggestionCounter implements ApplicationEventPubli
60 } 60 }
61 61
62 Map<String, Integer> filterMap = filter(pageNo, countMap); 62 Map<String, Integer> filterMap = filter(pageNo, countMap);
63 - logger.info("[{} business][pageNo={}][fetchWordSize={}][countMapSize={}][countMapSize={}]", flowName(), pageNo, fetchSize, countMapSize, filterMap.size()); 63 + logger.info("[{} business][pageNo={}][fetchWordSize={}][countMapSize={}][filterMapSize={}]", flowName(), pageNo, fetchSize, countMapSize, filterMap.size());
64 logger.trace("[{} business][pageNo={}][keywordMap={}][countMap={}][filterMap={}]", flowName(), pageNo, keywordMap, countMap, filterMap); 64 logger.trace("[{} business][pageNo={}][keywordMap={}][countMap={}][filterMap={}]", flowName(), pageNo, keywordMap, countMap, filterMap);
65 return persistence(keywordMap, filterMap); 65 return persistence(keywordMap, filterMap);
66 } 66 }
@@ -57,9 +57,11 @@ public class SuggestionKeywordCounter extends AbstractSuggestionCounter { @@ -57,9 +57,11 @@ public class SuggestionKeywordCounter extends AbstractSuggestionCounter {
57 Map<String, Integer> resultMap = super.filter(pageNo, countMap); 57 Map<String, Integer> resultMap = super.filter(pageNo, countMap);
58 58
59 // 检查一些不友好的推荐词 输出到日志中 59 // 检查一些不友好的推荐词 输出到日志中
  60 + if (logger.isInfoEnabled()) {
60 List<String> notNormalWords = resultMap.keySet().stream().filter(keyword -> notNormalKeyword(keyword)).collect(Collectors.toList()); 61 List<String> notNormalWords = resultMap.keySet().stream().filter(keyword -> notNormalKeyword(keyword)).collect(Collectors.toList());
61 if (notNormalWords != null && !notNormalWords.isEmpty()) { 62 if (notNormalWords != null && !notNormalWords.isEmpty()) {
62 - logger.warn("found not normal words: {}", notNormalWords.stream().map(item -> "'" + item + "'").collect(Collectors.joining(","))); 63 + logger.info("[{} check][pageNo={}]found not normal words: {}", flowName(), pageNo, notNormalWords.stream().map(item -> "'" + item + "'").collect(Collectors.joining(",")));
  64 + }
63 } 65 }
64 66
65 return resultMap; 67 return resultMap;
@@ -56,5 +56,9 @@ search.multiMatchQuery.type=BEST_FIELDS @@ -56,5 +56,9 @@ search.multiMatchQuery.type=BEST_FIELDS
56 56
57 #suggestion 57 #suggestion
58 suggestion.count.batch.limit=300 58 suggestion.count.batch.limit=300
59 -suggestion.count.batch.max.thread.size=3 59 +suggestion.count.batch.max.thread.size=1
  60 +suggestion.discover.batch.limit=500
  61 +suggestion.discover.batch.max.thread.size=3
  62 +suggestion.count.agg.size=10000
  63 +suggestion.count.agg.mindoc=5
60 64
@@ -59,3 +59,7 @@ search.multiMatchQuery.type=BEST_FIELDS @@ -59,3 +59,7 @@ search.multiMatchQuery.type=BEST_FIELDS
59 #suggestion 59 #suggestion
60 suggestion.count.batch.limit=${suggestion.count.batch.limit} 60 suggestion.count.batch.limit=${suggestion.count.batch.limit}
61 suggestion.count.batch.max.thread.size=${suggestion.count.batch.max.thread.size} 61 suggestion.count.batch.max.thread.size=${suggestion.count.batch.max.thread.size}
  62 +suggestion.discover.batch.limit=${suggestion.discover.batch.limit}
  63 +suggestion.discover.batch.max.thread.size=${suggestion.discover.batch.max.thread.size}
  64 +suggestion.count.agg.size=${suggestion.count.agg.size}
  65 +suggestion.count.agg.mindoc=${suggestion.count.agg.mindoc}