yoho-search / yohomars-search-service · Commits

Go to a project

Authored by sang
Commit bd6f3e4ce7a01da560ddde9f162ab33bc3a73398 2 parents c6c5760c 5a627349

Merge branch 'master' of http://git.yoho.cn/yoho-search/yohomars-search-service

Inline Side-by-side
Showing 5 changed files with 51 additions and 42 deletions
1 package com.yohomars.search.queue; 1 package com.yohomars.search.queue;
2 2
  3 +import java.util.Map;
  4 +import java.util.concurrent.ArrayBlockingQueue;
  5 +import java.util.concurrent.BlockingQueue;
  6 +import java.util.concurrent.ConcurrentHashMap;
  7 +import java.util.concurrent.ExecutorService;
  8 +import java.util.concurrent.Executors;
  9 +
3 import org.slf4j.Logger; 10 import org.slf4j.Logger;
4 import org.slf4j.LoggerFactory; 11 import org.slf4j.LoggerFactory;
5 -import org.springframework.stereotype.Service;  
6 -  
7 -import java.util.Map;  
8 -import java.util.concurrent.*;  
9 12
10 /** 13 /**
11 * @author gris.wang 14 * @author gris.wang
12 * @since 2018/3/8 15 * @since 2018/3/8
13 **/ 16 **/
14 -@Service  
15 public class RunnableQueueFactory { 17 public class RunnableQueueFactory {
16 18
17 private static final Logger LOGGER = LoggerFactory.getLogger(RunnableQueueFactory.class); 19 private static final Logger LOGGER = LoggerFactory.getLogger(RunnableQueueFactory.class);
18 20
19 - private static final int QUEUE_THREADS_SIZE = 20; 21 + private static final int QUEUE_THREADS_SIZE = 5;
20 private static final int MAX_QUEUE_CAPACITY = 100; 22 private static final int MAX_QUEUE_CAPACITY = 100;
21 23
22 private static Map<String,BlockingQueue<Runnable>> queueMap; 24 private static Map<String,BlockingQueue<Runnable>> queueMap;
1 package com.yohomars.search.job; 1 package com.yohomars.search.job;
2 2
3 -import com.yohomars.search.index.service.IYohoIndexService;  
4 -import com.yohomars.search.queue.RunnableQueueFactory;  
5 -import com.yohomars.search.utils.Index;  
6 import org.slf4j.Logger; 3 import org.slf4j.Logger;
7 import org.slf4j.LoggerFactory; 4 import org.slf4j.LoggerFactory;
8 import org.springframework.beans.factory.annotation.Autowired; 5 import org.springframework.beans.factory.annotation.Autowired;
9 import org.springframework.scheduling.annotation.Scheduled; 6 import org.springframework.scheduling.annotation.Scheduled;
10 import org.springframework.stereotype.Component; 7 import org.springframework.stereotype.Component;
11 8
  9 +import com.yohomars.search.index.service.IYohoIndexService;
  10 +import com.yohomars.search.utils.Index;
  11 +
12 /** 12 /**
13 * 定时任务 13 * 定时任务
14 */ 14 */
@@ -20,9 +20,6 @@ public class IndexRebuildJob { @@ -20,9 +20,6 @@ public class IndexRebuildJob {
20 @Autowired 20 @Autowired
21 private IYohoIndexService yohoIndexService; 21 private IYohoIndexService yohoIndexService;
22 22
23 - @Autowired  
24 - private RunnableQueueFactory runnableQueueFactory;  
25 -  
26 23
27 /** 24 /**
28 * 定时任务重建索引（每1小时执行一次） 25 * 定时任务重建索引（每1小时执行一次）
@@ -48,7 +45,7 @@ public class IndexRebuildJob { @@ -48,7 +45,7 @@ public class IndexRebuildJob {
48 */ 45 */
49 @Scheduled(cron = "50 */10 * * * ?") 46 @Scheduled(cron = "50 */10 * * * ?")
50 public void appendIndexJob() { 47 public void appendIndexJob() {
51 - this.appendIndex(Index.social_user); 48 + //this.appendIndex(Index.social_user);
52 this.appendIndex(Index.content); 49 this.appendIndex(Index.content);
53 } 50 }
54 51
@@ -62,12 +59,16 @@ public class IndexRebuildJob { @@ -62,12 +59,16 @@ public class IndexRebuildJob {
62 return; 59 return;
63 } 60 }
64 String indexName = index.getIndexName(); 61 String indexName = index.getIndexName();
65 - runnableQueueFactory.addTask(indexName, new Runnable() {  
66 - @Override  
67 - public void run() {  
68 - yohoIndexService.rebuildIndex(indexName);  
69 - }  
70 - }); 62 + yohoIndexService.rebuildIndex(indexName);
  63 +
  64 +
  65 +
  66 +// RunnableQueueFactory.getInstance().addTask(indexName, new Runnable() {
  67 +// @Override
  68 +// public void run() {
  69 +// yohoIndexService.rebuildIndex(indexName);
  70 +// }
  71 +// });
71 } catch (Exception e) { 72 } catch (Exception e) {
72 LOGGER.error("rebuildIndex error,cause:", e); 73 LOGGER.error("rebuildIndex error,cause:", e);
73 } 74 }
@@ -84,12 +85,16 @@ public class IndexRebuildJob { @@ -84,12 +85,16 @@ public class IndexRebuildJob {
84 return; 85 return;
85 } 86 }
86 String indexName = index.getIndexName(); 87 String indexName = index.getIndexName();
87 - runnableQueueFactory.addTask(indexName, new Runnable() {  
88 - @Override  
89 - public void run() {  
90 - yohoIndexService.appendIndex(indexName);  
91 - }  
92 - }); 88 + yohoIndexService.appendIndex(indexName);
  89 +
  90 +
  91 +
  92 +// RunnableQueueFactory.getInstance().addTask(indexName, new Runnable() {
  93 +// @Override
  94 +// public void run() {
  95 +// yohoIndexService.appendIndex(indexName);
  96 +// }
  97 +// });
93 } catch (Exception e) { 98 } catch (Exception e) {
94 LOGGER.error("buildAppendedIndex error,cause:", e); 99 LOGGER.error("buildAppendedIndex error,cause:", e);
95 } 100 }
1 package com.yohomars.search.restapi; 1 package com.yohomars.search.restapi;
2 2
3 -import com.yohomars.search.index.service.IYohoIndexService;  
4 -import com.yohomars.search.queue.RunnableQueueFactory;  
5 -import com.yohomars.search.service.IndexService;  
6 -import com.yohomars.search.utils.Index; 3 +import java.util.Map;
  4 +
  5 +import javax.servlet.http.HttpServletRequest;
  6 +
7 import org.apache.commons.lang.StringUtils; 7 import org.apache.commons.lang.StringUtils;
8 import org.slf4j.Logger; 8 import org.slf4j.Logger;
9 import org.slf4j.LoggerFactory; 9 import org.slf4j.LoggerFactory;
@@ -14,8 +14,9 @@ import org.springframework.web.bind.annotation.RequestMapping; @@ -14,8 +14,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
14 import org.springframework.web.bind.annotation.RequestMethod; 14 import org.springframework.web.bind.annotation.RequestMethod;
15 import org.springframework.web.bind.annotation.ResponseBody; 15 import org.springframework.web.bind.annotation.ResponseBody;
16 16
17 -import javax.servlet.http.HttpServletRequest;  
18 -import java.util.Map; 17 +import com.yohomars.search.index.service.IYohoIndexService;
  18 +import com.yohomars.search.service.IndexService;
  19 +import com.yohomars.search.utils.Index;
19 20
20 /** 21 /**
21 * 索引管理相关请求 22 * 索引管理相关请求
@@ -29,18 +30,19 @@ public class IndexController extends BaseController { @@ -29,18 +30,19 @@ public class IndexController extends BaseController {
29 private IYohoIndexService yohoIndexService; 30 private IYohoIndexService yohoIndexService;
30 31
31 @Autowired 32 @Autowired
32 - private RunnableQueueFactory runnableQueueFactory;  
33 -  
34 - @Autowired  
35 private IndexService indexService; 33 private IndexService indexService;
36 34
37 private void rebuild(String indexName){ 35 private void rebuild(String indexName){
38 - runnableQueueFactory.addTask(indexName, new Runnable() {  
39 - @Override  
40 - public void run() {  
41 - yohoIndexService.rebuildIndex(indexName);  
42 - }  
43 - }); 36 + yohoIndexService.rebuildIndex(indexName);
  37 +
  38 +
  39 +
  40 +// RunnableQueueFactory.getInstance().addTask(indexName, new Runnable() {
  41 +// @Override
  42 +// public void run() {
  43 +// yohoIndexService.rebuildIndex(indexName);
  44 +// }
  45 +// });
44 } 46 }
45 47
46 @RequestMapping(value = "/test") 48 @RequestMapping(value = "/test")
@@ -7,7 +7,7 @@ yohomarssearch.index.translog.flush_threshold_ops=10000 @@ -7,7 +7,7 @@ yohomarssearch.index.translog.flush_threshold_ops=10000
7 7
8 #indexRebuild 8 #indexRebuild
9 search.index.batch.limit=1000 9 search.index.batch.limit=1000
10 -search.index.batch.max.thread.size=10 10 +search.index.batch.max.thread.size=5
11 11
12 #search 12 #search
13 search.minimum.should.match=30% 13 search.minimum.should.match=30%
@@ -7,7 +7,7 @@ yohomarssearch.index.translog.flush_threshold_ops=${yohomarssearch.index.translo @@ -7,7 +7,7 @@ yohomarssearch.index.translog.flush_threshold_ops=${yohomarssearch.index.translo
7 7
8 #indexRebuild 8 #indexRebuild
9 search.index.batch.limit=1000 9 search.index.batch.limit=1000
10 -search.index.batch.max.thread.size=10 10 +search.index.batch.max.thread.size=5
11 11
12 #search 12 #search
13 search.minimum.should.match=30% 13 search.minimum.should.match=30%