yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by 胡古飞
Commit 7ea1989c03e734813b19f3e1524349aea3fde629 1 parent 5eaa6bb3

批量更新时需要检查当前是否正在全量建索引

Inline Side-by-side
Showing 4 changed files with 76 additions and 99 deletions
1 package com.yoho.search.consumer.index.common.impl; 1 package com.yoho.search.consumer.index.common.impl;
2 2
  3 +import java.io.InputStream;
  4 +import java.util.ArrayList;
  5 +import java.util.List;
  6 +import java.util.Map;
  7 +import java.util.Set;
  8 +import java.util.concurrent.ConcurrentHashMap;
  9 +import java.util.regex.Pattern;
  10 +
  11 +import javax.annotation.PostConstruct;
  12 +
  13 +import org.apache.commons.lang.StringUtils;
  14 +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
  15 +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
  16 +import org.elasticsearch.action.bulk.BulkResponse;
  17 +import org.elasticsearch.action.get.GetResponse;
  18 +import org.elasticsearch.action.get.MultiGetResponse;
  19 +import org.elasticsearch.cluster.health.ClusterHealthStatus;
  20 +import org.slf4j.Logger;
  21 +import org.slf4j.LoggerFactory;
  22 +import org.springframework.beans.factory.annotation.Autowired;
  23 +import org.springframework.context.ApplicationEventPublisher;
  24 +import org.springframework.context.ApplicationEventPublisherAware;
  25 +import org.springframework.stereotype.Component;
  26 +
3 import com.yoho.search.base.utils.FileUtils; 27 import com.yoho.search.base.utils.FileUtils;
4 import com.yoho.search.base.utils.JaxbBinder; 28 import com.yoho.search.base.utils.JaxbBinder;
5 import com.yoho.search.consumer.index.common.IIndexBuilder; 29 import com.yoho.search.consumer.index.common.IIndexBuilder;
@@ -17,28 +41,6 @@ import com.yoho.search.core.es.impl.YohoIndexHelper; @@ -17,28 +41,6 @@ import com.yoho.search.core.es.impl.YohoIndexHelper;
17 import com.yoho.search.core.es.model.ESBluk; 41 import com.yoho.search.core.es.model.ESBluk;
18 import com.yoho.search.core.es.model.SearchParam; 42 import com.yoho.search.core.es.model.SearchParam;
19 import com.yoho.search.core.es.model.SearchResult; 43 import com.yoho.search.core.es.model.SearchResult;
20 -import org.apache.commons.lang.StringUtils;  
21 -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;  
22 -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;  
23 -import org.elasticsearch.action.bulk.BulkResponse;  
24 -import org.elasticsearch.action.get.GetResponse;  
25 -import org.elasticsearch.action.get.MultiGetResponse;  
26 -import org.elasticsearch.cluster.health.ClusterHealthStatus;  
27 -import org.slf4j.Logger;  
28 -import org.slf4j.LoggerFactory;  
29 -import org.springframework.beans.factory.annotation.Autowired;  
30 -import org.springframework.context.ApplicationEventPublisher;  
31 -import org.springframework.context.ApplicationEventPublisherAware;  
32 -import org.springframework.stereotype.Component;  
33 -  
34 -import javax.annotation.PostConstruct;  
35 -import java.io.InputStream;  
36 -import java.util.ArrayList;  
37 -import java.util.List;  
38 -import java.util.Map;  
39 -import java.util.Set;  
40 -import java.util.concurrent.ConcurrentHashMap;  
41 -import java.util.regex.Pattern;  
42 44
43 @Component 45 @Component
44 public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEventPublisherAware { 46 public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEventPublisherAware {
@@ -55,6 +57,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -55,6 +57,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
55 private IYohoIndexDataLoader yohoIndexDataLoader; 57 private IYohoIndexDataLoader yohoIndexDataLoader;
56 @Autowired 58 @Autowired
57 private IndexRebuildListenerMgr indexRebuildListenerMgr; 59 private IndexRebuildListenerMgr indexRebuildListenerMgr;
  60 +
58 ApplicationEventPublisher publisher; 61 ApplicationEventPublisher publisher;
59 62
60 @Override 63 @Override
@@ -138,7 +141,8 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -138,7 +141,8 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
138 141
139 @Override 142 @Override
140 public String createIndex(final String yohoIndexName, final boolean force) throws Exception { 143 public String createIndex(final String yohoIndexName, final boolean force) throws Exception {
141 - return this.createIndex(yohoIndexName, yohoIndexName, force); 144 + String yohoIndexAliasName = yohoIndexName;
  145 + return this.createIndex(yohoIndexName, yohoIndexAliasName, force);
142 } 146 }
143 147
144 @Override 148 @Override
@@ -164,7 +168,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -164,7 +168,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
164 client.removeAlias(oldRealIndexName, alias); 168 client.removeAlias(oldRealIndexName, alias);
165 } 169 }
166 // 5、添加新索引别名 170 // 5、添加新索引别名
167 - client.addAlias(alias, newRealIndexName); 171 + client.addAlias(newRealIndexName, alias);
168 // 6、删除旧索引 172 // 6、删除旧索引
169 if (oldRealIndexName != null) { 173 if (oldRealIndexName != null) {
170 client.deleteIndex(oldRealIndexName); 174 client.deleteIndex(oldRealIndexName);
@@ -281,29 +285,31 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -281,29 +285,31 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
281 Map<String, List<String>> realNameToAliasesMap = client.getRealNameToAliasesMap(); 285 Map<String, List<String>> realNameToAliasesMap = client.getRealNameToAliasesMap();
282 286
283 // 2、获取真实的索引名称 287 // 2、获取真实的索引名称
284 - String oldIndexId = yohoIndexHelper.getRealIndexName(yohoIndexName, realNameToAliasesMap);  
285 - String newIndexId = yohoIndexHelper.getRealIndexName(yohoTemplateIndexName, realNameToAliasesMap);  
286 - if (newIndexId == null) { 288 + String oldIndexRealName = yohoIndexHelper.getRealIndexName(yohoIndexName, realNameToAliasesMap);
  289 + String newIndexRealName = yohoIndexHelper.getRealIndexName(yohoTemplateIndexName, realNameToAliasesMap);
  290 + if (newIndexRealName == null) {
287 return; 291 return;
288 } 292 }
289 293
  294 + // 3、获取有货索引的别名
  295 + String yohoIndexAliasName = yohoIndexName;
  296 +
290 // 4. 把老索引的索引名先删除 297 // 4. 把老索引的索引名先删除
291 - if (oldIndexId != null) {  
292 - client.removeAlias(oldIndexId, yohoIndexName); 298 + if (oldIndexRealName != null) {
  299 + client.removeAlias(oldIndexRealName, yohoIndexAliasName);
293 } 300 }
294 301
295 // 5. 把新索引的别名全部删除 302 // 5. 把新索引的别名全部删除
296 - List<String> yohoTemplateIndexNameAlias = realNameToAliasesMap.get(newIndexId);  
297 - for (String alias : yohoTemplateIndexNameAlias) {  
298 - client.removeAlias(newIndexId, alias);  
299 - } 303 + List<String> aliasList = realNameToAliasesMap.get(newIndexRealName);
  304 + String[] aliases = aliasList.toArray(new String[aliasList.size()]);
  305 + client.removeAlias(newIndexRealName, aliases);
300 306
301 // 6. 把老索引的索引名加到新索引上面 307 // 6. 把老索引的索引名加到新索引上面
302 - client.addAlias(yohoIndexName, newIndexId); 308 + client.addAlias(newIndexRealName, yohoIndexAliasName);
303 309
304 // 7. 删除老索引 310 // 7. 删除老索引
305 - if (oldIndexId != null) {  
306 - client.deleteIndex(oldIndexId); 311 + if (oldIndexRealName != null) {
  312 + client.deleteIndex(oldIndexRealName);
307 } 313 }
308 314
309 } 315 }
@@ -419,22 +425,6 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -419,22 +425,6 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
419 client.addIndexData(realIndexName, yohoIndexName, id, data); 425 client.addIndexData(realIndexName, yohoIndexName, id, data);
420 } 426 }
421 427
422 - // @Override  
423 - // public void addIndexData(final String yohoIndexName, final  
424 - // List<Map<String, Object>> dataList) throws Exception {  
425 - // IYohoIndex index = this.nameToIndexMap.get(yohoIndexName);  
426 - // if (index == null) {  
427 - // return;  
428 - // }  
429 - // IElasticsearchClient client = index.getIndexClient();  
430 - // String realIndexName = yohoIndexHelper.getRealIndexName(yohoIndexName,  
431 - // client);  
432 - // if (StringUtils.isBlank(realIndexName)) {  
433 - // return;  
434 - // }  
435 - // client.addIndexData(realIndexName, yohoIndexName, dataList);  
436 - // }  
437 -  
438 @Override 428 @Override
439 public void deleteIndexData(final String yohoIndexName, final String id) throws Exception { 429 public void deleteIndexData(final String yohoIndexName, final String id) throws Exception {
440 IYohoIndex index = this.nameToIndexMap.get(yohoIndexName); 430 IYohoIndex index = this.nameToIndexMap.get(yohoIndexName);
@@ -449,22 +439,6 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -449,22 +439,6 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
449 client.deleteIndexData(realIndexName, yohoIndexName, id); 439 client.deleteIndexData(realIndexName, yohoIndexName, id);
450 } 440 }
451 441
452 - // @Override  
453 - // public void deleteIndexDataByQuery(final String yohoIndexName, final  
454 - // String query) throws Exception {  
455 - // IYohoIndex index = this.nameToIndexMap.get(yohoIndexName);  
456 - // if (index == null) {  
457 - // return;  
458 - // }  
459 - // IElasticsearchClient client = index.getIndexClient();  
460 - // String realIndexName = yohoIndexHelper.getRealIndexName(yohoIndexName,  
461 - // client);  
462 - // if (StringUtils.isBlank(realIndexName)) {  
463 - // return;  
464 - // }  
465 - // client.deleteIndexDataByQuery(realIndexName, yohoIndexName, query);  
466 - // }  
467 -  
468 @Override 442 @Override
469 public void updateIndexData(final String yohoIndexName, final String id, final Object data) throws Exception { 443 public void updateIndexData(final String yohoIndexName, final String id, final Object data) throws Exception {
470 IYohoIndex index = this.nameToIndexMap.get(yohoIndexName); 444 IYohoIndex index = this.nameToIndexMap.get(yohoIndexName);
@@ -566,6 +540,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent @@ -566,6 +540,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
566 if (index == null) { 540 if (index == null) {
567 return null; 541 return null;
568 } 542 }
  543 + rebuildFlagService.waitingRebuildingIndex();
569 IElasticsearchClient client = index.getIndexClient(); 544 IElasticsearchClient client = index.getIndexClient();
570 BulkResponse bulkResponse = client.bulk(esBluks); 545 BulkResponse bulkResponse = client.bulk(esBluks);
571 if (bulkResponse.hasFailures()) { 546 if (bulkResponse.hasFailures()) {
1 package com.yoho.search.consumer.index.increment; 1 package com.yoho.search.consumer.index.increment;
2 2
3 -import com.alibaba.fastjson.JSONArray; 3 +import java.util.ArrayList;
  4 +import java.util.HashMap;
  5 +import java.util.List;
  6 +import java.util.Map;
  7 +import java.util.UUID;
  8 +
  9 +import org.apache.commons.collections.CollectionUtils;
  10 +import org.apache.commons.lang.StringUtils;
  11 +import org.slf4j.Logger;
  12 +import org.slf4j.LoggerFactory;
  13 +import org.springframework.amqp.core.Message;
  14 +import org.springframework.amqp.rabbit.core.ChannelAwareMessageListener;
  15 +import org.springframework.beans.factory.annotation.Autowired;
  16 +import org.springframework.stereotype.Component;
  17 +
4 import com.alibaba.fastjson.JSONObject; 18 import com.alibaba.fastjson.JSONObject;
5 import com.rabbitmq.client.Channel; 19 import com.rabbitmq.client.Channel;
6 import com.yoho.error.event.SearchEvent; 20 import com.yoho.error.event.SearchEvent;
@@ -10,26 +24,13 @@ import com.yoho.search.base.utils.ISearchConstants; @@ -10,26 +24,13 @@ import com.yoho.search.base.utils.ISearchConstants;
10 import com.yoho.search.consumer.common.CostStatistics; 24 import com.yoho.search.consumer.common.CostStatistics;
11 import com.yoho.search.consumer.index.increment.bulks.StorageSkuIndexBulkService; 25 import com.yoho.search.consumer.index.increment.bulks.StorageSkuIndexBulkService;
12 import com.yoho.search.consumer.index.increment.rule.AbstractStorageRelatedMqListener; 26 import com.yoho.search.consumer.index.increment.rule.AbstractStorageRelatedMqListener;
13 -import com.yoho.search.consumer.service.base.GoodsService;  
14 import com.yoho.search.consumer.service.base.ProductService; 27 import com.yoho.search.consumer.service.base.ProductService;
15 import com.yoho.search.consumer.service.base.StorageService; 28 import com.yoho.search.consumer.service.base.StorageService;
16 -import com.yoho.search.consumer.service.bo.ProductGoodsBO;  
17 -import com.yoho.search.consumer.service.logic.productIndex.ProductGoodsLogicService;  
18 import com.yoho.search.consumer.service.logic.productIndex.StorageUpdateTimeLogicService; 29 import com.yoho.search.consumer.service.logic.productIndex.StorageUpdateTimeLogicService;
19 import com.yoho.search.core.es.utils.IgnoreSomeException; 30 import com.yoho.search.core.es.utils.IgnoreSomeException;
20 import com.yoho.search.dal.model.Product; 31 import com.yoho.search.dal.model.Product;
21 import com.yoho.search.dal.model.Storage; 32 import com.yoho.search.dal.model.Storage;
22 import com.yoho.search.dal.model.StorageUpdateTime; 33 import com.yoho.search.dal.model.StorageUpdateTime;
23 -import org.apache.commons.collections.CollectionUtils;  
24 -import org.apache.commons.lang.StringUtils;  
25 -import org.slf4j.Logger;  
26 -import org.slf4j.LoggerFactory;  
27 -import org.springframework.amqp.core.Message;  
28 -import org.springframework.amqp.rabbit.core.ChannelAwareMessageListener;  
29 -import org.springframework.beans.factory.annotation.Autowired;  
30 -import org.springframework.stereotype.Component;  
31 -  
32 -import java.util.*;  
33 34
34 @Component 35 @Component
35 public class StorageMqListener extends AbstractStorageRelatedMqListener implements ChannelAwareMessageListener { 36 public class StorageMqListener extends AbstractStorageRelatedMqListener implements ChannelAwareMessageListener {
1 package com.yoho.search.consumer.index.increment; 1 package com.yoho.search.consumer.index.increment;
2 2
  3 +import java.util.ArrayList;
  4 +import java.util.List;
  5 +import java.util.Map;
  6 +import java.util.UUID;
  7 +
  8 +import org.apache.commons.collections.CollectionUtils;
  9 +import org.apache.commons.lang.StringUtils;
  10 +import org.slf4j.Logger;
  11 +import org.slf4j.LoggerFactory;
  12 +import org.springframework.amqp.core.Message;
  13 +import org.springframework.amqp.rabbit.core.ChannelAwareMessageListener;
  14 +import org.springframework.beans.factory.annotation.Autowired;
  15 +import org.springframework.context.ApplicationEventPublisher;
  16 +import org.springframework.stereotype.Component;
  17 +
3 import com.alibaba.fastjson.JSONObject; 18 import com.alibaba.fastjson.JSONObject;
4 import com.rabbitmq.client.Channel; 19 import com.rabbitmq.client.Channel;
5 import com.yoho.error.event.SearchEvent; 20 import com.yoho.error.event.SearchEvent;
@@ -14,20 +29,6 @@ import com.yoho.search.consumer.service.logic.TblShopsLogicService; @@ -14,20 +29,6 @@ import com.yoho.search.consumer.service.logic.TblShopsLogicService;
14 import com.yoho.search.core.es.model.ESBluk; 29 import com.yoho.search.core.es.model.ESBluk;
15 import com.yoho.search.core.es.utils.IgnoreSomeException; 30 import com.yoho.search.core.es.utils.IgnoreSomeException;
16 import com.yoho.search.dal.model.TblBrand; 31 import com.yoho.search.dal.model.TblBrand;
17 -import org.apache.commons.collections.CollectionUtils;  
18 -import org.apache.commons.lang.StringUtils;  
19 -import org.slf4j.Logger;  
20 -import org.slf4j.LoggerFactory;  
21 -import org.springframework.amqp.core.Message;  
22 -import org.springframework.amqp.rabbit.core.ChannelAwareMessageListener;  
23 -import org.springframework.beans.factory.annotation.Autowired;  
24 -import org.springframework.context.ApplicationEventPublisher;  
25 -import org.springframework.stereotype.Component;  
26 -  
27 -import java.util.ArrayList;  
28 -import java.util.List;  
29 -import java.util.Map;  
30 -import java.util.UUID;  
31 32
32 /** 33 /**
33 * Created by wangnan on 2016/12/16. 34 * Created by wangnan on 2016/12/16.
@@ -27,11 +27,11 @@ public class RebuildFlagService { @@ -27,11 +27,11 @@ public class RebuildFlagService {
27 rebuildFlag.setRebuilding(true); 27 rebuildFlag.setRebuilding(true);
28 } catch (Exception e) { 28 } catch (Exception e) {
29 logger.error(e.getMessage(), e); 29 logger.error(e.getMessage(), e);
30 - } finally { 30 + }finally{
31 writeLock.unlock(); 31 writeLock.unlock();
32 } 32 }
33 } 33 }
34 - 34 +
35 public void updateIsBuildingFalse() { 35 public void updateIsBuildingFalse() {
36 try { 36 try {
37 writeLock.lock(); 37 writeLock.lock();
@@ -39,7 +39,7 @@ public class RebuildFlagService { @@ -39,7 +39,7 @@ public class RebuildFlagService {
39 rebuildFlag.setRebuilding(false); 39 rebuildFlag.setRebuilding(false);
40 } catch (Exception e) { 40 } catch (Exception e) {
41 logger.error(e.getMessage(), e); 41 logger.error(e.getMessage(), e);
42 - } finally { 42 + }finally{
43 writeLock.unlock(); 43 writeLock.unlock();
44 } 44 }
45 } 45 }