yoho-search / yohomars-search-service · Commits

Go to a project

Authored by wangnan
Commit 173184192e66f350f191934cd18742b8f7f64b32 1 parent 37c7e676

整理代码

Inline Side-by-side
Showing 8 changed files with 8 additions and 338 deletions
package com.yohomars.search.index.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.yoho.error.event.SearchEvent;
import com.yohomars.search.es.IElasticsearchClient;
import com.yohomars.search.index.builder.IIndexBuilder;
import com.yohomars.search.index.model.ESBluk;
import com.yohomars.search.utils.EventReportEnum;
import com.yohomars.search.utils.ISearchConstans;
import com.yohomars.search.utils.PerformanceMonitor;
import org.elasticsearch.action.bulk.BulkResponse;
... ... @@ -14,8 +12,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
... ... @@ -26,14 +22,12 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@Component
public class YohoIndexDataLoader implements ApplicationEventPublisherAware, ApplicationContextAware {
public class YohoIndexDataLoader implements ApplicationContextAware {
private final Logger INDEX_REBUILD_LOG = LoggerFactory.getLogger("INDEX_REBULDER");
private static final ExecutorService threadPool = Executors.newFixedThreadPool(ISearchConstans.SEARCH_INDEX_BATCH_MAX_THREAD_SIZE);
ApplicationEventPublisher publisher;
ApplicationContext applicationContext;
private PerformanceMonitor performanceMonitor = new PerformanceMonitor("YohoIndexDataLoader",0,INDEX_REBUILD_LOG);
... ... @@ -119,9 +113,6 @@ public class YohoIndexDataLoader implements ApplicationEventPublisherAware, Appl
// BulkResponse bulkResponse = client.addIndexDataBean(tempIndexRealName, yohoIndexName, dataList);
if(bulkResponse.hasFailures()){
Exception e = new Exception(String.format("addIndexDataBean has fail,[yohoIndexName=[%s]],[pageNo=%s],[failureMessage=%s]", yohoIndexName,pageNo,bulkResponse.buildFailureMessage()));
publisher.publishEvent(new SearchEvent(EventReportEnum.YOHOINDEXDATALOADER_DOLOADDATA.getEventName(),
EventReportEnum.YOHOINDEXDATALOADER_DOLOADDATA.getFunctionName(),
EventReportEnum.YOHOINDEXDATALOADER_DOLOADDATA.getMoudleName(),"exception",e,null));
throw e;
}
long end = System.currentTimeMillis();
... ... @@ -143,13 +134,6 @@ public class YohoIndexDataLoader implements ApplicationEventPublisherAware, Appl
private void handelException(String yohoIndexName, int pageNo, Exception e) {
// 1、上报异常至日志
INDEX_REBUILD_LOG.error(e.getMessage(), e);
// 2、上报异常至influxdb
// LogEvent event = new LogEvent.Builder("search_consumer").catalog("search_consumer").addArg("ip", LocalIp.getLocalIp()).addArg("yohoIndexName", yohoIndexName).addArg("pageNo", pageNo)
// .addArg("exception", e.getMessage()).build();
// publisher.publishEvent(event);
publisher.publishEvent(new SearchEvent(EventReportEnum.YOHOINDEXDATALOADER_HANDELEXCEPTION.getEventName(),
EventReportEnum.YOHOINDEXDATALOADER_HANDELEXCEPTION.getFunctionName(),
EventReportEnum.YOHOINDEXDATALOADER_HANDELEXCEPTION.getMoudleName(),"exception",e,"yohoIndexName="+yohoIndexName+"/pageNo="+pageNo));
}
@Override
... ... @@ -157,8 +141,4 @@ public class YohoIndexDataLoader implements ApplicationEventPublisherAware, Appl
this.applicationContext = applicationContext;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.publisher = applicationEventPublisher;
}
}
... ...
... ... @@ -27,8 +27,6 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
... ... @@ -41,7 +39,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
@Component
public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEventPublisherAware {
public class YohoIndexServiceImpl implements IYohoIndexService {
private final Logger logger = LoggerFactory.getLogger(YohoIndexServiceImpl.class);
... ... @@ -54,12 +52,7 @@ public class YohoIndexServiceImpl implements IYohoIndexService, ApplicationEvent
@Autowired
private YohoIndexDataLoader yohoIndexDataLoader;
ApplicationEventPublisher publisher;
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.publisher = applicationEventPublisher;
}
// 索引配置文件
private String configFile = "index.xml";
... ...
package com.yohomars.search.utils;
public enum EventReportEnum {
CUSTOMSIMPLECANALCLIENT_AFTERPROPERTIESSET("afterPropertiesSet","CustomSimpleCanalClient.afterPropertiesSet", MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_STARTSHOPCLIENT("startShopClient","CustomSimpleCanalClient.startShopClient",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_STARTOPERATIONCLIENT("startOperationClient","CustomSimpleCanalClient.startOperationClient",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_STARTERPPRODUCTCLIENT("startErpproductClient","CustomSimpleCanalClient.startErpproductClient",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_HANDLECANALMESSAGE("handleCanalMessage","CustomSimpleCanalClient.handleCanalMessage",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_PUBLISHTOREDIS("publishToRedis","CustomSimpleCanalClient.publishToRedis",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_GETDELETEDATA("getDeleteData","CustomSimpleCanalClient.getDeleteData",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_GETUPDATEDATA("getUpdateData","CustomSimpleCanalClient.getUpdateData",MoudleEnum.producer),
CUSTOMSIMPLECANALCLIENT_GETDATA("getData","CustomSimpleCanalClient.getData",MoudleEnum.producer),
INDEXREBUILDJOB_REBUILDINDEXWITHLOG("rebuildIndexWithlog","IndexRebuildJob.rebuildIndexWithlog",MoudleEnum.consumer),
TPLADAPTORJOB_GETTPLPRODUCTCOUNT("getTplProductCount","TplAdaptorJob.getTplProductCount",MoudleEnum.consumer),
TPLADAPTORJOB_GETPRODUCTINDEXLIST("getProductIndexList","TplAdaptorJob.getProductIndexList",MoudleEnum.consumer),
TPLADAPTORJOB_ADDPRODUCTINDEXINTOES("addProductIndexIntoEs","TplAdaptorJob.addProductIndexIntoEs",MoudleEnum.consumer),
ABSTRACTMQLISTENER_WAITINGREBUILDINGINDEX("waitingRebuildingIndex","AbstractMqListener.waitingRebuildingIndex",MoudleEnum.consumer),
ABSTRACTMQLISTENER_UPDATEPRODUCTINDEXWITHDATAMAP("updateProductIndexWithDataMap","AbstractMqListener.updateProductIndexWithDataMap",MoudleEnum.consumer),
ABSTRACTMQLISTENER_SENDPRODUCTINDEXMESSAGE("sendProductIndexMessage","AbstractMqListener.sendProductIndexMessage",MoudleEnum.consumer),
ACTIVITYPRODUCTMQLISTENER_ONMESSAGE("onMessage","ActivityProductMqListener.onMessage",MoudleEnum.consumer),
BRANDMQLISTENER_ONMESSAGE("onMessage","BrandMqListener.onMessage",MoudleEnum.consumer),
GOODSIMAGESMQLISTENER_ONMESSAGE("onMessage","GoodsImagesMqListener.onMessage",MoudleEnum.consumer),
GOODSMQLISTENER_ONMESSAGE("onMessage","GoodsMqListener.onMessage",MoudleEnum.consumer),
PARAMETERMAKEMQLISTENER_ONMESSAGE("onMessage","ParameterMakeMqListener.onMessage",MoudleEnum.consumer),
PRODUCTACTIVITIESLINKMQLISTENER_ONMESSAGE("onMessage","ProductActivitiesLinkMqListener.onMessage",MoudleEnum.consumer),
PRODUCTCOLORMQLISTENER_ONMESSAGE("onMessage","ProductColorMqListener.onMessage",MoudleEnum.consumer),
PRODUCTINDEXMQLISTENER_ONMESSAGE("onMessage","ProductIndexMqListener.onMessage",MoudleEnum.consumer),
PRODUCTKEYWORDSMQLISTENER_ONMESSAGE("onMessage","ProductKeywordsMqListener.onMessage",MoudleEnum.consumer),
PRODUCTMQLISTENER_ONMESSAGE("onMessage","ProductMqListener.onMessage",MoudleEnum.consumer),
PRODUCTPOOLDETAILMQLISTENER_ONMESSAGE("onMessage","ProductPoolDetailMqListener.onMessage",MoudleEnum.consumer),
PRODUCTMQLISTENER_UPDATEDATA("updateData","ProductMqListener.updateData",MoudleEnum.consumer),
PRODUCTPRICEMQLISTENER_ONMESSAGE("onMessage","ProductPriceMqListener.onMessage",MoudleEnum.consumer),
PRODUCTSEARCHMQLISTENER_ONMESSAGE("onMessage","ProductSearchMqListener.onMessage",MoudleEnum.consumer),
PRODUCTSORTMQLISTENER_ONMESSAGE("onMessage","ProductSortMqListener.onMessage",MoudleEnum.consumer),
PRODUCTSTANDARDRELATIONMQLISTENER_ONMESSAGE("onMessage","ProductStandardRelationMqListener.onMessage",MoudleEnum.consumer),
PRODUCTSTYLERELATIONMQLISTENER_ONMESSAGE("onMessage","ProductStyleRelationMqListener.onMessage",MoudleEnum.consumer),
SEARCHACTIONMQLISTENER_ONMESSAGE("onMessage","SearchActionMqListener.onMessage",MoudleEnum.consumer),
SIZEMQLISTENER_ONMESSAGE("onMessage","SizeMqListener.onMessage",MoudleEnum.consumer),
STORAGEMQLISTENER_ONMESSAGE("onMessage","StorageMqListener.onMessage",MoudleEnum.consumer),
STYLEMQLISTENER_ONMESSAGE("onMessage","StyleMqListener.onMessage",MoudleEnum.consumer),
YOHOODPRODUCTMQLISTENER_ONMESSAGE("onMessage","YohoodProductMqListener.onMessage",MoudleEnum.consumer),
YOHOODPRODUCTMQLISTENER_UPDATEINDEX("updateIndex","YohoodProductMqListener.updateIndex",MoudleEnum.consumer),
STANDARDMQLISTENER_ONMESSAGE("onMessage","StandardMqListener.onMessage",MoudleEnum.consumer),
STORAGESKUINDEXSERVICE_UPDATESTORAGESKUINDEXBYPRODUCTID("updateStorageSkuIndexByProductId","StorageSkuIndexService.updateStorageSkuIndexByProductId",MoudleEnum.consumer),
STORAGESKUINDEXSERVICE_UPDATESTORAGESKUINDEX("updateStorageSkuIndex","StorageSkuIndexService.updateStorageSkuIndex",MoudleEnum.consumer),
INDEXCONTROLLER_INDEX_CREATE("/index/create","IndexController.create",MoudleEnum.consumer),
INDEXCONTROLLER_INDEX_EXIST("/index/exist","IndexController.exist",MoudleEnum.consumer),
INDEXCONTROLLER_INDEX_REBUILD("/index/rebuild","IndexController.rebuild",MoudleEnum.consumer),
INDEXCONTROLLER_INDEX_UPDATE("/index/update","IndexController.update",MoudleEnum.consumer),
BRANDCONTROLLER_BRAND_1("/brand/{brandId}","BrandController.brand",MoudleEnum.service),
BRANDCONTROLLER_BRAND_2("/brand/{start}/{size}","BrandController.brand",MoudleEnum.service),
BRANDCONTROLLER_BRAND_LIST("/brand/list","BrandController.list",MoudleEnum.service),
SEARCHCONTROLLER_SEARCHOLD("/searchOld","SearchController.searchProducts",MoudleEnum.service),
SEARCHCONTROLLER_SEARCH("/search","SearchController.searchProductsNew",MoudleEnum.service),
SEARCHCONTROLLER_DISCOUNT("/discount","SearchController.discount",MoudleEnum.service),
SEARCHCONTROLLER_RECENT("/recent","SearchController.recent",MoudleEnum.service),
SEARCHCONTROLLER_NEW_SHELVE("/new-shelve","SearchController.searchNewestProductWithDiffBrand",MoudleEnum.service),
SEARCHCONTROLLER_NEW_PRODUCT("/new_product","SearchController.searchNewestProductWithParamBrand",MoudleEnum.service),
SEARCHCONTROLLER_BRANDS("/brands","SearchController.brands",MoudleEnum.service),
SEARCHCONTROLLER_GROUP_BRANDS("/group_brands","SearchController.groupBrands",MoudleEnum.service),
SEARCHCONTROLLER_SORTGROUP("/sortgroup","SearchController.sortGroup",MoudleEnum.service),
SEARCHCONTROLLER_COUNT("/count","SearchController.searchCount",MoudleEnum.service),
SEARCHCONTROLLER_SUGGEST("/suggest","SearchController.suggest",MoudleEnum.service),
SEARCHCONTROLLER_SHOPS("/shops","SearchController.shops",MoudleEnum.service),
SEARCHCONTROLLER_GROUP_SHOPS("/group_shops","SearchController.groupShops",MoudleEnum.service),
SEARCHCONTROLLER_SORT_SIZE_PRODUCTS("/sort_size_products","SearchController.sort_size_products",MoudleEnum.service),
TBLPRODUCTCONTROLLER_TBLPRODUCT_SEARCH("/tblproduct/search","TblProductController.search",MoudleEnum.service),
TBLPRODUCTCONTROLLER_TBLPRODUCT_UPDATE("/tblproduct/update/{product_skn}","TblProductController.updateIndex",MoudleEnum.service),
PRODUCTINDEXSERVICEIMPL_GETSEARCHAPIRESULT("getSearchApiResult","ProductIndexServiceImpl.getSearchApiResult",MoudleEnum.service),
AGGREGATIONSERVICE_GETAGGNAMEANDRESPONSE("getAggNameAndResponse","AggregationService.getAggNameAndResponse",MoudleEnum.service),
AGGREGATIONSERVICE_GETAGGNAMEANDRESPONSEWITHOUTCACHE("getAggNameAndResponseWithOutCache","AggregationService.getAggNameAndResponseWithOutCache",MoudleEnum.service),
SEARCHCOMMONSERVICE_DOSEARCH("doSearch","SearchCommonService.doSearch",MoudleEnum.service),
YOHOINDEXSERVICEIMPL_REBUILD("rebuild","YohoIndexServiceImpl.rebuild",MoudleEnum.core),
YOHOINDEXSERVICEIMPL_CONFIGURE("configure","YohoIndexServiceImpl.configure",MoudleEnum.core),
DATASYNTASKTHREAD_RUN("run","DataSynTaskThread.run",MoudleEnum.core),
YOHOINDEXIMPL_GETINDEXBUILDER("getIndexBuilder","YohoIndexImpl.getIndexBuilder",MoudleEnum.core),
YOHOINDEXDATALOADER_DOLOADDATA("doLoadData","YohoIndexDataLoader.doLoadData",MoudleEnum.core),
YOHOINDEXDATALOADER_HANDELEXCEPTION("handelException","YohoIndexDataLoader.handelException",MoudleEnum.core);
private String eventName;
private String functionName;
private String moudleName;
EventReportEnum(String eventName,String functionName,String moudleName){
this.eventName=eventName;
this.functionName=functionName;
this.moudleName=moudleName;
}
public String getEventName() {
return eventName;
}
public String getFunctionName() {
return functionName;
}
public String getMoudleName() {
return moudleName;
}
}
package com.yoho.search.dal.model;
public class SearchAction {
private Integer id;
private Integer productSkn;
private Integer objectId;
private Integer pid;
private Byte status;
private Integer createTime;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public Integer getProductSkn() {
return productSkn;
}
public void setProductSkn(Integer productSkn) {
this.productSkn = productSkn;
}
public Integer getObjectId() {
return objectId;
}
public void setObjectId(Integer objectId) {
this.objectId = objectId;
}
public Integer getPid() {
return pid;
}
public void setPid(Integer pid) {
this.pid = pid;
}
public Byte getStatus() {
return status;
}
public void setStatus(Byte status) {
this.status = status;
}
public Integer getCreateTime() {
return createTime;
}
public void setCreateTime(Integer createTime) {
this.createTime = createTime;
}
}
\ No newline at end of file
package com.yoho.search.dal.model;
import java.io.Serializable;
public class SearchAdmin implements Serializable {
/**
*
*/
private static final long serialVersionUID = 1L;
private Long id;
private String sceneCode;
private String fieldName1;
private Long weight1;
private String fieldName2;
private Long weight2;
private String fieldName3;
private Long weight3;
private String fieldName4;
private Long weight4;
private String fieldName5;
private Long weight5;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getSceneCode() {
return sceneCode;
}
public void setSceneCode(String sceneCode) {
this.sceneCode = sceneCode == null ? null : sceneCode.trim();
}
public String getFieldName1() {
return fieldName1;
}
public void setFieldName1(String fieldName1) {
this.fieldName1 = fieldName1 == null ? null : fieldName1.trim();
}
public Long getWeight1() {
return weight1;
}
public void setWeight1(Long weight1) {
this.weight1 = weight1;
}
public String getFieldName2() {
return fieldName2;
}
public void setFieldName2(String fieldName2) {
this.fieldName2 = fieldName2 == null ? null : fieldName2.trim();
}
public Long getWeight2() {
return weight2;
}
public void setWeight2(Long weight2) {
this.weight2 = weight2;
}
public String getFieldName3() {
return fieldName3;
}
public void setFieldName3(String fieldName3) {
this.fieldName3 = fieldName3 == null ? null : fieldName3.trim();
}
public Long getWeight3() {
return weight3;
}
public void setWeight3(Long weight3) {
this.weight3 = weight3;
}
public String getFieldName4() {
return fieldName4;
}
public void setFieldName4(String fieldName4) {
this.fieldName4 = fieldName4 == null ? null : fieldName4.trim();
}
public Long getWeight4() {
return weight4;
}
public void setWeight4(Long weight4) {
this.weight4 = weight4;
}
public String getFieldName5() {
return fieldName5;
}
public void setFieldName5(String fieldName5) {
this.fieldName5 = fieldName5 == null ? null : fieldName5.trim();
}
public Long getWeight5() {
return weight5;
}
public void setWeight5(Long weight5) {
this.weight5 = weight5;
}
}
\ No newline at end of file
... ... @@ -24,17 +24,17 @@ public class IndexRebuildJob {
/**
* 定时任务重建所有索引（每5分钟执行一次）
* 定时任务重建所有索引（每10分钟执行一次）
*/
@Scheduled(cron = "0 0/10 * * * ?")
public void execute() {
long begin = System.currentTimeMillis();
logger.info("indexRebuildJob execute start----[begin={}]", begin);
this.rebuildIndex(ISearchConstans.INDEX_NAME_BIZAREA);
this.rebuildIndex(ISearchConstans.INDEX_NAME_COMMENT);
this.rebuildIndex(ISearchConstans.INDEX_NAME_LINE);
this.rebuildIndex(ISearchConstans.INDEX_NAME_STORE);
this.rebuildIndex(ISearchConstans.INDEX_NAME_TOPIC);
//this.rebuildIndex(ISearchConstans.INDEX_NAME_BIZAREA);
//this.rebuildIndex(ISearchConstans.INDEX_NAME_COMMENT);
//this.rebuildIndex(ISearchConstans.INDEX_NAME_LINE);
//this.rebuildIndex(ISearchConstans.INDEX_NAME_TOPIC);
logger.info("indexRebuildJob execute end----[end={}][cost={}ms]", System.currentTimeMillis(), (System.currentTimeMillis() - begin));
}
... ...
... ... @@ -10,24 +10,17 @@ import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.suggest.SuggestResponse;
import org.elasticsearch.search.suggest.Suggest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.stereotype.Service;
import java.util.*;
@Service
public class SearchCommonService implements ApplicationEventPublisherAware {
public class SearchCommonService{
@Autowired
private IYohoIndexService yohoIndexService;
private ApplicationEventPublisher publisher;
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.publisher = applicationEventPublisher;
}
private void publishSearchResultEvent(final String indexName, SearchParam searchParam, SearchResult searchResult) {
if (searchParam == null || searchResult == null) {
... ...
... ... @@ -429,7 +429,6 @@ public class SearchServiceHelper {
store.setStoreName("name");
store.setLatitude(123.2);
store.setLongitude(321.1);
jsonBuild.startObject().field("id", store.getId()).field("name", store.getStoreName()).startArray("location").value(store.getLatitude()).value(store.getLongitude()).endArray()
.endObject();
jsonData = jsonBuild.string();
... ...