yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by Gino Zhang
Commit 3f7e958f0e397cdb9ebd1b936b869d6ccea13c0e 1 parent 6eec50b4

SuggestWordCustom增量更新ES数据

Inline Side-by-side
Showing 2 changed files with 60 additions and 7 deletions
... ... @@ -97,7 +97,7 @@ public class SuggestConversionCustomMqListener extends AbstractMqListener implem
if (sourceData != null) {
// 删除数据
suggestConversionCustomService.delete(Integer.valueOf(id));
logger.info("[func=deleteData][step=success][tableName={}][id={}][cost={}ms]", tableName, id, (System.currentTimeMillis() - begin));
logger.info("[func=deleteData][step=success][tableName={}][id={}][cost={}ms]", tableName, id, System.currentTimeMillis() - begin);
if ((suggestConversion = suggestConversionService.selectBySource(sourceData.getSource())) != null) {
// 还原成自动爬虫生成的数据
... ... @@ -107,10 +107,10 @@ public class SuggestConversionCustomMqListener extends AbstractMqListener implem
dataMap.put("updateTime", suggestConversion.getUpdateTime());
dataMap.put("status", SuggestionConstants.VALID_STATUS);
indexService.updateIndexData(ISearchConstants.INDEX_NAME_CONVERSION, MD5Util.string2MD5(sourceData.getSource().trim().toLowerCase()), dataMap);
logger.info("[func=updateDataToES][step=success][tableName={}][id={}][cost={}ms]", tableName, id, (System.currentTimeMillis() - begin));
logger.info("[func=updateDataToES][step=success][tableName={}][id={}][cost={}ms]", tableName, id, System.currentTimeMillis() - begin);
} else {
indexService.deleteIndexData(ISearchConstants.INDEX_NAME_CONVERSION, MD5Util.string2MD5(sourceData.getSource().trim().toLowerCase()));
logger.info("[func=deleteDataFromES][step=success][tableName={}][id={}][cost={}ms]", tableName, id, (System.currentTimeMillis() - begin));
logger.info("[func=deleteDataFromES][step=success][tableName={}][id={}][cost={}ms]", tableName, id, System.currentTimeMillis() - begin);
}
}
}
... ...
... ... @@ -3,13 +3,16 @@ package com.yoho.search.consumer.index.increment;
import com.alibaba.fastjson.JSONObject;
import com.rabbitmq.client.Channel;
import com.yoho.error.event.SearchEvent;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.*;
import com.yoho.search.consumer.index.common.IYohoIndexService;
import com.yoho.search.consumer.service.base.SuggestWordCustomService;
import com.yoho.search.consumer.service.base.SuggestWordDefService;
import com.yoho.search.consumer.suggests.common.KeywordType;
import com.yoho.search.consumer.suggests.common.SuggestionConstants;
import com.yoho.search.consumer.suggests.counter.KeywordCounterService;
import com.yoho.search.core.es.utils.IgnoreSomeException;
import com.yoho.search.dal.model.SuggestWordCustom;
import com.yoho.search.dal.model.SuggestWordDef;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
... ... @@ -18,6 +21,7 @@ import org.springframework.amqp.rabbit.core.ChannelAwareMessageListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.Map;
/**
... ... @@ -32,6 +36,12 @@ public class SuggestWordCustomMqListener extends AbstractMqListener implements C
private SuggestWordCustomService suggestWordCustomService;
@Autowired
private SuggestWordDefService suggestWordDefService;
@Autowired
private KeywordCounterService keywordCounterService;
@Autowired
private IYohoIndexService indexService;
@Override
... ... @@ -67,10 +77,30 @@ public class SuggestWordCustomMqListener extends AbstractMqListener implements C
if (suggestWordCustom == null || suggestWordCustom.getId() == null) {
return;
}
// 更新数据库
suggestWordCustomService.saveOrUpdate(suggestWordCustom);
logger.info("[func=updateData][step=success][tableName={}][id={}][cost={}ms]", tableName, suggestWordCustom.getId(), (System.currentTimeMillis() - begin));
// 更新索引
String standardKeyword = CharUtils.standardized(suggestWordCustom.getKeyword());
Map<String, Object> dataMap = new HashMap<>();
dataMap.put("keyword", suggestWordCustom.getKeyword());
dataMap.put("standardKeyword", standardKeyword);
dataMap.put("type", KeywordType.Customized.getType());
dataMap.put("weight", KeywordType.Customized.getWeightValue());
dataMap.put("status", suggestWordCustom.getStatus());
if (SuggestionConstants.VALID_STATUS.equals(suggestWordCustom.getStatus())) {
// 获取count
SuggestWordDef suggestWordDef = keywordCounterService.countKeyword(suggestWordCustom.getKeyword());
dataMap.put("count", suggestWordDef.getCount());
dataMap.put("countForApp", suggestWordDef.getCountForApp());
dataMap.put("countForBlk", suggestWordDef.getCountForBlk());
}
indexService.updateIndexData(ISearchConstants.INDEX_NAME_SUGGEST, MD5Util.string2MD5(standardKeyword), dataMap);
logger.info("[func=updateDataToES][step=success][tableName={}][keyword={}][cost={}ms]", tableName, suggestWordCustom.getKeyword(), (System.currentTimeMillis() - begin));
}
public void deleteData(final String id, final String tableName) throws Exception {
... ... @@ -79,11 +109,34 @@ public class SuggestWordCustomMqListener extends AbstractMqListener implements C
}
long begin = System.currentTimeMillis();
SuggestWordDef suggestWordDef;
SuggestWordCustom sourceData = suggestWordCustomService.selectByPrimaryKey(Integer.valueOf(id));
if (sourceData != null) {
// 删除数据
suggestWordCustomService.delete(Integer.valueOf(id));
logger.info("[func=deleteData][step=success][tableName={}][id={}][cost={}ms]", tableName, id, (System.currentTimeMillis() - begin));
logger.info("[func=deleteData][step=success][tableName={}][id={}][cost={}ms]", tableName, id, System.currentTimeMillis() - begin);
String keyword = sourceData.getKeyword();
String standardKeyword = CharUtils.standardized(keyword);
if ((suggestWordDef = suggestWordDefService.selectByKeyword(sourceData.getKeyword())) != null && SuggestionConstants.VALID_STATUS.equals(suggestWordDef.getStatus())) {
// 还原成原始自动生成的数据
Map<String, Object> dataMap = new HashMap<>();
SuggestWordDef suggestWordCounter = keywordCounterService.countKeyword(keyword);
dataMap.put("count", suggestWordCounter.getCount());
dataMap.put("countForApp", suggestWordCounter.getCountForApp());
dataMap.put("countForBlk", suggestWordCounter.getCountForBlk());
dataMap.put("keyword", keyword);
dataMap.put("standardKeyword", standardKeyword);
dataMap.put("type", suggestWordDef.getType());
dataMap.put("weight", suggestWordDef.getWeight());
dataMap.put("status", suggestWordDef.getStatus());
indexService.updateIndexData(ISearchConstants.INDEX_NAME_CONVERSION, MD5Util.string2MD5(standardKeyword), dataMap);
logger.info("[func=updateDataToES][step=success][tableName={}][id={}][cost={}ms]", tableName, id, System.currentTimeMillis() - begin);
} else {
indexService.deleteIndexData(ISearchConstants.INDEX_NAME_SUGGEST, MD5Util.string2MD5(CharUtils.standardized(sourceData.getKeyword())));
logger.info("[func=deleteDataFromES][step=success][tableName={}][id={}][cost={}ms]", tableName, id, System.currentTimeMillis() - begin);
}
}
}
}
... ...