yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by Gino Zhang
Commit 251056a7161f51bde854c2e72a93f121264a5d7d 2 parents 75b4cbc6 d7017a99

Merge branch 'master_zf_ikwords' into zf_future_ret

Inline Side-by-side
Showing 3 changed files with 180 additions and 22 deletions
... ... @@ -2,6 +2,7 @@ package com.yoho.search.dynwords;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
... ... @@ -124,7 +125,7 @@ public class DynSegWordsToolsController {
sb.append("file:").append(entry.getKey()).append(", wordType:").append(entry.getValue())
.append(", added count:").append(wordsToAdd.size()).append("|");
}
esWordsService.deleteWordsFromFile("_delete.dic");
rtnMap.put("code", "200");
... ... @@ -135,6 +136,89 @@ public class DynSegWordsToolsController {
return rtnMap;
}
@RequestMapping(value = "/dictTools/addNewWords")
@ResponseBody
public Map<String, Object> addNewWords(HttpServletRequest request) {
Map<String, Object> rtnMap = new HashMap<String, Object>();
long begin = System.currentTimeMillis();
logger.info("[DynSegWords]DynSegWordsToolsController.addNewWords start");
String words = "";
if (request.getParameter("words") != null && request.getParameter("words").trim().length() > 0) {
words = request.getParameter("words").trim();
}
String wordType = "1";
if (request.getParameter("wordType") != null && request.getParameter("wordType").trim().length() > 0) {
wordType = request.getParameter("wordType").trim();
}
if ("7".equals(wordType)) {
// 调用/dictTools/addNewSynonymRule添加同义词
rtnMap.put("code", "404");
rtnMap.put("msg", "invalid word type");
logger.info("[DynSegWords]DynSegWordsToolsController.addNewWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
Set<String> wordSet = new HashSet<String>();
for (String word : words.split(",")) {
if (word != null && word.trim().length() > 0) {
wordSet.add(word.trim());
}
}
Set<String> addedWords = esWordsService.addNewWords(wordSet, Integer.valueOf(wordType));
rtnMap.put("code", "200");
rtnMap.put("msg", "Add new word succeed.");
rtnMap.put("detail", addedWords);
logger.info("[DynSegWords]DynSegWordsToolsController.addNewWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
@RequestMapping(value = "/dictTools/reactiveWords")
@ResponseBody
public Map<String, Object> reactiveWords(HttpServletRequest request) {
Map<String, Object> rtnMap = new HashMap<String, Object>();
long begin = System.currentTimeMillis();
logger.info("[DynSegWords]DynSegWordsToolsController.reactiveWords start");
String words = "";
if (request.getParameter("words") != null && request.getParameter("words").trim().length() > 0) {
words = request.getParameter("words").trim();
}
String wordType = "1";
if (request.getParameter("wordType") != null && request.getParameter("wordType").trim().length() > 0) {
wordType = request.getParameter("wordType").trim();
}
if ("7".equals(wordType)) {
// 调用/dictTools/addNewSynonymRule添加同义词
rtnMap.put("code", "404");
rtnMap.put("msg", "invalid word type");
logger.info("[DynSegWords]DynSegWordsToolsController.reactiveWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
Set<String> wordSet = new HashSet<String>();
for (String word : words.split(",")) {
if (word != null && word.trim().length() > 0) {
wordSet.add(word.trim());
}
}
esWordsService.reactiveWords(wordSet, Integer.valueOf(wordType));
rtnMap.put("code", "200");
rtnMap.put("msg", "reactive words succeed.");
logger.info("[DynSegWords]DynSegWordsToolsController.reactiveWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
@RequestMapping(value = "/dictTools/deleteWords")
@ResponseBody
public Map<String, Object> deleteWords(HttpServletRequest request) {
... ... @@ -156,7 +240,7 @@ public class DynSegWordsToolsController {
esWordsService.deleteWords(list);
rtnMap.put("code", "200");
rtnMap.put("msg", "Load word file succeed.");
rtnMap.put("msg", "delete words succeed.");
logger.info("[DynSegWords]DynSegWordsToolsController.deleteWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
... ... @@ -176,7 +260,7 @@ public class DynSegWordsToolsController {
esWordsService.deleteWordsFromFile(wordFile);
rtnMap.put("code", "200");
rtnMap.put("msg", "Load word file succeed.");
rtnMap.put("msg", "delete words succeed.");
logger.info("[DynSegWords]DynSegWordsToolsController.deleteWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
... ...
... ... @@ -24,24 +24,86 @@ import com.yoho.search.dal.model.ESWordDef;
@Service
public class ESWordsService {
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
private static Logger logger = LoggerFactory.getLogger(DynSegWordsToolsController.class);
private static final int BATCH_NUMBER = 5000;
private static final int ACTIVE_WORD_STATUS = 1;
private static final int DISABLED_WORD_STATUS = 0;
@Autowired
private ESWordDefMapper esWordDefMapper;
public Set<String> addWords(String wordFile, int wordType) {
logger.info("[DynSegWords]ESWordsService.addWords start. wordFile: {}, wordType: {}", wordFile, wordType);
String classpath = this.getClass().getResource("/").getPath();
Set<String> words = readFile(classpath + "/dicts/" + wordFile);
public Set<String> reactiveWords(Set<String> words, int wordType) {
logger.info("[DynSegWords]ESWordsService.reactiveWords start");
int size = words != null ? words.size() : 0;
logger.info("[DynSegWords]Size of words in the file is {}.", size);
logger.info("[DynSegWords]Size of words to be reactived is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list to be reactived: " + LINE_SEPARATOR + words);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.reactiveWords finished for no words to be reactived.");
return words;
}
// 1. 先从数据库里查询出所有的
Set<String> wordsInDB = getESWordsFromDBByType(wordType);
logger.info("[DynSegWords]Size of words in the DB is {}.", wordsInDB.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the DB: " + LINE_SEPARATOR + wordsInDB);
}
// 2. 获取所有增量的词
Set<String> toAddWords = new HashSet<String>();
List<String> toUpdWords = new ArrayList<String>();
for (String word : words) {
if (wordsInDB.contains(word)) {
toUpdWords.add(word);
} else {
toAddWords.add(word);
}
}
logger.info("[DynSegWords]Size of words need to add is {} and need to reactive is {}", toAddWords.size(),
toUpdWords.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the file: " + System.getProperty("line.separator") + words);
logger.debug("[DynSegWords]The words need to add: " + LINE_SEPARATOR + toAddWords);
logger.debug("[DynSegWords]The words need to reactive: " + LINE_SEPARATOR + toUpdWords);
}
// 3. 插入新的词
List<ESWordDef> wordList = new ArrayList<ESWordDef>();
for (String word : toAddWords) {
wordList.add(new ESWordDef(word, wordType));
}
if (!wordList.isEmpty()) {
esWordDefMapper.insertBatch(wordList);
logger.info("[DynSegWords]Batch insert words to DB. size: " + wordList.size());
}
if (!toUpdWords.isEmpty()) {
esWordDefMapper.updateStatusBatch(toUpdWords, ACTIVE_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]Batch active words in DB. size: " + toUpdWords.size());
}
logger.info("[DynSegWords]ESWordsService.reactiveWords finished.");
return words;
}
public Set<String> addNewWords(Set<String> words, int wordType) {
logger.info("[DynSegWords]ESWordsService.addNewWords start");
int size = words != null ? words.size() : 0;
logger.info("[DynSegWords]Size of words to be added is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list to be added: " + LINE_SEPARATOR + words);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.addWords finished for no words in file.");
logger.info("[DynSegWords]ESWordsService.addNewWords finished for no words to be added.");
return words;
}
... ... @@ -49,18 +111,18 @@ public class ESWordsService {
Set<String> wordsInDB = getESWordsFromDBByType(wordType);
logger.info("[DynSegWords]Size of words in the DB is {}.", wordsInDB.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the DB: " + System.getProperty("line.separator") + wordsInDB);
logger.debug("[DynSegWords]The words list in the DB: " + LINE_SEPARATOR + wordsInDB);
}
// 2. 获取所有增量的词
words.removeAll(wordsInDB);
logger.info("[DynSegWords]Size of words need to add is {}.", words.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words need to add: " + System.getProperty("line.separator") + words);
logger.debug("[DynSegWords]The words need to add: " + LINE_SEPARATOR + words);
}
if (words.isEmpty()) {
logger.info("[DynSegWords]ESWordsService.addWords finished for no words to add.");
logger.info("[DynSegWords]ESWordsService.addNewWords finished for no words to add.");
return words;
}
... ... @@ -80,7 +142,17 @@ public class ESWordsService {
logger.info("[DynSegWords]Batch insert words to DB. size: " + wordList.size());
}
// 4.记录插入的文件
logger.info("[DynSegWords]ESWordsService.addNewWords finished.");
return words;
}
public Set<String> addWords(String wordFile, int wordType) {
logger.info("[DynSegWords]ESWordsService.addWords start. wordFile: {}, wordType: {}", wordFile, wordType);
String classpath = this.getClass().getResource("/").getPath();
Set<String> words = readFile(classpath + "/dicts/" + wordFile);
words = addNewWords(words, wordType);
// 记录插入的文件
writeFile(classpath + "/dicts/" + wordFile + ".tmp", words);
logger.info("[DynSegWords]ESWordsService.addWords finished.");
... ... @@ -143,7 +215,7 @@ public class ESWordsService {
int size = words != null ? words.size() : 0;
logger.info("[DynSegWords]Size of words in the file is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the file: " + System.getProperty("line.separator") + words);
logger.debug("[DynSegWords]The words list in the file: " + LINE_SEPARATOR + words);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.deleteWordsFromFile finished for no words in file.");
... ... @@ -157,14 +229,16 @@ public class ESWordsService {
wordList.add(word);
}
if (wordList.size() == BATCH_NUMBER) {
esWordDefMapper.updateStatusBatch(wordList, (int) (System.currentTimeMillis() / 1000));
esWordDefMapper.updateStatusBatch(wordList, DISABLED_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]Batch diabled words to DB. size: " + wordList.size());
wordList = new ArrayList<String>();
}
}
if (wordList.size() > 0) {
esWordDefMapper.updateStatusBatch(wordList, (int) (System.currentTimeMillis() / 1000));
esWordDefMapper.updateStatusBatch(wordList, DISABLED_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]Batch diabled words to DB. size: " + wordList.size());
}
... ... @@ -176,15 +250,15 @@ public class ESWordsService {
int size = wordsToDelete != null ? wordsToDelete.size() : 0;
logger.info("[DynSegWords]Size of words to be delete is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list to be deleted: " + System.getProperty("line.separator")
+ wordsToDelete);
logger.debug("[DynSegWords]The words list to be deleted: " + LINE_SEPARATOR + wordsToDelete);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.addWords finished for no words to be deleted.");
logger.info("[DynSegWords]ESWordsService.deleteWords finished for no words to be deleted.");
return;
}
esWordDefMapper.updateStatusBatch(wordsToDelete, (int) (System.currentTimeMillis() / 1000));
esWordDefMapper.updateStatusBatch(wordsToDelete, DISABLED_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]ESWordsService.deleteWords finished.");
}
}
... ...
... ... @@ -100,7 +100,7 @@ public class ProductPoolDetailMqListener extends AbstractMqListener implements C
updateIndex(productPoolDetail, System.currentTimeMillis(), key);
logger.info("[func=updateData][step=success][key={}][indexName={}][skn={}][cost={}ms]", key, ISearchConstans.INDEX_NAME_PRODUCT_INDEX, productPoolDetail.getProductSkn(),
(System.currentTimeMillis() - begin));
updateProductPoolIndex(productPoolDetail, begin, key);
//updateProductPoolIndex(productPoolDetail, begin, key);
logger.info("[func=updateData][step=success][key={}][indexName={}][skn={}][cost={}ms]", key, ISearchConstans.INDEX_NAME_PRODUCT_POOL, productPoolDetail.getProductSkn(),
(System.currentTimeMillis() - begin));
}
... ...