Authored by hugufei

Merge branch 'master' into fix_productpool_speed

# Conflicts:
#	consumer/src/main/java/com/yoho/search/mq/ProductPoolDetailMqListener.java
... ... @@ -2,6 +2,7 @@ package com.yoho.search.dynwords;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
... ... @@ -124,7 +125,7 @@ public class DynSegWordsToolsController {
sb.append("file:").append(entry.getKey()).append(", wordType:").append(entry.getValue())
.append(", added count:").append(wordsToAdd.size()).append("|");
}
esWordsService.deleteWordsFromFile("_delete.dic");
rtnMap.put("code", "200");
... ... @@ -135,6 +136,89 @@ public class DynSegWordsToolsController {
return rtnMap;
}
@RequestMapping(value = "/dictTools/addNewWords")
@ResponseBody
public Map<String, Object> addNewWords(HttpServletRequest request) {
Map<String, Object> rtnMap = new HashMap<String, Object>();
long begin = System.currentTimeMillis();
logger.info("[DynSegWords]DynSegWordsToolsController.addNewWords start");
String words = "";
if (request.getParameter("words") != null && request.getParameter("words").trim().length() > 0) {
words = request.getParameter("words").trim();
}
String wordType = "1";
if (request.getParameter("wordType") != null && request.getParameter("wordType").trim().length() > 0) {
wordType = request.getParameter("wordType").trim();
}
if ("7".equals(wordType)) {
// 调用/dictTools/addNewSynonymRule添加同义词
rtnMap.put("code", "404");
rtnMap.put("msg", "invalid word type");
logger.info("[DynSegWords]DynSegWordsToolsController.addNewWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
Set<String> wordSet = new HashSet<String>();
for (String word : words.split(",")) {
if (word != null && word.trim().length() > 0) {
wordSet.add(word.trim());
}
}
Set<String> addedWords = esWordsService.addNewWords(wordSet, Integer.valueOf(wordType));
rtnMap.put("code", "200");
rtnMap.put("msg", "Add new word succeed.");
rtnMap.put("detail", addedWords);
logger.info("[DynSegWords]DynSegWordsToolsController.addNewWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
@RequestMapping(value = "/dictTools/reactiveWords")
@ResponseBody
public Map<String, Object> reactiveWords(HttpServletRequest request) {
Map<String, Object> rtnMap = new HashMap<String, Object>();
long begin = System.currentTimeMillis();
logger.info("[DynSegWords]DynSegWordsToolsController.reactiveWords start");
String words = "";
if (request.getParameter("words") != null && request.getParameter("words").trim().length() > 0) {
words = request.getParameter("words").trim();
}
String wordType = "1";
if (request.getParameter("wordType") != null && request.getParameter("wordType").trim().length() > 0) {
wordType = request.getParameter("wordType").trim();
}
if ("7".equals(wordType)) {
// 调用/dictTools/addNewSynonymRule添加同义词
rtnMap.put("code", "404");
rtnMap.put("msg", "invalid word type");
logger.info("[DynSegWords]DynSegWordsToolsController.reactiveWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
Set<String> wordSet = new HashSet<String>();
for (String word : words.split(",")) {
if (word != null && word.trim().length() > 0) {
wordSet.add(word.trim());
}
}
esWordsService.reactiveWords(wordSet, Integer.valueOf(wordType));
rtnMap.put("code", "200");
rtnMap.put("msg", "reactive words succeed.");
logger.info("[DynSegWords]DynSegWordsToolsController.reactiveWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
}
@RequestMapping(value = "/dictTools/deleteWords")
@ResponseBody
public Map<String, Object> deleteWords(HttpServletRequest request) {
... ... @@ -156,7 +240,7 @@ public class DynSegWordsToolsController {
esWordsService.deleteWords(list);
rtnMap.put("code", "200");
rtnMap.put("msg", "Load word file succeed.");
rtnMap.put("msg", "delete words succeed.");
logger.info("[DynSegWords]DynSegWordsToolsController.deleteWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
... ... @@ -176,7 +260,7 @@ public class DynSegWordsToolsController {
esWordsService.deleteWordsFromFile(wordFile);
rtnMap.put("code", "200");
rtnMap.put("msg", "Load word file succeed.");
rtnMap.put("msg", "delete words succeed.");
logger.info("[DynSegWords]DynSegWordsToolsController.deleteWords end. cost: {}",
System.currentTimeMillis() - begin);
return rtnMap;
... ...
... ... @@ -24,24 +24,86 @@ import com.yoho.search.dal.model.ESWordDef;
@Service
public class ESWordsService {
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
private static Logger logger = LoggerFactory.getLogger(DynSegWordsToolsController.class);
private static final int BATCH_NUMBER = 5000;
private static final int ACTIVE_WORD_STATUS = 1;
private static final int DISABLED_WORD_STATUS = 0;
@Autowired
private ESWordDefMapper esWordDefMapper;
public Set<String> addWords(String wordFile, int wordType) {
logger.info("[DynSegWords]ESWordsService.addWords start. wordFile: {}, wordType: {}", wordFile, wordType);
String classpath = this.getClass().getResource("/").getPath();
Set<String> words = readFile(classpath + "/dicts/" + wordFile);
public Set<String> reactiveWords(Set<String> words, int wordType) {
logger.info("[DynSegWords]ESWordsService.reactiveWords start");
int size = words != null ? words.size() : 0;
logger.info("[DynSegWords]Size of words in the file is {}.", size);
logger.info("[DynSegWords]Size of words to be reactived is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list to be reactived: " + LINE_SEPARATOR + words);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.reactiveWords finished for no words to be reactived.");
return words;
}
// 1. 先从数据库里查询出所有的
Set<String> wordsInDB = getESWordsFromDBByType(wordType);
logger.info("[DynSegWords]Size of words in the DB is {}.", wordsInDB.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the DB: " + LINE_SEPARATOR + wordsInDB);
}
// 2. 获取所有增量的词
Set<String> toAddWords = new HashSet<String>();
List<String> toUpdWords = new ArrayList<String>();
for (String word : words) {
if (wordsInDB.contains(word)) {
toUpdWords.add(word);
} else {
toAddWords.add(word);
}
}
logger.info("[DynSegWords]Size of words need to add is {} and need to reactive is {}", toAddWords.size(),
toUpdWords.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the file: " + System.getProperty("line.separator") + words);
logger.debug("[DynSegWords]The words need to add: " + LINE_SEPARATOR + toAddWords);
logger.debug("[DynSegWords]The words need to reactive: " + LINE_SEPARATOR + toUpdWords);
}
// 3. 插入新的词
List<ESWordDef> wordList = new ArrayList<ESWordDef>();
for (String word : toAddWords) {
wordList.add(new ESWordDef(word, wordType));
}
if (!wordList.isEmpty()) {
esWordDefMapper.insertBatch(wordList);
logger.info("[DynSegWords]Batch insert words to DB. size: " + wordList.size());
}
if (!toUpdWords.isEmpty()) {
esWordDefMapper.updateStatusBatch(toUpdWords, ACTIVE_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]Batch active words in DB. size: " + toUpdWords.size());
}
logger.info("[DynSegWords]ESWordsService.reactiveWords finished.");
return words;
}
public Set<String> addNewWords(Set<String> words, int wordType) {
logger.info("[DynSegWords]ESWordsService.addNewWords start");
int size = words != null ? words.size() : 0;
logger.info("[DynSegWords]Size of words to be added is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list to be added: " + LINE_SEPARATOR + words);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.addWords finished for no words in file.");
logger.info("[DynSegWords]ESWordsService.addNewWords finished for no words to be added.");
return words;
}
... ... @@ -49,18 +111,18 @@ public class ESWordsService {
Set<String> wordsInDB = getESWordsFromDBByType(wordType);
logger.info("[DynSegWords]Size of words in the DB is {}.", wordsInDB.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the DB: " + System.getProperty("line.separator") + wordsInDB);
logger.debug("[DynSegWords]The words list in the DB: " + LINE_SEPARATOR + wordsInDB);
}
// 2. 获取所有增量的词
words.removeAll(wordsInDB);
logger.info("[DynSegWords]Size of words need to add is {}.", words.size());
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words need to add: " + System.getProperty("line.separator") + words);
logger.debug("[DynSegWords]The words need to add: " + LINE_SEPARATOR + words);
}
if (words.isEmpty()) {
logger.info("[DynSegWords]ESWordsService.addWords finished for no words to add.");
logger.info("[DynSegWords]ESWordsService.addNewWords finished for no words to add.");
return words;
}
... ... @@ -80,7 +142,17 @@ public class ESWordsService {
logger.info("[DynSegWords]Batch insert words to DB. size: " + wordList.size());
}
// 4.记录插入的文件
logger.info("[DynSegWords]ESWordsService.addNewWords finished.");
return words;
}
public Set<String> addWords(String wordFile, int wordType) {
logger.info("[DynSegWords]ESWordsService.addWords start. wordFile: {}, wordType: {}", wordFile, wordType);
String classpath = this.getClass().getResource("/").getPath();
Set<String> words = readFile(classpath + "/dicts/" + wordFile);
words = addNewWords(words, wordType);
// 记录插入的文件
writeFile(classpath + "/dicts/" + wordFile + ".tmp", words);
logger.info("[DynSegWords]ESWordsService.addWords finished.");
... ... @@ -143,7 +215,7 @@ public class ESWordsService {
int size = words != null ? words.size() : 0;
logger.info("[DynSegWords]Size of words in the file is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list in the file: " + System.getProperty("line.separator") + words);
logger.debug("[DynSegWords]The words list in the file: " + LINE_SEPARATOR + words);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.deleteWordsFromFile finished for no words in file.");
... ... @@ -157,14 +229,16 @@ public class ESWordsService {
wordList.add(word);
}
if (wordList.size() == BATCH_NUMBER) {
esWordDefMapper.updateStatusBatch(wordList, (int) (System.currentTimeMillis() / 1000));
esWordDefMapper.updateStatusBatch(wordList, DISABLED_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]Batch diabled words to DB. size: " + wordList.size());
wordList = new ArrayList<String>();
}
}
if (wordList.size() > 0) {
esWordDefMapper.updateStatusBatch(wordList, (int) (System.currentTimeMillis() / 1000));
esWordDefMapper.updateStatusBatch(wordList, DISABLED_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]Batch diabled words to DB. size: " + wordList.size());
}
... ... @@ -176,15 +250,15 @@ public class ESWordsService {
int size = wordsToDelete != null ? wordsToDelete.size() : 0;
logger.info("[DynSegWords]Size of words to be delete is {}.", size);
if (logger.isDebugEnabled()) {
logger.debug("[DynSegWords]The words list to be deleted: " + System.getProperty("line.separator")
+ wordsToDelete);
logger.debug("[DynSegWords]The words list to be deleted: " + LINE_SEPARATOR + wordsToDelete);
}
if (size == 0) {
logger.info("[DynSegWords]ESWordsService.addWords finished for no words to be deleted.");
logger.info("[DynSegWords]ESWordsService.deleteWords finished for no words to be deleted.");
return;
}
esWordDefMapper.updateStatusBatch(wordsToDelete, (int) (System.currentTimeMillis() / 1000));
esWordDefMapper.updateStatusBatch(wordsToDelete, DISABLED_WORD_STATUS,
(int) (System.currentTimeMillis() / 1000));
logger.info("[DynSegWords]ESWordsService.deleteWords finished.");
}
}
... ...
... ... @@ -163,6 +163,9 @@ public class ProductPriceMqListener extends AbstractMqListener implements Channe
indexData.put("specialoffer", specialoffer);
indexData.put("isDiscount", isDiscount);
// 是否支持返币 当学生返币比率大于0时支持返币
indexData.put("isstudentrebate", productPrice.getStudentCoinRate() == null || productPrice.getStudentCoinRate().compareTo(BigDecimal.ZERO) == 0 ? "N" : "Y");
// 更新商品索引
this.updateProductIndexWithDataMap(indexData, productId, key, begin);
}
... ...
... ... @@ -120,9 +120,19 @@ public class ProductSearchMqListener extends AbstractMqListener implements Chann
if (productSearch.getProjectId() == 1 && productSearch.getModelId() == 1) {
indexData.put("productId", productId);
indexData.put("brandWeight", productSearch.getIntValue());
indexData.put("sortWeight", 0);
indexData.put("shopWeight", 0);
} else if (productSearch.getProjectId() == 4 && productSearch.getModelId() == 1) {
indexData.put("productId", productId);
indexData.put("sortWeight", productSearch.getIntValue());
indexData.put("brandWeight", 0);
indexData.put("shopWeight", 0);
} else if (productSearch.getProjectId() == 5 && productSearch.getModelId() == 1) {
indexData.put("productId", productId);
indexData.put("brandWeight", 0);
indexData.put("sortWeight", 0);
indexData.put("shopWeight", productSearch.getIntValue());
}
// 3、更新商品索引数据
... ...
... ... @@ -11,7 +11,7 @@ levis=>levi's
范斯=>vans
万斯=>vans
水杯,水壶
鞋子=>
鞋子,
男士=>男
男子=>男
女士=>女
... ... @@ -19,4 +19,8 @@ levis=>levi's
阿迪=>adidas
耐克=>nike
童=>潮童
shirt,恤
\ No newline at end of file
shirt,恤
裤子,裤
裙子,裙
袜子,袜
帽子,帽
\ No newline at end of file
... ...
... ... @@ -11,10 +11,10 @@
<!-- DEBUG、INFO、WARN、ERROR级别日志 appender -->
<appender name="DEBUG_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/debug.log</file>
<file>${catalina.home}/logs/search-consumer/debug.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/debug.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/debug.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -29,14 +29,14 @@
<!-- WARN、ERROR级别日志 appender -->
<appender name="WARN_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/warn-log.log</file>
<file>${catalina.home}/logs/search-consumer/warn-log.log</file>
<!-- 过滤器,过滤掉 TRACE 和 DEBUG 和 INFO 级别的日志 -->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>WARN</level>
</filter>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/warn.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/warn.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -52,10 +52,10 @@
<!-- HTTP请求超时 appender -->
<appender name="REQUEST_TIMEOUT" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/request-timeout.log</file>
<file>${catalina.home}/logs/search-consumer/request-timeout.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/request-timeout.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/request-timeout.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -70,10 +70,10 @@
<!-- 数据库超时日志 appender -->
<appender name="DATABASE_TIMEOUT" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/database-timeout.log</file>
<file>${catalina.home}/logs/search-consumer/database-timeout.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/database-timeout.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/database-timeout.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -88,10 +88,10 @@
<!-- 服务请求统计 appender -->
<appender name="REQUEST_STAT" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/request-stat.log</file>
<file>${catalina.home}/logs/search-consumer/request-stat.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/request-stat.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/request-stat.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -106,10 +106,10 @@
<!-- 全量建索引 appender -->
<appender name="INDEX_REBULDER_APPAND" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/index-rebuild.log</file>
<file>${catalina.home}/logs/search-consumer/index-rebuild.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/index-rebuild.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/index-rebuild.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -124,10 +124,10 @@
<!-- 事件上报 appender -->
<appender name="SEARCH_EVENT_APPAND" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>/Data/logs/search-consumer/search-event.log</file>
<file>${catalina.home}/logs/search-consumer/search-event.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>/Data/logs/search-consumer/archived/search-event.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<fileNamePattern>${catalina.home}/logs/search-consumer/archived/search-event.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>10MB</maxFileSize>
... ... @@ -141,18 +141,18 @@
</appender>
<!-- 数据库操作日志 暂时屏蔽spring框架日志-->
<logger name="java.sql.PreparedStatement" value="DEBUG" />
<logger name="java.sql.Connection" value="DEBUG" />
<logger name="java.sql.Statement" value="DEBUG" />
<logger name="com.ibatis" value="DEBUG" />
<logger name="com.ibatis.common.jdbc.SimpleDataSource" value="DEBUG" />
<logger name="com.ibatis.common.jdbc.ScriptRunner" level="DEBUG"/>
<logger name="com.ibatis.sqlmap.engine.impl.SqlMapClientDelegate" value="DEBUG" />
<logger name="java.sql.PreparedStatement" value="INFO" />
<logger name="java.sql.Connection" value="INFO" />
<logger name="java.sql.Statement" value="INFO" />
<logger name="com.ibatis" value="INFO" />
<logger name="com.ibatis.common.jdbc.SimpleDataSource" value="INFO" />
<logger name="com.ibatis.common.jdbc.ScriptRunner" level="INFO"/>
<logger name="com.ibatis.sqlmap.engine.impl.SqlMapClientDelegate" value="INFO" />
<logger name="org.springframework" level="INFO"/>
<logger name="com.yoho.search" level="DEBUG"/>
<logger name="com.yoho.search" level="INFO"/>
<logger name="org.apache.zookeeper.ClientCnxn" level="WARN"/>
<!-- root级别 DEBUG -->
<root level="DEBUG">
<!-- root级别 INFO -->
<root level="INFO">
<!-- 文件输出 -->
<appender-ref ref="DEBUG_LOG" />
<appender-ref ref="WARN_LOG" />
... ...