|
|
|
package com.yoho.search.consumer.dynwords;
|
|
|
|
|
|
|
|
import java.io.BufferedReader;
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.io.InputStreamReader;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Date;
|
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
|
|
|
|
import javax.servlet.http.HttpServletRequest;
|
|
|
|
import javax.servlet.http.HttpServletResponse;
|
|
|
|
|
|
|
|
import com.yoho.search.dal.DynSegWordMapper;
|
|
|
|
import com.yoho.search.dal.model.DynSegWord;
|
|
|
|
import org.apache.http.client.ClientProtocolException;
|
|
|
|
import org.apache.http.client.config.RequestConfig;
|
|
|
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
|
|
|
import org.apache.http.client.methods.HttpGet;
|
|
|
|
import org.apache.http.impl.client.CloseableHttpClient;
|
|
|
|
import org.apache.http.impl.client.HttpClients;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.stereotype.Controller;
|
|
|
|
import org.springframework.web.bind.annotation.RequestMapping;
|
|
|
|
import org.springframework.web.bind.annotation.RequestMethod;
|
|
|
|
import org.springframework.web.bind.annotation.ResponseBody;
|
|
|
|
|
|
|
|
@Controller
|
|
|
|
public class DynSegWordsController {
|
|
|
|
|
|
|
|
private static Logger logger = LoggerFactory.getLogger(DynSegWordsController.class);
|
|
|
|
|
|
|
|
private static final int NO_RECORDS_CHANGED_STATUS_CODE = 304;
|
|
|
|
|
|
|
|
private static final int WORD_TYPE_MAIN = 1;
|
|
|
|
|
|
|
|
private static final int WORD_TYPE_STOP = 2;
|
|
|
|
|
|
|
|
private static final int DEFAULT_STARTTIME = 0;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private DynSegWordMapper dynSegWordMapper;
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
private UpgradeDynSegWordsFlagService upgradeDynSegWordsFlagService;
|
|
|
|
|
|
|
|
@RequestMapping(value = "/dictService/forceUpgrade")
|
|
|
|
@ResponseBody
|
|
|
|
public Map<String, Object> forceUpgradeDynWords() {
|
|
|
|
upgradeDynSegWordsFlagService.updateForceUpgradeTrue();
|
|
|
|
Map<String, Object> rtnMap = new HashMap<String, Object>();
|
|
|
|
rtnMap.put("code", "200");
|
|
|
|
rtnMap.put("msg", "Update force upgrade seg words succeed.");
|
|
|
|
return rtnMap;
|
|
|
|
}
|
|
|
|
|
|
|
|
@RequestMapping(method = RequestMethod.HEAD, value = "/dictService/getRemoteMainDict")
|
|
|
|
public void checkMainWords(HttpServletRequest request, HttpServletResponse response) {
|
|
|
|
// ES的ik分词插件发送HEAD请求 判断主词典是否有变更
|
|
|
|
logger.info("[DynSegWords]enter DynSegWordsController.checkMainWords");
|
|
|
|
checkNeedUpdateWordsDict(request, response, WORD_TYPE_MAIN);
|
|
|
|
logger.info("[DynSegWords]end DynSegWordsController.checkMainWords.");
|
|
|
|
}
|
|
|
|
|
|
|
|
@RequestMapping(method = RequestMethod.HEAD, value = "/dictService/getRemoteStopDict")
|
|
|
|
public void checkStopWords(HttpServletRequest request, HttpServletResponse response) {
|
|
|
|
// ES的ik分词插件发送HEAD请求 判断停用词是否有变更
|
|
|
|
logger.info("[DynSegWords]enter DynSegWordsController.checkStopWords");
|
|
|
|
checkNeedUpdateWordsDict(request, response, WORD_TYPE_STOP);
|
|
|
|
logger.info("[DynSegWords]end DynSegWordsController.checkStopWords.");
|
|
|
|
}
|
|
|
|
|
|
|
|
private void checkNeedUpdateWordsDict(HttpServletRequest request, HttpServletResponse response, Integer wordType) {
|
|
|
|
String esIp = getIpAddress(request);
|
|
|
|
logger.info("[DynSegWords]Begin to check need update seg words. wordType: {}, esIP: {}", wordType, esIp);
|
|
|
|
if (upgradeDynSegWordsFlagService.canUpgrade()
|
|
|
|
&& dynSegWordMapper.selectWordsCount(getLastModifyTime(request)) > 0) {
|
|
|
|
// 只有在开始全量建索引的时候才开始增加动态词库
|
|
|
|
String currentTime = String.valueOf((new Date()).getTime() / 1000L);
|
|
|
|
response.setHeader("Last-Modified", currentTime);
|
|
|
|
logger.info("[DynSegWords]Need to update seg words. wordType: {}", wordType);
|
|
|
|
} else {
|
|
|
|
response.setStatus(NO_RECORDS_CHANGED_STATUS_CODE);
|
|
|
|
logger.info("[DynSegWords]No need to update seg words. wordType: {}", wordType);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private Integer getLastModifyTime(HttpServletRequest request) {
|
|
|
|
String lastModifyTime = request.getHeader("If-Modified-Since");
|
|
|
|
logger.info("[DynSegWords]Get last modify time {} from request header.", lastModifyTime);
|
|
|
|
if (lastModifyTime != null && lastModifyTime.trim().length() > 0) {
|
|
|
|
try {
|
|
|
|
return Integer.valueOf(lastModifyTime.trim());
|
|
|
|
} catch (Exception e) {
|
|
|
|
// Ignore the exception
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return DEFAULT_STARTTIME;
|
|
|
|
}
|
|
|
|
|
|
|
|
@RequestMapping(method = RequestMethod.GET, value = "/dictService/getRemoteMainDict")
|
|
|
|
public void getSegMainWords(HttpServletRequest request, HttpServletResponse response) {
|
|
|
|
logger.info("[DynSegWords]enter DynSegWordsController.getSegMainWords");
|
|
|
|
addDynWordsByType(request, response, WORD_TYPE_MAIN);
|
|
|
|
logger.info("[DynSegWords]end DynSegWordsController.getSegMainWords.");
|
|
|
|
}
|
|
|
|
|
|
|
|
@RequestMapping(method = RequestMethod.GET, value = "/dictService/getRemoteStopDict")
|
|
|
|
public void getSegStopWords(HttpServletRequest request, HttpServletResponse response) {
|
|
|
|
logger.info("[DynSegWords]enter DynSegWordsController.getSegStopWords");
|
|
|
|
addDynWordsByType(request, response, WORD_TYPE_STOP);
|
|
|
|
logger.info("[DynSegWords]end DynSegWordsController.getSegStopWords.");
|
|
|
|
}
|
|
|
|
|
|
|
|
private void addDynWordsByType(HttpServletRequest request, HttpServletResponse response, Integer wordType) {
|
|
|
|
String esIp = getIpAddress(request);
|
|
|
|
logger.info("[DynSegWords]Begin to add dyn seg words. wordType: {}, esIP: {}", wordType, esIp);
|
|
|
|
List<DynSegWord> words = dynSegWordMapper.selectByWordType(wordType);
|
|
|
|
int count = words != null ? words.size() : 0;
|
|
|
|
logger.info("[DynSegWords]Find {} words from DB for the word type {}.", count, wordType);
|
|
|
|
response.setContentType("text/html;charset=UTF-8");
|
|
|
|
if (count > 0) {
|
|
|
|
try {
|
|
|
|
for (DynSegWord word : words) {
|
|
|
|
response.getWriter().println(word.getWord());
|
|
|
|
}
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.warn("[DynSegWords]Add dyn words occur exception.", e);
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info("[DynSegWords]Send {} words to es {}.", count, esIp);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private String getIpAddress(HttpServletRequest request) {
|
|
|
|
String ip = request.getHeader("x-forwarded-for");
|
|
|
|
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
|
|
|
|
ip = request.getHeader("Proxy-Client-IP");
|
|
|
|
}
|
|
|
|
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
|
|
|
|
ip = request.getHeader("WL-Proxy-Client-IP");
|
|
|
|
}
|
|
|
|
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
|
|
|
|
ip = request.getHeader("HTTP_CLIENT_IP");
|
|
|
|
}
|
|
|
|
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
|
|
|
|
ip = request.getHeader("HTTP_X_FORWARDED_FOR");
|
|
|
|
}
|
|
|
|
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
|
|
|
|
ip = request.getRemoteAddr();
|
|
|
|
}
|
|
|
|
return ip;
|
|
|
|
}
|
|
|
|
|
|
|
|
public static void main(String[] args) {
|
|
|
|
List<String> buffer = new ArrayList<String>();
|
|
|
|
|
|
|
|
RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10000).setConnectTimeout(10000)
|
|
|
|
.setSocketTimeout(60000).build();
|
|
|
|
CloseableHttpClient httpclient = HttpClients.createDefault();
|
|
|
|
|
|
|
|
String location = "http://localhost:8080/yoho-search-consumer-web/dictService/getRemoteMainDict";
|
|
|
|
HttpGet get = new HttpGet(location);
|
|
|
|
get.setConfig(rc);
|
|
|
|
try {
|
|
|
|
CloseableHttpResponse response = httpclient.execute(get);
|
|
|
|
if (response.getStatusLine().getStatusCode() == 200) {
|
|
|
|
String charset = "UTF-8";
|
|
|
|
if (response.getEntity().getContentType().getValue().contains("charset=")) {
|
|
|
|
String contentType = response.getEntity().getContentType().getValue();
|
|
|
|
charset = contentType.substring(contentType.lastIndexOf("=") + 1);
|
|
|
|
}
|
|
|
|
BufferedReader in = new BufferedReader(
|
|
|
|
new InputStreamReader(response.getEntity().getContent(), charset));
|
|
|
|
String line;
|
|
|
|
while ((line = in.readLine()) != null) {
|
|
|
|
buffer.add(line);
|
|
|
|
}
|
|
|
|
in.close();
|
|
|
|
response.close();
|
|
|
|
}
|
|
|
|
response.close();
|
|
|
|
} catch (ClientProtocolException e) {
|
|
|
|
logger.error("getRemoteWords {} error", e, new Object[] { location });
|
|
|
|
} catch (IllegalStateException e) {
|
|
|
|
logger.error("getRemoteWords {} error", e, new Object[] { location });
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error("getRemoteWords {} error", e, new Object[] { location });
|
|
|
|
}
|
|
|
|
System.out.println(buffer);
|
|
|
|
}
|
|
|
|
} |
