yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit d5509f42116f7cf6197feed370603fe24887e444 2 parents 33b538c5 f5c5c078

Merge branch 'master_perfomance' into dev

Inline Side-by-side
Showing 8 changed files with 185 additions and 60 deletions
... ... @@ -31,7 +31,7 @@ public class CacheService {
private static final int cacheTimeInMinute = 3;//结果缓存的时间
private static final int logPeriodInMinute = 3;//记录缓存命中的时间
private static final int clearCachePeriodInMinute = 30;//清除整个缓存对象的时间,防止内存使用过多
private static final int clearCachePeriodInMinute = 5;//清除整个缓存对象的时间,防止内存使用过多
@PostConstruct
void init(){
... ...
package com.yoho.search.interceptor;
import java.text.SimpleDateFormat;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.NamedThreadLocal;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.ModelAndView;
import com.yoho.search.utils.HttpServletRequestUtils;
public class ControllerCostInterceptor implements HandlerInterceptor {
private static final Logger logger = LoggerFactory.getLogger(ControllerCostInterceptor.class);
private static final ThreadLocal<Long> startTimeThreadLocal = new NamedThreadLocal<Long>("ThreadLocal StartTime");
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handle) throws Exception {
long beginTime = System.currentTimeMillis();// 1、开始时间
startTimeThreadLocal.set(beginTime); // 线程绑定变量（该数据只有当前请求的线程可见）
return true;
}
@Override
public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handle, ModelAndView modelAndView) throws Exception {
}
@Override
public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handle, Exception ex) throws Exception {
long beginTime = startTimeThreadLocal.get();// 1、得到线程绑定的局部变量（开始时间）
long endTime = System.currentTimeMillis(); // 2、结束时间
long cost = endTime - beginTime;
if (cost > 500) {
logger.info("run more than 500ms：{} cost：{}ms uri: {}, param:{}",new SimpleDateFormat("yyyy-HH-MM hh:mm:ss").format(endTime), cost, request.getRequestURI(),HttpServletRequestUtils.transParamType(request));
}
}
}
... ...
package com.yoho.search.restapi;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
... ... @@ -20,6 +19,7 @@ import com.yoho.search.service.SearchService;
import com.yoho.search.service.SearchShopService;
import com.yoho.search.service.SearchSortSizeService;
import com.yoho.search.service.SearchSuggestService;
import com.yoho.search.utils.HttpServletRequestUtils;
import com.yoho.search.utils.JsonUtil;
import com.yoho.search.vo.SearchApiResult;
import com.yoho.search.vo.SizeSortReqBO;
... ... @@ -265,19 +265,8 @@ public class SearchController {
/**
* 将HttpServletRequest中被锁定的ParameterMap转化为普通的HashMap
* */
@SuppressWarnings("unchecked")
private Map<String, String> transParamType(HttpServletRequest request) {
Map<String, Object> paramMap = request.getParameterMap();
Map<String, String> resultMap = new HashMap<String, String>();
Iterator<String> itKeys = paramMap.keySet().iterator();
while (itKeys.hasNext()) {
String key = itKeys.next();
String val = ((String[]) paramMap.get(key))[0];
resultMap.put(key, val);
}
resultMap.put("Referer", request.getHeader("Referer"));
resultMap.put("User-Agent", request.getHeader("User-Agent"));
return resultMap;
return HttpServletRequestUtils.transParamType(request);
}
private Map<String, Object> errorReturn(final String funName, final Map<String, ?> paramMap, final Exception e) {
... ...
... ... @@ -27,7 +27,6 @@ public class SearchCommonService {
private IndexService indexService;
@Autowired
private CacheService cacheService;
/**
* 通用的查询接口
... ... @@ -38,10 +37,10 @@ public class SearchCommonService {
*/
public SearchResult doSearch(final String indexName, final SearchParam searchParam) {
// 1、先从缓存中取结果
SearchResult resultFromCache = cacheService.getSearchResultFromCache(indexName, searchParam);
if (resultFromCache != null) {
return resultFromCache;
}
// SearchResult resultFromCache = cacheService.getSearchResultFromCache(indexName, searchParam);
// if (resultFromCache != null) {
// return resultFromCache;
// }
// 2、取不到再从ES获取结果
SearchResult searchResult = null;
Index firstIndex = indexService.getIndex(indexName);
... ... @@ -53,10 +52,10 @@ public class SearchCommonService {
}
}, firstIndex);
}
// 3、将ES中的缓存结果放进缓存中
if (searchResult != null) {
cacheService.addSearchResultToCache(indexName, searchParam, searchResult);
}
// // 3、将ES中的缓存结果放进缓存中
// if (searchResult != null) {
// cacheService.addSearchResultToCache(indexName, searchParam, searchResult);
// }
return searchResult;
}
... ... @@ -111,11 +110,11 @@ public class SearchCommonService {
if (idList == null || idList.size() == 0) {
return result;
}
// 1、先从缓存中取结果
List<Map<String, Object>> resultFromCache = cacheService.getMultiGetResultFromCache(indexName, idList, fields);
if (resultFromCache != null) {
return resultFromCache;
}
// // 1、先从缓存中取结果
// List<Map<String, Object>> resultFromCache = cacheService.getMultiGetResultFromCache(indexName, idList, fields);
// if (resultFromCache != null) {
// return resultFromCache;
// }
// 2、先ES中批量获取结果
Index firstIndex = indexService.getIndex(indexName);
if (firstIndex != null) {
... ... @@ -126,10 +125,10 @@ public class SearchCommonService {
}
}, firstIndex);
}
// 3、将搜索结果加入到缓存中
if (result != null) {
cacheService.addMultiGetResultResultToCache(indexName, idList,fields, result);
}
// // 3、将搜索结果加入到缓存中
// if (result != null) {
// cacheService.addMultiGetResultResultToCache(indexName, idList,fields, result);
// }
return result;
}
... ...
package com.yoho.search.utils;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
public class HttpServletRequestUtils {
@SuppressWarnings("unchecked")
public static Map<String, String> transParamType(HttpServletRequest request) {
Map<String, Object> paramMap = request.getParameterMap();
Map<String, String> resultMap = new HashMap<String, String>();
Iterator<String> itKeys = paramMap.keySet().iterator();
while (itKeys.hasNext()) {
String key = itKeys.next();
String val = ((String[]) paramMap.get(key))[0];
resultMap.put(key, val);
}
resultMap.put("ip", getIpAddress(request));
return resultMap;
}
public static String getIpAddress(HttpServletRequest request){
String ip = request.getHeader("x-forwarded-for");
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("Proxy-Client-IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("WL-Proxy-Client-IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("HTTP_CLIENT_IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("HTTP_X_FORWARDED_FOR");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getRemoteAddr();
}
return ip;
}
}
... ...
... ... @@ -30,19 +30,20 @@
</list>
</property>
</bean>
<!-- 服务错误码异常处理 -->
<bean class="com.yoho.error.exception.handler.ServiceGlobalExceptionHandler" />
<!-- 启动SpringMVC的注解功能，完成请求和注解POJO的映射 -->
<mvc:annotation-driven>
<mvc:message-converters>
<ref bean="stringConverter" />
<ref bean="jsonConverter" />
</mvc:message-converters>
</mvc:annotation-driven>
<!-- controller耗时的监听器 -->
<bean id="controllerCostInterceptor" class="com.yoho.search.interceptor.ControllerCostInterceptor"/>
<mvc:interceptors>
<!-- 使用bean定义一个Interceptor，直接定义在mvc:interceptors根下面的Interceptor将拦截所有的请求 -->
<mvc:interceptor>
<mvc:mapping path="/**"/>
<!-- 需排除拦截的地址 -->
<mvc:exclude-mapping path="/" />
<mvc:exclude-mapping path="/test" />
<ref bean="controllerCostInterceptor"/>
</mvc:interceptor>
</mvc:interceptors>
<mvc:interceptors>
<ref bean="threadProfileInterceptor"/>
</mvc:interceptors>
... ... @@ -57,7 +58,6 @@
</property>
</bean>
<!-- json转换器 application/json -->
<bean id="jsonConverter"
class="com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter">
... ... @@ -65,23 +65,26 @@
<array value-type="com.alibaba.fastjson.serializer.SerializerFeature">
<value>DisableCircularReferenceDetect</value>
<value>WriteMapNullValue</value>
</array>
</property>
</bean>
<!-- 启动SpringMVC的注解功能，完成请求和注解POJO的映射 -->
<mvc:annotation-driven>
<mvc:message-converters>
<ref bean="stringConverter" />
<ref bean="jsonConverter" />
</mvc:message-converters>
</mvc:annotation-driven>
<!--service接口调用拦截器-->
<bean id="serviceMethodInterceptor" class="com.yoho.core.common.monitor.interceptor.MethodProfileInterceptor">
</bean>
<bean id="serviceMethodInterceptor" class="com.yoho.core.common.monitor.interceptor.MethodProfileInterceptor"/>
<aop:config>
<aop:pointcut id="serviceMethodPoint"
expression="(execution(* com.yoho.*.*.service.*.*(..))) or (execution(* com.yoho.*.*.service.impl.*.*(..))) or (execution(* com.yoho.*.*.restapi.*.*(..)))
"/>
<aop:pointcut id="serviceMethodPoint" expression="(execution(* com.yoho.*.*.service.*.*(..))) or (execution(* com.yoho.*.*.service.impl.*.*(..))) or (execution(* com.yoho.*.*.restapi.*.*(..)))"/>
<aop:advisor pointcut-ref="serviceMethodPoint" advice-ref="serviceMethodInterceptor"/>
</aop:config>
<bean id="applicationContextUtil" class="com.yoho.search.utils.ApplicationContextUtil">
</bean>
<!-- Redis 配置 end -->
<bean id="serviceGlobalExceptionHandler" class="com.yoho.error.exception.handler.ServiceGlobalExceptionHandler" />
<bean id="applicationContextUtil" class="com.yoho.search.utils.ApplicationContextUtil"></bean>
</beans>
\ No newline at end of file
... ...
... ... @@ -71,7 +71,7 @@
</encoder>
</appender>
<!-- 缓存命中统计 appender -->
<!-- 缓存命中统计 appender -->
<appender name="CACHE_MATCH" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${catalina.home}/logs/%d{yyyy-MM-dd}/cache-match.log</fileNamePattern>
... ... @@ -85,6 +85,21 @@
<pattern>%-1relative - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{0} -- %msg%n</pattern>
</encoder>
</appender>
<!-- controller-cost appender -->
<appender name="CONTROLLER_COST" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${catalina.home}/logs/%d{yyyy-MM-dd}/controller-cost.log</fileNamePattern>
<!-- 日志最大的保存天数 -->
<maxHistory>${maxHistory}</maxHistory>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>${maxFileSize}</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>%-1relative - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{0} -- %msg%n</pattern>
</encoder>
</appender>
<!-- 数据库操作日志 -->
<logger name="java.sql.PreparedStatement" value="INFO" />
... ... @@ -115,11 +130,17 @@
<appender-ref ref="DATABASE_TIMEOUT"/>
</logger>
<!-- 缓存命中统计日志 -->
<!-- 缓存命中统计日志 -->
<logger name="com.yoho.search.cache.CacheService" additivity="false">
<level value="INFO"/>
<appender-ref ref="CACHE_MATCH"/>
</logger>
<!-- 耗时统计 -->
<logger name="com.yoho.search.interceptor.ControllerCostInterceptor" additivity="false">
<level value="INFO"/>
<appender-ref ref="CONTROLLER_COST"/>
</logger>
<logger name="org.apache.zookeeper.ClientCnxn" level="WARN"/>
</configuration>
\ No newline at end of file
... ...
... ... @@ -130,7 +130,7 @@
</encoder>
</appender>
<!-- 缓存命中统计 appender -->
<!-- 缓存命中统计 appender -->
<appender name="CACHE_MATCH" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${yoho.logs.basedir}/${yoho.search.service.env.namespace}/cache-match.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
... ... @@ -147,6 +147,24 @@
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
</encoder>
</appender>
<!-- 请求超时 appender -->
<appender name="CONTROLLER_COST" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${yoho.logs.basedir}/${yoho.search.service.env.namespace}/controller-cost.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>${yoho.logs.basedir}/${yoho.search.service.env.namespace}/archived/controller-cost.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB -->
<maxFileSize>${yoho.logs.maxFileSize}</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
</encoder>
</appender>
<!-- 数据库操作日志 -->
<logger name="java.sql.PreparedStatement" value="${yoho.logs.level}" />
... ... @@ -189,5 +207,11 @@
<level value="INFO"/>
<appender-ref ref="CACHE_MATCH"/>
</logger>
<!-- 耗时统计 -->
<logger name="com.yoho.search.interceptor.ControllerCostInterceptor" additivity="false">
<level value="INFO"/>
<appender-ref ref="CONTROLLER_COST"/>
</logger>
</configuration>
\ No newline at end of file
... ...