Authored by wangning

update

... ... @@ -21,6 +21,9 @@ public class SnsMobileConfig {
@Value("${java_service_alarm_flag}")
private boolean javaServiceFlag;
@Value("${java_service_alarm_cost_threshold}")
private Integer javaServiceCostThreshold;
public String getBaseMobile() {
return baseMobile;
}
... ... @@ -53,4 +56,12 @@ public class SnsMobileConfig {
public void setJavaServiceFlag(boolean javaServiceFlag) {
this.javaServiceFlag = javaServiceFlag;
}
public Integer getJavaServiceCostThreshold() {
return javaServiceCostThreshold;
}
public void setJavaServiceCostThreshold(Integer javaServiceCostThreshold) {
this.javaServiceCostThreshold = javaServiceCostThreshold;
}
}
... ...
... ... @@ -32,5 +32,5 @@ public interface ServiceAccessMapper {
Map<String,List<String>> getTimeoutInfoByContextAndApiName(NewJavaApiInfoReq req);
List<Map> getAlarmData();
List<Map> getAlarmData(Integer costThreshold);
}
... ...
... ... @@ -11,8 +11,6 @@ import org.apache.commons.lang.StringUtils;
import org.influxdb.dto.QueryResult;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.util.*;
... ... @@ -398,15 +396,16 @@ public class ServiceAccessMapperImpl extends InfluxDBQuery implements ServiceAcc
}
@Override
public List<Map> getAlarmData(){
String sql = String.format("select mean(cost) from service_access where %s and time > now() - 10s group by hostAddress,context",contextFilter);
public List<Map> getAlarmData(Integer costThreshold){
String sql = String.format("select mean(cost) from service_access where %s and time > now() - 10000s group by hostAddress,context",contextFilter);
List<Map> list = new ArrayList<>();
list.addAll(getAlarmData(InfluxDBContants.AWS,sql));
list.addAll(getAlarmData(InfluxDBContants.Q_CLOUD,sql));
list.addAll(getAlarmData(InfluxDBContants.AWS,sql,costThreshold));
list.addAll(getAlarmData(InfluxDBContants.Q_CLOUD,sql,costThreshold));
return list;
}
private List<Map> getAlarmData(String source,String sql){
private List<Map> getAlarmData(String source,String sql,Integer costThreshold){
List<Map> returnList = new ArrayList<>();
QueryResult queryResult = query(source, sql, InfluxDBContants.YOHO_EVENT);
QueryResult.Result rel = queryResult.getResults().get(0);
... ... @@ -417,7 +416,7 @@ public class ServiceAccessMapperImpl extends InfluxDBQuery implements ServiceAcc
String hostAddress = series.getTags().get("hostAddress");
String serviceName = series.getTags().get("context");
Double mean = (Double)series.getValues().get(0).get(1);
if(mean > 100){
if(mean > costThreshold){
Map map = new HashMap();
map.put("hostAddress",hostAddress);
map.put("serviceName",serviceName);
... ...
... ... @@ -208,7 +208,7 @@ public class NewJavaApiInfoCtrl {
for(int i = 0;i<strs.length;i++){
String ip = strs[i];
Point point = Point.measurement("service_access")
.tag("context","brower").tag("event","/brower").tag("host","/gateway")
.tag("context","brower").tag("event","/product/new/product/queryProductDetailByProductId").tag("host","/gateway")
.tag("hostAddress",ip).tag("src_service","app.product.promotion").tag("status_code","200")
.addField("cost", RandomUtils.nextInt(199) + 2000).addField("ip",ip)
.addField("stack","Total Delay [2005ms] /gateway\n +---[2005ms] - com.yoho.core.common.monitor.ThreadProfileInterceptor.preHandle - [enter:1477966424008,exit:1477966426013]\n +---[2005ms] - app.passport.signinAES. - [enter:1477966424008,exit:1477966426013]\n +---[1ms] - com.yoho.core.redis.YHValueOperations.increment - [enter:1477966424008,exit:1477966424009]\n +---[1ms] - com.yoho.core.redis.YHValueOperations.get - [enter:1477966424011,exit:1477966424012]\n +---[2000ms] - com.yoho.core.rest.client.HystrixServiceCaller.usersingle.login - [enter:1477966424012,exit:1477966426012]\n")
... ... @@ -228,7 +228,7 @@ public class NewJavaApiInfoCtrl {
for(int i = 0;i<strs.length;i++){
String ip = strs[i];
Point point = Point.measurement("service_server_exception")
.tag("context","brower").tag("event","/brower").tag("host","/brower/favorite/isFavorite")
.tag("context","brower").tag("event","/product/new/product/queryProductDetailByProductId").tag("host","/brower/favorite/isFavorite")
.tag("hostAddress",ip).tag("service_name","brower/favorite/isFavorite")
.addField("ip",ip)
.addField("stack","org.apache.catalina\n.connector.ClientAbortException: java.io.IOException: 断开的管道\n\tat org.apache.catalina.connector.OutputBuffer\n.realWriteBytes(OutputBuffer.java:393)\n\tat org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk\n.java:426)\n\tat org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:342)\n\tat org\n.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:317)\n\tat org.apache.catalina.connector\n.CoyoteOutputStream.flush(CoyoteOutputStream.java:110)\n\tat com.fasterxml.jackson.core.json.UTF8JsonGenerator\n.flush(UTF8JsonGenerator.java:1022)\n\tat com.fasterxml.jackson.databind.ObjectWriter.writeValue(ObjectWriter\n.java:891)\n\tat org.springframework.http.converter.json.AbstractJackson2HttpMessageConverter.writeInternal\n(AbstractJackson2HttpMessageConverter.java:264)\n\tat org.springframework.http.converter.AbstractGenericHttpMessageConverter\n.write(AbstractGenericHttpMessageConverter.java:100)\n\tat org.springframework.web.servlet.mvc.method\n.annotation.AbstractMessageConverterMethodProcessor.writeWithMessageConverters(AbstractMessageConverterMethodProcessor\n.java:202)\n\tat org.springframework.web.servlet.mvc.method.annotation.AbstractMessageConverterMethodProcessor\n.writeWithMessageConverters(AbstractMessageConverterMethodProcessor.java:133)\n\tat org.springframework\n.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor.handleReturnValue(RequestResponseBodyMethodProcessor\n.java:165)\n\tat org.springframework.web.method.support.HandlerMethodReturnValueHandlerComposite.handleReturnValue\n(HandlerMethodReturnValueHandlerComposite.java:80)\n\tat org.springframework.web.servlet.mvc.method.annotation\n.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:126)\n\tat org.springframework\n.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter\n.java:806)\n\tat org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal\n(RequestMappingHandlerAdapter.java:729)\n\tat org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter\n.handle(AbstractHandlerMethodAdapter.java:85)\n\tat org.springframework.web.servlet.DispatcherServlet\n.doDispatch(DispatcherServlet.java:959)\n\tat org.springframework.web.servlet.DispatcherServlet.doService\n(DispatcherServlet.java:893)\n\tat org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet\n.java:970)\n\tat org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)\n\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:648)\n\tat org.springframework.web.servlet\n.FrameworkServlet.service(FrameworkServlet.java:846)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet\n.java:729)\n\tat org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain\n.java:291)\n\tat org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java\n:206)\n\tat org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)\n\tat org.apache.catalina\n.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:239)\n\tat org.apache.catalina\n.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)\n\tat org.springframework.web\n.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:85)\n\tat org.springframework\n.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)\n\tat org.apache.catalina.core\n.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:239)\n\tat org.apache.catalina.core\n.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)\n\tat org.apache.catalina.core.StandardWrapperValve\n.invoke(StandardWrapperValve.java:217)\n\tat org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve\n.java:106)\n\tat org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502\n)\n\tat org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:142)\n\tat org.apache\n.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)\n\tat org.apache.catalina.core.StandardEngineValve\n.invoke(StandardEngineValve.java:88)\n\tat org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter\n.java:518)\n\tat org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java\n:1091)\n\tat org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java\n:673)\n\tat org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1500)\n\tat\n org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1456)\n\tat java.util.concurrent\n.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker\n.run(ThreadPoolExecutor.java:617)\n\tat org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run\n(TaskThread.java:61)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: java.io.IOException: 断开\n的管道\n\tat sun.nio.ch.FileDispatcherImpl.write0(Native Method)\n\tat sun.nio.ch.SocketDispatcher.write\n(SocketDispatcher.java:47)\n\tat sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)\n\tat sun.nio\n.ch.IOUtil.write(IOUtil.java:65)\n\tat sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471\n)\n\tat org.apache.tomcat.util.net.NioChannel.write(NioChannel.java:124)\n\tat org.apache.tomcat.util\n.net.NioBlockingSelector.write(NioBlockingSelector.java:101)\n\tat org.apache.tomcat.util.net.NioSelectorPool\n.write(NioSelectorPool.java:172)\n\tat org.apache.coyote.http11.InternalNioOutputBuffer.writeToSocket\n(InternalNioOutputBuffer.java:139)\n\tat org.apache.coyote.http11.InternalNioOutputBuffer.addToBB(InternalNioOutputBuffer\n.java:197)\n\tat org.apache.coyote.http11.InternalNioOutputBuffer.access$000(InternalNioOutputBuffer\n.java:41)\n\tat org.apache.coyote.http11.InternalNioOutputBuffer$SocketOutputBuffer.doWrite(InternalNioOutputBuffer\n.java:320)\n\tat org.apache.coyote.http11.filters.ChunkedOutputFilter.doWrite(ChunkedOutputFilter.java\n:118)\n\tat org.apache.coyote.http11.AbstractOutputBuffer.doWrite(AbstractOutputBuffer.java:256)\n\tat\n org.apache.coyote.Response.doWrite(Response.java:501)\n\tat org.apache.catalina.connector.OutputBuffer\n.realWriteBytes(OutputBuffer.java:388)\n\t... 47 more\n")
... ...
... ... @@ -17,6 +17,7 @@ import com.monitor.model.response.NewJavaApiInfoRep;
import com.monitor.mysql.mapper.MObjectInfoMapper;
import com.monitor.mysql.mapper.MTypeInfoMapper;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -366,13 +367,29 @@ public class NewJavaApiInfoServiceImpl implements NewJavaApiInfoService {
@Override
public void sendAlarmMsg()
{
List<Map> list = serviceAccessMapper.getAlarmData();
//获取aws和qq所有机器ip
List<MObjectInfo> mObjectInfos = mObjectInfoMapper.selectAllDistinctHostIP();
List<String> distinctIpList = new ArrayList<>();
for(MObjectInfo mObjectInfo : mObjectInfos){
distinctIpList.add(mObjectInfo.getMoHostIp());
}
//获取报警阀值
Integer costThreshold = snsMobileConfig.getJavaServiceCostThreshold();
if(costThreshold == null){
costThreshold = 200;
}
String nowString = DateTime.now().toString("yyyy-MM-dd HH:mm:ss");
List<Map> list = serviceAccessMapper.getAlarmData(costThreshold);
//获取cpu、内存、带宽使用情况
Map vmInfoMap = vmInfoMapper.getVMInfo();
String costMsgContentTemplate = "服务名:%s,ip:%s,平均耗时%dms,CPU使用率:%s,可用内存/总内存(MB):%s";
String bandWidthContentTemplate = "ip:%s,输入带宽 :%.2fMbps,输出带宽:%.2fMbps";
String costMsgContentTemplate = "时间:%s,服务名:%s,ip:%s,平均耗时%dms,CPU使用率:%s,可用内存/总内存(MB):%s";
String bandWidthContentTemplate = "时间:%s,ip:%s,输入带宽 :%.2fMbps,输出带宽:%.2fMbps";
for(Map map : list){
String hostAddress = (String)map.get("hostAddress");
if(!distinctIpList.contains(hostAddress)){
continue;
}
String serviceName = (String)map.get("serviceName");
Integer mean = ((Double)map.get("mean")).intValue();
String cpuRate = "";
... ... @@ -383,7 +400,7 @@ public class NewJavaApiInfoServiceImpl implements NewJavaApiInfoService {
memoryRate = "" + vmList.get(2) + "/" + vmList.get(3);
}
String costMsgContent = String.format(costMsgContentTemplate,serviceName,hostAddress,mean,cpuRate,memoryRate);
String costMsgContent = String.format(costMsgContentTemplate,nowString,serviceName,hostAddress,mean,cpuRate,memoryRate);
log.info("sendAlarmMsg costMsgContent is {}",costMsgContent);
boolean result = alarmMsgService.sendSms("JavaServiceAlarm",costMsgContent,snsMobileConfig.getJavaServiceMobile());
log.info("sendAlarmMsg result is {}",result);
... ... @@ -399,7 +416,7 @@ public class NewJavaApiInfoServiceImpl implements NewJavaApiInfoService {
Double inBandwidth = (Double)vmList.get(4);
Double outBandwidth = (Double)vmList.get(5);
if(inBandwidth > 300 || outBandwidth > 300){
String bandWidthContent = String.format(bandWidthContentTemplate,ip,inBandwidth,outBandwidth);
String bandWidthContent = String.format(bandWidthContentTemplate,nowString,ip,inBandwidth,outBandwidth);
log.info("sendAlarmMsg bandWidthContent is {}",bandWidthContent);
boolean result = alarmMsgService.sendSms("JavaServiceAlarm",bandWidthContent,snsMobileConfig.getJavaServiceMobile());
log.info("sendAlarmMsg result is {}",result);
... ...
... ... @@ -28,5 +28,7 @@ public interface MObjectInfoMapper {
List<MObjectInfo> selectJavaDistinctHostIP();
List<MObjectInfo> selectAllDistinctHostIP();
/* List<MObjectInfo> getHostMosInfo(int hostId);*/
}
... ...
... ... @@ -75,4 +75,9 @@
AND a.`parent_id` = (SELECT id FROM `type_info` t WHERE t.`alias` = 'java_app') GROUP BY b.`host_ip`
</select>
<select id="selectAllDistinctHostIP" resultType="com.model.MObjectInfo" resultMap="mobjectInfoMapper">
SELECT distinct host_ip FROM mobject_info
</select>
</mapper>
\ No newline at end of file
... ...
... ... @@ -13,6 +13,6 @@ redis_exception_mobile=13515100825,15905144483,18751986615,18502542319
java_service_alarm_mobile=13515100825,15905144483,18751986615,18502542319,18614066537
java_service_alarm_flag=true
java_service_alarm_cost_threshold=200
... ...
... ... @@ -17,5 +17,6 @@ redis_exception_mobile=15905144483
java_service_alarm_mobile=18614066537
java_service_alarm_flag=true
java_service_alarm_cost_threshold=200
... ...