Authored by jack

1.增加zabbix 全网机器性能监控

1 zabbix.jdbc.url=jdbc:mysql://10.66.0.191:3306/zabbix 1 zabbix.jdbc.url=jdbc:mysql://10.66.0.191:3306/zabbix
2 -  
3 zabbix.jdbc.user=zabbix_ops 2 zabbix.jdbc.user=zabbix_ops
4 -  
5 zabbix.jdbc.password=yoho@zabbix 3 zabbix.jdbc.password=yoho@zabbix
  4 +zabbix_type=gateway,order,product,promotion,resources,sns,users,uic,message,redis,twemproxy,memcached,rabbitmq
  5 +zabbix_cpu_alarm=60.00
  6 +zabbix_mem_alarm=0.3
  7 +zabbix_net_alarm=300
6 8
7 -zabbix_type=gateway,order,product,promotion,resources,sns,users,uic,message,redis,twemproxy,memcached,rabbitmq  
1 -  
2 zabbix.jdbc.url=jdbc:mysql://192.168.102.76:3306/zabbix 1 zabbix.jdbc.url=jdbc:mysql://192.168.102.76:3306/zabbix
3 -  
4 zabbix.jdbc.user=root 2 zabbix.jdbc.user=root
5 -  
6 zabbix.jdbc.password=123456 3 zabbix.jdbc.password=123456
7 -  
8 zabbix_type=jack 4 zabbix_type=jack
  5 +zabbix_cpu_alarm=60.00
  6 +zabbix_mem_alarm=0.3
  7 +zabbix_net_alarm=300
9 8
10 9
11 10
@@ -7,6 +7,7 @@ import com.monitor.zabbix.model.ZabbixItemInfo; @@ -7,6 +7,7 @@ import com.monitor.zabbix.model.ZabbixItemInfo;
7 import java.util.HashMap; 7 import java.util.HashMap;
8 import java.util.List; 8 import java.util.List;
9 import java.util.Map; 9 import java.util.Map;
  10 +import java.util.concurrent.ConcurrentHashMap;
10 11
11 /** 12 /**
12 * Created by yoho on 2016/10/11. 13 * Created by yoho on 2016/10/11.
@@ -27,9 +28,13 @@ public interface Constants { @@ -27,9 +28,13 @@ public interface Constants {
27 28
28 ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 29 ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
29 30
30 - Map<String, Integer> HOSTIP_TO_IDS = new HashMap<>(); 31 + Map<String, Integer> HOSTIP_TO_IDS = new ConcurrentHashMap<>();
31 32
32 - Map<Integer, String> HOSTID_TO_IPS = new HashMap<>(); 33 + Map<Integer, String> HOSTID_TO_IPS = new ConcurrentHashMap<>();
33 34
34 - Map<Integer, List<ZabbixItemInfo>> HOSTID_TO_ITEMS = new HashMap<>(); 35 + Map<Integer, List<ZabbixItemInfo>> HOSTID_TO_ITEMS = new ConcurrentHashMap<>();
  36 +
  37 + Map<Integer, String> ZABBIXALARMIDMPA = new ConcurrentHashMap<>();
  38 +
  39 + Map<Integer, List<ZabbixItemInfo>> ZABBIXALARMITEMMAP = new ConcurrentHashMap<>();
35 } 40 }
@@ -60,7 +60,7 @@ public class PointBuilder { @@ -60,7 +60,7 @@ public class PointBuilder {
60 return point; 60 return point;
61 } 61 }
62 62
63 - private static Double findValueByKey(String key, List<ZabbixItemInfo> items, List<Object> historyItems) { 63 + public static Double findValueByKey(String key, List<ZabbixItemInfo> items, List<Object> historyItems) {
64 64
65 int itemId = 0; 65 int itemId = 0;
66 66
@@ -10,4 +10,6 @@ import java.util.List; @@ -10,4 +10,6 @@ import java.util.List;
10 public interface ZabbixHostMapper { 10 public interface ZabbixHostMapper {
11 11
12 List<ZabbixHostInfo> queryHostInfo(List<String> ipList); 12 List<ZabbixHostInfo> queryHostInfo(List<String> ipList);
  13 +
  14 + List<ZabbixHostInfo> queryAllHostInfo();
13 } 15 }
  1 +package com.monitor.zabbix.service;
  2 +
  3 +/**
  4 + * Created by yoho on 2016/11/2.
  5 + */
  6 +
  7 +import com.monitor.common.config.SnsMobileConfig;
  8 +import com.monitor.common.service.AlarmMsgService;
  9 +import com.monitor.zabbix.constants.Constants;
  10 +import com.monitor.zabbix.enums.NetIfEnum;
  11 +import com.monitor.zabbix.enums.SystemCpuEnum;
  12 +import com.monitor.zabbix.enums.VmMemoryEnum;
  13 +import com.monitor.zabbix.impl.PointBuilder;
  14 +import com.monitor.zabbix.mapper.ZabbixHistoryMapper;
  15 +import com.monitor.zabbix.mapper.ZabbixHostMapper;
  16 +import com.monitor.zabbix.mapper.ZabbixItemMapper;
  17 +import com.monitor.zabbix.model.ZabbixDHistoryInfo;
  18 +import com.monitor.zabbix.model.ZabbixHostInfo;
  19 +import com.monitor.zabbix.model.ZabbixItemInfo;
  20 +import com.monitor.zabbix.model.ZabbixUHistoryInfo;
  21 +import org.apache.commons.lang.StringUtils;
  22 +import org.joda.time.DateTime;
  23 +import org.slf4j.Logger;
  24 +import org.slf4j.LoggerFactory;
  25 +import org.springframework.beans.factory.annotation.Autowired;
  26 +import org.springframework.beans.factory.annotation.Value;
  27 +import org.springframework.scheduling.annotation.EnableScheduling;
  28 +import org.springframework.scheduling.annotation.Scheduled;
  29 +import org.springframework.stereotype.Service;
  30 +
  31 +import java.util.ArrayList;
  32 +import java.util.List;
  33 +import java.util.concurrent.ExecutorService;
  34 +import java.util.concurrent.Executors;
  35 +import java.util.concurrent.ThreadFactory;
  36 +
  37 +/**
  38 + * 全网监控
  39 + */
  40 +@Service
  41 +@EnableScheduling
  42 +public class ZabbixAlarm {
  43 +
  44 + private static String ALARMTEMPLATE = "时间:%s,ip:%s,CPU使用率:%s,可用内存/总内存(MB):%s,输入带宽:%.2fMbps,输出带宽:%.2fMbps";
  45 +
  46 + public static final Logger DEBUG = LoggerFactory.getLogger(ZabbixAlarm.class);
  47 +
  48 + @Autowired
  49 + ZabbixHostMapper hostMapper;
  50 +
  51 + @Autowired
  52 + ZabbixItemMapper itemMapper;
  53 +
  54 + @Autowired
  55 + ZabbixHistoryMapper historyMapper;
  56 +
  57 + @Value("${zabbix_cpu_alarm}")
  58 + Double zabbixCpuAlarm;
  59 +
  60 + @Value("${zabbix_mem_alarm}")
  61 + Double zabbixMemAlarm;
  62 +
  63 + @Value("${zabbix_net_alarm}")
  64 + Double zabbixNetAlarm;
  65 +
  66 + @Autowired
  67 + private AlarmMsgService alarmMsgService;
  68 +
  69 + @Autowired
  70 + private SnsMobileConfig snsMobileConfig;
  71 +
  72 +
  73 + private final class ZabbixTask implements Runnable {
  74 + List<Integer> hostIdList;
  75 +
  76 + public ZabbixTask(List<Integer> idList) {
  77 +
  78 + this.hostIdList = idList;
  79 + }
  80 +
  81 + @Override
  82 + public void run() {
  83 + if (null == hostIdList || 0 == hostIdList.size()) {
  84 + return;
  85 + }
  86 + queryIdToItem(hostIdList);
  87 +
  88 + for (Integer hostId : hostIdList) {
  89 +
  90 + queryHistory(hostId);
  91 + }
  92 + }
  93 + }
  94 +
  95 +
  96 + @Scheduled(fixedRate = 60 * 1000)
  97 + public void dispatchAlarm() {
  98 +
  99 + queryAllHostInfo();
  100 +
  101 + List<Integer> allIdList = new ArrayList<>();
  102 +
  103 + allIdList.addAll(Constants.ZABBIXALARMIDMPA.keySet());
  104 +
  105 + int executorSize = allIdList.size() / 100 + 1;
  106 +
  107 + ExecutorService executor = Executors.newFixedThreadPool(executorSize, new ThreadFactory() {
  108 + @Override
  109 + public Thread newThread(Runnable r) {
  110 + Thread thread = new Thread(r);
  111 + thread.setName("zabbix-alarm-executor");
  112 + thread.setDaemon(true);
  113 + return thread;
  114 + }
  115 + });
  116 +
  117 + int start = 0;
  118 +
  119 + int end = 0;
  120 +
  121 + for (int i = 0; i < executorSize; i++) {
  122 +
  123 + if (end >= allIdList.size() - 1) {
  124 + break;
  125 + }
  126 +
  127 + start = end;
  128 +
  129 + end += (i + 1) * 100;
  130 +
  131 + end = end >= allIdList.size() ? (allIdList.size() - 1) : end;
  132 +
  133 + List<Integer> taskList = allIdList.subList(start, end + 1);
  134 + //提交监控任务
  135 + executor.submit(new ZabbixTask(taskList));
  136 + }
  137 +
  138 + executor.shutdown();
  139 + }
  140 +
  141 + //查询所有host信息
  142 + private void queryAllHostInfo() {
  143 + List<ZabbixHostInfo> zabbixHostInfoList = hostMapper.queryAllHostInfo();
  144 +
  145 + for (ZabbixHostInfo zabbixHostInfo : zabbixHostInfoList) {
  146 +
  147 + Constants.ZABBIXALARMIDMPA.put(zabbixHostInfo.getHostId(), zabbixHostInfo.getIp());
  148 + }
  149 + }
  150 +
  151 + /**
  152 + * 根据id查询item,转存内存map中,供后续使用
  153 + */
  154 + private void queryIdToItem(List<Integer> idList) {
  155 + List<Integer> queryIdList = new ArrayList<>();
  156 +
  157 + for (Integer id : idList) {
  158 +
  159 + if (Constants.ZABBIXALARMIDMPA.containsKey(id)) {
  160 + continue;
  161 + }
  162 +
  163 + queryIdList.add(id);
  164 + }
  165 +
  166 + if (idList.isEmpty()) {
  167 + DEBUG.info("Not found new hostId in {}, no need to query item....", idList);
  168 + return;
  169 + }
  170 +
  171 + List<ZabbixItemInfo> itemInfoList = itemMapper.queryItemInfo(idList);
  172 +
  173 + DEBUG.info("Found id to items info {}", itemInfoList);
  174 +
  175 + for (ZabbixItemInfo itemInfo : itemInfoList) {
  176 + List<ZabbixItemInfo> itemInfos = Constants.ZABBIXALARMITEMMAP.get(itemInfo.getHostId());
  177 +
  178 + if (null == itemInfos) {
  179 + itemInfos = new ArrayList<>();
  180 + }
  181 +
  182 + itemInfos.add(itemInfo);
  183 +
  184 + Constants.ZABBIXALARMITEMMAP.put(itemInfo.getHostId(), itemInfos);
  185 + }
  186 + }
  187 +
  188 +
  189 + /**
  190 + * 根据ip查询 item的历史记录
  191 + */
  192 + private void queryHistory(Integer hostId) {
  193 +
  194 + List<ZabbixItemInfo> itemInfos = Constants.ZABBIXALARMITEMMAP.get(hostId);
  195 +
  196 + if (null == itemInfos) {
  197 + return;
  198 + }
  199 +
  200 + List<Object> historyList = new ArrayList<>();
  201 +
  202 + for (ZabbixItemInfo itemInfo : itemInfos) {
  203 + if (null == itemInfo) {
  204 + continue;
  205 + }
  206 +
  207 + //system.cpu在另外一个库中,调用另一个接口
  208 + if (StringUtils.startsWith(itemInfo.getKeyName(), "system.cpu")) {
  209 +
  210 + ZabbixDHistoryInfo dHistoryInfo = historyMapper.queryDHistoryInfo(itemInfo.getItemId());
  211 +
  212 + if (null != dHistoryInfo) {
  213 + historyList.add(dHistoryInfo);
  214 + }
  215 + } else {
  216 + ZabbixUHistoryInfo uHistoryInfo = historyMapper.queryUHistoryInfo(itemInfo.getItemId());
  217 +
  218 + if (null != uHistoryInfo) {
  219 + historyList.add(uHistoryInfo);
  220 + }
  221 + }
  222 + }
  223 +
  224 + checkAlarm(hostId, itemInfos, historyList);
  225 + }
  226 +
  227 + private void checkAlarm(Integer hostId, List<ZabbixItemInfo> itemInfos, List<Object> historyList) {
  228 +
  229 + String ip = Constants.ZABBIXALARMIDMPA.get(hostId);
  230 +
  231 + if (StringUtils.isBlank(ip) || StringUtils.equals("127.0.0.1", ip)) {
  232 + return;
  233 + }
  234 +
  235 + Double idleCpu = PointBuilder.findValueByKey(SystemCpuEnum.CPU_UTIL_IDLE.key(), itemInfos, historyList);
  236 +
  237 + idleCpu = (0 == idleCpu) ? 100 : idleCpu;
  238 +
  239 + Double avMem = PointBuilder.findValueByKey(VmMemoryEnum.MEMORY_SIZE_AVAILABLE.key(), itemInfos, historyList);
  240 +
  241 + Double toMem = PointBuilder.findValueByKey(VmMemoryEnum.MEMORY_SIZE_TOTAL.key(), itemInfos, historyList);
  242 +
  243 + Double memPer = (0 == toMem) ? 1 : (avMem / toMem);
  244 +
  245 + Double inNet = PointBuilder.findValueByKey(NetIfEnum.NET_IF_IN.key(), itemInfos, historyList);
  246 +
  247 + Double outNet = PointBuilder.findValueByKey(NetIfEnum.NET_IF_OUT.key(), itemInfos, historyList);
  248 +
  249 + if (zabbixCpuAlarm > idleCpu || zabbixMemAlarm > memPer || zabbixNetAlarm < inNet || zabbixNetAlarm < outNet) {
  250 +
  251 + String nowString = DateTime.now().toString("yyyy-MM-dd HH:mm:ss");
  252 +
  253 + String alarmInfo = String.format(ALARMTEMPLATE, nowString, ip, String.valueOf(100 - idleCpu), String.valueOf(avMem) + " / " + String.valueOf(toMem), inNet, outNet);
  254 +
  255 + DEBUG.info("Alarm vm info {}", alarmInfo);
  256 +
  257 + alarmMsgService.sendSms("服务器性能告警", alarmInfo, snsMobileConfig.getBaseMobile());
  258 + }
  259 + }
  260 +}
@@ -16,4 +16,8 @@ @@ -16,4 +16,8 @@
16 #{item} 16 #{item}
17 </foreach> 17 </foreach>
18 </select> 18 </select>
  19 +
  20 + <select id="queryAllHostInfo" resultMap="hostInfoMapper">
  21 + SELECT interfaceid,hostid,ip FROM `interface` ;
  22 + </select>
19 </mapper> 23 </mapper>