Authored by jack

1.增加zabbix 全网机器性能监控

zabbix.jdbc.url=jdbc:mysql://10.66.0.191:3306/zabbix
zabbix.jdbc.user=zabbix_ops
zabbix.jdbc.password=yoho@zabbix
zabbix_type=gateway,order,product,promotion,resources,sns,users,uic,message,redis,twemproxy,memcached,rabbitmq
zabbix_cpu_alarm=60.00
zabbix_mem_alarm=0.3
zabbix_net_alarm=300
... ...
zabbix.jdbc.url=jdbc:mysql://192.168.102.76:3306/zabbix
zabbix.jdbc.user=root
zabbix.jdbc.password=123456
zabbix_type=jack
zabbix_cpu_alarm=60.00
zabbix_mem_alarm=0.3
zabbix_net_alarm=300
... ...
... ... @@ -7,6 +7,7 @@ import com.monitor.zabbix.model.ZabbixItemInfo;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Created by yoho on 2016/10/11.
... ... @@ -27,9 +28,13 @@ public interface Constants {
ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
Map<String, Integer> HOSTIP_TO_IDS = new HashMap<>();
Map<String, Integer> HOSTIP_TO_IDS = new ConcurrentHashMap<>();
Map<Integer, String> HOSTID_TO_IPS = new HashMap<>();
Map<Integer, String> HOSTID_TO_IPS = new ConcurrentHashMap<>();
Map<Integer, List<ZabbixItemInfo>> HOSTID_TO_ITEMS = new HashMap<>();
Map<Integer, List<ZabbixItemInfo>> HOSTID_TO_ITEMS = new ConcurrentHashMap<>();
Map<Integer, String> ZABBIXALARMIDMPA = new ConcurrentHashMap<>();
Map<Integer, List<ZabbixItemInfo>> ZABBIXALARMITEMMAP = new ConcurrentHashMap<>();
}
... ...
... ... @@ -60,7 +60,7 @@ public class PointBuilder {
return point;
}
private static Double findValueByKey(String key, List<ZabbixItemInfo> items, List<Object> historyItems) {
public static Double findValueByKey(String key, List<ZabbixItemInfo> items, List<Object> historyItems) {
int itemId = 0;
... ...
... ... @@ -10,4 +10,6 @@ import java.util.List;
public interface ZabbixHostMapper {
List<ZabbixHostInfo> queryHostInfo(List<String> ipList);
List<ZabbixHostInfo> queryAllHostInfo();
}
... ...
package com.monitor.zabbix.service;
/**
* Created by yoho on 2016/11/2.
*/
import com.monitor.common.config.SnsMobileConfig;
import com.monitor.common.service.AlarmMsgService;
import com.monitor.zabbix.constants.Constants;
import com.monitor.zabbix.enums.NetIfEnum;
import com.monitor.zabbix.enums.SystemCpuEnum;
import com.monitor.zabbix.enums.VmMemoryEnum;
import com.monitor.zabbix.impl.PointBuilder;
import com.monitor.zabbix.mapper.ZabbixHistoryMapper;
import com.monitor.zabbix.mapper.ZabbixHostMapper;
import com.monitor.zabbix.mapper.ZabbixItemMapper;
import com.monitor.zabbix.model.ZabbixDHistoryInfo;
import com.monitor.zabbix.model.ZabbixHostInfo;
import com.monitor.zabbix.model.ZabbixItemInfo;
import com.monitor.zabbix.model.ZabbixUHistoryInfo;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* 全网监控
*/
@Service
@EnableScheduling
public class ZabbixAlarm {
private static String ALARMTEMPLATE = "时间:%s,ip:%s,CPU使用率:%s,可用内存/总内存(MB):%s,输入带宽:%.2fMbps,输出带宽:%.2fMbps";
public static final Logger DEBUG = LoggerFactory.getLogger(ZabbixAlarm.class);
@Autowired
ZabbixHostMapper hostMapper;
@Autowired
ZabbixItemMapper itemMapper;
@Autowired
ZabbixHistoryMapper historyMapper;
@Value("${zabbix_cpu_alarm}")
Double zabbixCpuAlarm;
@Value("${zabbix_mem_alarm}")
Double zabbixMemAlarm;
@Value("${zabbix_net_alarm}")
Double zabbixNetAlarm;
@Autowired
private AlarmMsgService alarmMsgService;
@Autowired
private SnsMobileConfig snsMobileConfig;
private final class ZabbixTask implements Runnable {
List<Integer> hostIdList;
public ZabbixTask(List<Integer> idList) {
this.hostIdList = idList;
}
@Override
public void run() {
if (null == hostIdList || 0 == hostIdList.size()) {
return;
}
queryIdToItem(hostIdList);
for (Integer hostId : hostIdList) {
queryHistory(hostId);
}
}
}
@Scheduled(fixedRate = 60 * 1000)
public void dispatchAlarm() {
queryAllHostInfo();
List<Integer> allIdList = new ArrayList<>();
allIdList.addAll(Constants.ZABBIXALARMIDMPA.keySet());
int executorSize = allIdList.size() / 100 + 1;
ExecutorService executor = Executors.newFixedThreadPool(executorSize, new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("zabbix-alarm-executor");
thread.setDaemon(true);
return thread;
}
});
int start = 0;
int end = 0;
for (int i = 0; i < executorSize; i++) {
if (end >= allIdList.size() - 1) {
break;
}
start = end;
end += (i + 1) * 100;
end = end >= allIdList.size() ? (allIdList.size() - 1) : end;
List<Integer> taskList = allIdList.subList(start, end + 1);
//提交监控任务
executor.submit(new ZabbixTask(taskList));
}
executor.shutdown();
}
//查询所有host信息
private void queryAllHostInfo() {
List<ZabbixHostInfo> zabbixHostInfoList = hostMapper.queryAllHostInfo();
for (ZabbixHostInfo zabbixHostInfo : zabbixHostInfoList) {
Constants.ZABBIXALARMIDMPA.put(zabbixHostInfo.getHostId(), zabbixHostInfo.getIp());
}
}
/**
* 根据id查询item,转存内存map中,供后续使用
*/
private void queryIdToItem(List<Integer> idList) {
List<Integer> queryIdList = new ArrayList<>();
for (Integer id : idList) {
if (Constants.ZABBIXALARMIDMPA.containsKey(id)) {
continue;
}
queryIdList.add(id);
}
if (idList.isEmpty()) {
DEBUG.info("Not found new hostId in {}, no need to query item....", idList);
return;
}
List<ZabbixItemInfo> itemInfoList = itemMapper.queryItemInfo(idList);
DEBUG.info("Found id to items info {}", itemInfoList);
for (ZabbixItemInfo itemInfo : itemInfoList) {
List<ZabbixItemInfo> itemInfos = Constants.ZABBIXALARMITEMMAP.get(itemInfo.getHostId());
if (null == itemInfos) {
itemInfos = new ArrayList<>();
}
itemInfos.add(itemInfo);
Constants.ZABBIXALARMITEMMAP.put(itemInfo.getHostId(), itemInfos);
}
}
/**
* 根据ip查询 item的历史记录
*/
private void queryHistory(Integer hostId) {
List<ZabbixItemInfo> itemInfos = Constants.ZABBIXALARMITEMMAP.get(hostId);
if (null == itemInfos) {
return;
}
List<Object> historyList = new ArrayList<>();
for (ZabbixItemInfo itemInfo : itemInfos) {
if (null == itemInfo) {
continue;
}
//system.cpu在另外一个库中,调用另一个接口
if (StringUtils.startsWith(itemInfo.getKeyName(), "system.cpu")) {
ZabbixDHistoryInfo dHistoryInfo = historyMapper.queryDHistoryInfo(itemInfo.getItemId());
if (null != dHistoryInfo) {
historyList.add(dHistoryInfo);
}
} else {
ZabbixUHistoryInfo uHistoryInfo = historyMapper.queryUHistoryInfo(itemInfo.getItemId());
if (null != uHistoryInfo) {
historyList.add(uHistoryInfo);
}
}
}
checkAlarm(hostId, itemInfos, historyList);
}
private void checkAlarm(Integer hostId, List<ZabbixItemInfo> itemInfos, List<Object> historyList) {
String ip = Constants.ZABBIXALARMIDMPA.get(hostId);
if (StringUtils.isBlank(ip) || StringUtils.equals("127.0.0.1", ip)) {
return;
}
Double idleCpu = PointBuilder.findValueByKey(SystemCpuEnum.CPU_UTIL_IDLE.key(), itemInfos, historyList);
idleCpu = (0 == idleCpu) ? 100 : idleCpu;
Double avMem = PointBuilder.findValueByKey(VmMemoryEnum.MEMORY_SIZE_AVAILABLE.key(), itemInfos, historyList);
Double toMem = PointBuilder.findValueByKey(VmMemoryEnum.MEMORY_SIZE_TOTAL.key(), itemInfos, historyList);
Double memPer = (0 == toMem) ? 1 : (avMem / toMem);
Double inNet = PointBuilder.findValueByKey(NetIfEnum.NET_IF_IN.key(), itemInfos, historyList);
Double outNet = PointBuilder.findValueByKey(NetIfEnum.NET_IF_OUT.key(), itemInfos, historyList);
if (zabbixCpuAlarm > idleCpu || zabbixMemAlarm > memPer || zabbixNetAlarm < inNet || zabbixNetAlarm < outNet) {
String nowString = DateTime.now().toString("yyyy-MM-dd HH:mm:ss");
String alarmInfo = String.format(ALARMTEMPLATE, nowString, ip, String.valueOf(100 - idleCpu), String.valueOf(avMem) + " / " + String.valueOf(toMem), inNet, outNet);
DEBUG.info("Alarm vm info {}", alarmInfo);
alarmMsgService.sendSms("服务器性能告警", alarmInfo, snsMobileConfig.getBaseMobile());
}
}
}
... ...
... ... @@ -16,4 +16,8 @@
#{item}
</foreach>
</select>
<select id="queryAllHostInfo" resultMap="hostInfoMapper">
SELECT interfaceid,hostid,ip FROM `interface` ;
</select>
</mapper>
\ No newline at end of file
... ...