ops / monitor-service · Commits

Go to a project

Authored by linlong
Commit 7831716f7619726c6beb8ad66c2dfbb6135744fd 1 parent 97cca17b

update

Inline Side-by-side
Showing 1 changed file with 48 additions and 30 deletions
... ... @@ -55,10 +55,12 @@ public class MemcachedMonitorImpl {
@Scheduled(fixedRate = 2 * 60 * 1000)
public void monitor() {
//遍历所有的redis / twemproxy
MemConstants.MEMCACHED_NAME_MAP.clear();
//遍历所有的memcached
serversMap = getServersMap();
List<MObjectInfo> memcachedList = new ArrayList<>();
List<MObjectInfo> memcachedList = new ArrayList<>();
StringBuffer monitorUrl = new StringBuffer(); //监控url，将所有memcached的ip port进行拼接
for (Map.Entry<String, List<MObjectInfo>> entry : serversMap.entrySet()) {
List<MObjectInfo> info = entry.getValue();
... ... @@ -67,20 +69,24 @@ public class MemcachedMonitorImpl {
memcachedList.add(mobjectInfo);
String ip = mobjectInfo.getMoHostIp();
String port = mobjectInfo.getMoTags().split(":")[1];
MemConstants.MEMCACHED_NAME_MAP.put(ip + ":" + port,mobjectInfo.getMoName());
MemConstants.MEMCACHED_NAME_MAP.put(ip + ":" + port, mobjectInfo.getMoName());
monitorUrl.append(ip + ":" + port + " ");
}
}
}
doMonitor(monitorUrl);
doMonitor(monitorUrl);
}
/**
* 执行监控任务
* @param monitorUrl 监控url
*/
private void doMonitor(StringBuffer monitorUrl) {
Map<InetSocketAddress, Map<String, String>> result = new HashMap<>();
try {
MemcachedClient client = new XMemcachedClient(AddrUtil.getAddresses(monitorUrl.toString().trim()));
result = client.getStats();
LOGGER.debug("MemcachedMonitorImpl.doMonitor:get memcached status is {}",result);
LOGGER.debug("MemcachedMonitorImpl.doMonitor:get memcached status is {}", result);
client.shutdown();
} catch (IOException e) {
LOGGER.error("MemcachedMonitorImpl.doMonitor:creatr/shutdown memcached client occurs Exception: ", e.getMessage());
... ... @@ -95,13 +101,23 @@ public class MemcachedMonitorImpl {
LOGGER.error("MemcachedMonitorImpl.doMonitor:get memcached status occurs Exception: ", e.getMessage());
this.getAlarmMsgService().sendSms("Memcached", "Can not get Memcached status" + monitorUrl.toString() + " .", this.getSnsMobileConfig().getBaseMobile());
}
checkAllStatus(monitorUrl.toString().trim(),result);
if(!CollectionUtils.isEmpty(result)){
for(Map.Entry<InetSocketAddress, Map<String, String>> entry : result.entrySet()){
//校验是否所有memcached都返回状态，对未返回状态的memcached短信告警
checkAllStatus(monitorUrl.toString().trim(), result);
processAllStatus(result);
}
/**
* 处理memcached的返回结果,缓存到本地的 MemConstants.MEMCACHED_INFO_MAP
* @param result
*/
private void processAllStatus(Map<InetSocketAddress, Map<String, String>> result) {
if (!CollectionUtils.isEmpty(result)) {
for (Map.Entry<InetSocketAddress, Map<String, String>> entry : result.entrySet()) {
InetSocketAddress address = entry.getKey();
Map<String, String> value = entry.getValue();
String url = address.getAddress()+":"+address.getPort();
url = url.substring(1,url.length());
String url = address.getAddress() + ":" + address.getPort();
url = url.substring(1, url.length());
MemcachedInfo info = new MemcachedInfo();
//每秒请求数
... ... @@ -111,10 +127,10 @@ public class MemcachedMonitorImpl {
info.setOldSet(cmdSet);
if (MemConstants.MEMCACHED_INFO_MAP.containsKey(url)) {
long oldGet = MemConstants.MEMCACHED_INFO_MAP.get(url).getOldGet();
info.setCmdGet((cmdGet-oldGet)/120.0);
info.setCmdGet((cmdGet - oldGet) / 120.0);
long oldSet = MemConstants.MEMCACHED_INFO_MAP.get(url).getOldSet();
info.setCmdSet((cmdSet-oldSet)/120.0);
info.setCmdSet((cmdSet - oldSet) / 120.0);
}
info.setCurConnections(Integer.parseInt(value.get("curr_connections")));
... ... @@ -124,46 +140,49 @@ public class MemcachedMonitorImpl {
long limit_maxbytes = Long.parseLong(value.get("limit_maxbytes"));
long bytes = Long.parseLong(value.get("bytes"));
long uptime = Long.parseLong(value.get("uptime"));
LOGGER.info(url+" monitor info： com_get is {},cmd_set is {},get_hit is {},get_misses is {},limit_maxbytes is {},bytes is {},uptime is {}",cmdGet,cmdSet,get_hit,get_misses,limit_maxbytes,bytes,uptime);
if((get_hit+get_misses)==0){
LOGGER.debug(url + " monitor info： com_get is {},cmd_set is {},get_hit is {},get_misses is {},limit_maxbytes is {},bytes is {},uptime is {}", cmdGet, cmdSet, get_hit, get_misses, limit_maxbytes, bytes, uptime);
if ((get_hit + get_misses) == 0) {
info.setHitRate("100.0%");
}else{
info.setHitRate(new DecimalFormat("0.00").format((get_hit*100.0/(get_hit+get_misses)))+"%");
} else {
info.setHitRate(new DecimalFormat("0.00").format((get_hit * 100.0 / (get_hit + get_misses))) + "%");
}
info.setMaxMemory(new DecimalFormat("0.00").format(limit_maxbytes*1.0/1024/1024/1024)+" G");
info.setUpTime(uptime/(60*60*24)+" Days");
info.setUseMemory(new DecimalFormat("0.00").format(bytes*1.0/1024/1024/1024)+" G");
MemConstants.MEMCACHED_INFO_MAP.put(url,info);
info.setMaxMemory(new DecimalFormat("0.00").format(limit_maxbytes * 1.0 / 1024 / 1024 / 1024) + " G");
info.setUpTime(uptime / (60 * 60 * 24) + " Days");
info.setUseMemory(new DecimalFormat("0.00").format(bytes * 1.0 / 1024 / 1024 / 1024) + " G");
MemConstants.MEMCACHED_INFO_MAP.put(url, info);
}
}
}
/**
* 检查monitorUrl的地址是否都有返回状态
*
* @param monitorUrl
* @param result
*/
private void checkAllStatus(String monitorUrl, Map<InetSocketAddress, Map<String, String>> result) {
String[] monitorUrls = monitorUrl.split(" ");
Set<String> monitorSet = new HashSet(Arrays.asList(monitorUrls));
for(String s:monitorSet ){
s="/"+s;
Set<InetSocketAddress> monitorUrlSet = new HashSet<>();
for (String s : monitorSet) {
String[] ipPort = s.split(":");
String ip = ipPort[0];
int port = Integer.parseInt(ipPort[1]);
monitorUrlSet.add(new InetSocketAddress(ip, port));
}
//如果监控url的地址 多于返回的状态条数，则检查哪个url未得到状态并告警
if(!CollectionUtils.isEmpty(result)&&monitorUrls.length>result.size()){
Set<InetSocketAddress> statusSet = result.keySet();
monitorSet.removeAll(statusSet);
if (!CollectionUtils.isEmpty(result) && monitorUrls.length > result.size()) {
Set<InetSocketAddress> statusSet = result.keySet();
monitorUrlSet.removeAll(statusSet);
StringBuffer failUrl = new StringBuffer();
for(String s:monitorSet){
failUrl.append(s+" ");
for (InetSocketAddress address: monitorUrlSet) {
failUrl.append(address.toString()+" ");
}
this.getAlarmMsgService().sendSms("Memcached", "Failed to get Memcached status" + failUrl.toString() + " .", this.getSnsMobileConfig().getBaseMobile());
}
}
private Map<String, List<MObjectInfo>> getServersMap() {
Map<String, List<MObjectInfo>> serversMap = new HashMap<>();
... ... @@ -184,7 +203,6 @@ public class MemcachedMonitorImpl {
List<TypeInfo> childTypeList = typeService.queryChildTypesInfo(parentTypeInfo.getTypeId());
for (TypeInfo typeInfo : childTypeList) {
List<MObjectInfo> mObjectInfos = mobjectService.queryMObjectsInfoByType(typeInfo.getTypeId());
... ...