Showing
1 changed file
with
0 additions
and
337 deletions
1 | -package com.monitor.middleware.redis.service.impl; | ||
2 | - | ||
3 | -import com.alibaba.fastjson.JSONArray; | ||
4 | -import com.alibaba.fastjson.JSONObject; | ||
5 | -import com.model.MObjectInfo; | ||
6 | -import com.model.RedisMonitor; | ||
7 | -import com.model.TypeInfo; | ||
8 | -import com.monitor.common.config.SnsMobileConfig; | ||
9 | -import com.monitor.common.service.AlarmMsgService; | ||
10 | -import com.monitor.common.util.HttpRestClient; | ||
11 | -import com.monitor.common.util.RedisCommonUtil; | ||
12 | -import com.monitor.common.util.TelnetUtils; | ||
13 | -import com.monitor.middleware.redis.service.IRedisMonitorHandleService; | ||
14 | -import com.monitor.mysql.mapper.MObjectInfoMapper; | ||
15 | -import com.monitor.mysql.mapper.MTypeInfoMapper; | ||
16 | -import com.monitor.mysql.mapper.RedisMonitorMapper; | ||
17 | -import org.apache.commons.lang.StringUtils; | ||
18 | -import org.slf4j.Logger; | ||
19 | -import org.slf4j.LoggerFactory; | ||
20 | -import org.springframework.beans.factory.annotation.Autowired; | ||
21 | -import org.springframework.scheduling.annotation.EnableScheduling; | ||
22 | -import org.springframework.stereotype.Service; | ||
23 | -import org.springframework.util.CollectionUtils; | ||
24 | -import java.math.BigDecimal; | ||
25 | -import java.util.*; | ||
26 | - | ||
27 | -@EnableScheduling | ||
28 | -@Service | ||
29 | -public class RedisMonitorHandleServiceImpl implements IRedisMonitorHandleService { | ||
30 | - | ||
31 | - Logger log = LoggerFactory.getLogger(RedisMonitorHandleServiceImpl.class); | ||
32 | - | ||
33 | - @Autowired | ||
34 | - HttpRestClient httpRestClient; | ||
35 | - | ||
36 | - @Autowired | ||
37 | - MTypeInfoMapper mTypeInfoMapper; | ||
38 | - | ||
39 | - @Autowired | ||
40 | - MObjectInfoMapper mObjectInfoMapper; | ||
41 | - | ||
42 | - @Autowired | ||
43 | - RedisMonitorMapper redisMonitorMapper; | ||
44 | - | ||
45 | - @Autowired | ||
46 | - public AlarmMsgService alarmMsgService; | ||
47 | - | ||
48 | - @Autowired | ||
49 | - private SnsMobileConfig snsMobileConfig; | ||
50 | - | ||
51 | - @Override | ||
52 | - public void redisMonitor() { | ||
53 | - List<RedisMonitor> redisInfoList=new ArrayList<RedisMonitor>(); | ||
54 | - List<String> twemproxyAlarmList=new ArrayList<String>(); | ||
55 | - List<String> redisAlarmList=new ArrayList<String>(); | ||
56 | - List<String> slaveAlarmList=new ArrayList<String>(); | ||
57 | - | ||
58 | - TypeInfo typeInfo=mTypeInfoMapper.selectTypeInfoByName("redis"); | ||
59 | - if(typeInfo==null){ | ||
60 | - return; | ||
61 | - } | ||
62 | - | ||
63 | - List<TypeInfo> typeInfoList=mTypeInfoMapper.getChildTypesInfo(typeInfo.getTypeId()); | ||
64 | - if(CollectionUtils.isEmpty(typeInfoList)){ | ||
65 | - return; | ||
66 | - } | ||
67 | - | ||
68 | - Map<Integer,String> redisTweproxyMap=new HashMap<Integer,String>(); | ||
69 | - List<Integer> querParamList=new ArrayList<Integer>(); | ||
70 | - for(TypeInfo tObj:typeInfoList){ | ||
71 | - List<TypeInfo> nextList=mTypeInfoMapper.getChildTypesInfo(tObj.getTypeId()); | ||
72 | - for(TypeInfo t1:nextList){ | ||
73 | - if(t1.getTypeIsLeaf()==1){ | ||
74 | - if("twemproxy".equals(t1.getTypeName())){ | ||
75 | - querParamList.add(t1.getTypeId()); | ||
76 | - redisTweproxyMap.put(t1.getTypeId(),tObj.getTypeName()); | ||
77 | - redisInfoList.add(new RedisMonitor(tObj.getTypeName(),tObj.getTypeName(),0,1,0,t1.getTypeId(),null)); | ||
78 | - } | ||
79 | - } | ||
80 | - } | ||
81 | - | ||
82 | - } | ||
83 | - | ||
84 | - if(CollectionUtils.isEmpty(querParamList)){ | ||
85 | - return; | ||
86 | - } | ||
87 | - | ||
88 | - List<MObjectInfo> redisProxymList=mObjectInfoMapper.selectMObjectsInfoByTypes(querParamList); | ||
89 | - | ||
90 | - /********************************************************************** | ||
91 | - *1、处理twemproxy | ||
92 | - ***********************************************************************/ | ||
93 | - RedisMonitor redisMonitor=null; | ||
94 | - if(CollectionUtils.isEmpty(redisProxymList)){ | ||
95 | - return; | ||
96 | - } | ||
97 | - StringBuffer paramMonitor=null; | ||
98 | - //遍历twemproxy | ||
99 | - Map<String,List<String>> tMap=new HashMap<String,List<String>>(); | ||
100 | - List<String> ipList=null; | ||
101 | - | ||
102 | - for(MObjectInfo obj:redisProxymList){ | ||
103 | - paramMonitor=new StringBuffer(); | ||
104 | - String[] ports=obj.getMoTags().split(","); | ||
105 | - log.info("two port is {}",obj.getMoTags()); | ||
106 | - String result= TelnetUtils.getResult(obj.getMoHostIp(),Integer.valueOf(ports[0])); | ||
107 | - redisMonitor = new RedisMonitor(); | ||
108 | - if(StringUtils.isNotBlank(result)){ | ||
109 | - boolean proxyFlag=RedisCommonUtil.getRedisIsSlave(obj.getMoHostIp(),Integer.valueOf(ports[1])); | ||
110 | - if(proxyFlag){ | ||
111 | - paramMonitor.append("探测成功;"); | ||
112 | - }else{ | ||
113 | - paramMonitor.append("探测失败;"); | ||
114 | - } | ||
115 | - //取舍成功重新设为1 | ||
116 | - JSONObject response=JSONObject.parseObject(result); | ||
117 | - | ||
118 | - //记录代理所有request的和 | ||
119 | - long requestCount = 0; | ||
120 | - long upTime = -1; | ||
121 | - int redisCount = 0; | ||
122 | - | ||
123 | - //用ipString作为lastRequestCountMap lastUpTimeMap中的key | ||
124 | - String ipString =""; | ||
125 | - | ||
126 | - if(null != response){ | ||
127 | - int total_connections=(Integer)response.get("total_connections"); | ||
128 | - int curr_connections=(Integer)response.get("curr_connections"); | ||
129 | - upTime = Long.valueOf(response.get("uptime").toString()); | ||
130 | - if(total_connections>0){ | ||
131 | - //总连接数 | ||
132 | - paramMonitor.append("总连接数:"+total_connections+";"); | ||
133 | - } | ||
134 | - if (total_connections > 0) { | ||
135 | - paramMonitor.append("当前连接数:"+curr_connections+";"); | ||
136 | - } | ||
137 | - | ||
138 | - //记录temproxy开启时间 | ||
139 | - if (upTime > 0) { | ||
140 | - paramMonitor.append("upTime:" + upTime + ";"); | ||
141 | - log.info("upTime is : " + upTime); | ||
142 | - } | ||
143 | - | ||
144 | - //查看代理下的redis 求出所有代理request总量 | ||
145 | - JSONObject alpha=response.getJSONObject("alpha"); | ||
146 | - ipList=new ArrayList<String>(); | ||
147 | - | ||
148 | - for (Map.Entry<String, Object> entry : alpha.entrySet()) { | ||
149 | - String key=entry.getKey(); | ||
150 | - if(key.indexOf(":")>1){ | ||
151 | - ipList.add(key+":"+obj.getMoTypeId()); | ||
152 | - JSONObject ipObj= alpha.getJSONObject(key); | ||
153 | - | ||
154 | - // 取出requests的long值 | ||
155 | - requestCount += Long.valueOf(ipObj.get("requests").toString()); | ||
156 | - redisCount ++; | ||
157 | - log.info("ip:" + key + "request:" + requestCount); | ||
158 | - } | ||
159 | - } | ||
160 | - | ||
161 | - ipString = obj.getMoHostIp()+":"+ports[1]; | ||
162 | - | ||
163 | - /** | ||
164 | - * 计算caps值 (当前requestCount - lastRequestCount)/ (uptime - lastUpTime)/ 1000 | ||
165 | - * 在param_monitor字段数据中中添加 caps值 uptime值 | ||
166 | - * 将caps uptime数据存到全局map中,供下次计算使用 | ||
167 | - */ | ||
168 | - double caps = -1; | ||
169 | - | ||
170 | - //如果数据为0,即没有探测出数据,不记录该干扰数据 | ||
171 | - if (0 < requestCount) { | ||
172 | - //twemproxy文档说明 request值为每30秒的汇总数据 | ||
173 | - caps = requestCount / 30 / redisCount; | ||
174 | - } | ||
175 | - | ||
176 | - paramMonitor.append("caps:" + caps + ";"); | ||
177 | - | ||
178 | - tMap.put(obj.getMoHostIp()+":"+ports[1],ipList); | ||
179 | - redisMonitor.setIsFailed(1); | ||
180 | - redisMonitor.setParamMonitor(paramMonitor.toString()); | ||
181 | - log.info("redisMonitor.setParaMonitor: " + paramMonitor.toString()); | ||
182 | - } | ||
183 | - }else{ | ||
184 | - twemproxyAlarmList.add("失败:"+obj.getMoHostIp()+":"+ports[0]); | ||
185 | - redisMonitor.setIsFailed(0); | ||
186 | - redisMonitor.setParamMonitor(obj.getMoHostIp()+":"+obj.getMoTags()); | ||
187 | - } | ||
188 | - redisMonitor.setNodeFrom(redisTweproxyMap.get(obj.getMoTypeId())); | ||
189 | - redisMonitor.setNodeTo(obj.getMoHostIp()+":"+ports[1]); | ||
190 | - redisMonitor.setLevel(1); | ||
191 | - redisMonitor.setRedisType(obj.getMoTypeId()); | ||
192 | - redisInfoList.add(redisMonitor); | ||
193 | - } | ||
194 | - | ||
195 | - /********************************************************************** | ||
196 | - *2、处理Redis | ||
197 | - ***********************************************************************/ | ||
198 | - if(tMap.size()==0){ | ||
199 | - return; | ||
200 | - } | ||
201 | - | ||
202 | - for (Map.Entry<String, List<String>> entry : tMap.entrySet()) { | ||
203 | - String key = entry.getKey().toString(); | ||
204 | - List<String> rlist = entry.getValue(); | ||
205 | - log.info("loook twemproxy value {},redis list",key,rlist); | ||
206 | - if(!CollectionUtils.isEmpty(rlist)){ | ||
207 | - for(String ipStr:rlist){ | ||
208 | - String[] ipConfig=ipStr.split(":"); | ||
209 | - Map<String,Object> result=null; | ||
210 | - if(ipConfig.length==3){ | ||
211 | - log.info("redis info ip,port",ipConfig[0],ipConfig[1]); | ||
212 | - int isFailed=0; | ||
213 | - result= RedisCommonUtil.getRedisInfo(ipConfig[0], Integer.valueOf(ipConfig[1])); | ||
214 | - paramMonitor=new StringBuffer(); | ||
215 | - if(null==result){ | ||
216 | - redisAlarmList.add(ipConfig[0]+":"+ipConfig[1]+","); | ||
217 | - paramMonitor.append("0,"); | ||
218 | - }else{ | ||
219 | - isFailed=1; | ||
220 | - paramMonitor.append("1,"); | ||
221 | - String role=(String)result.get("role"); | ||
222 | - paramMonitor.append(role+","); | ||
223 | - try { | ||
224 | - long maxmemory=RedisCommonUtil.getRedisMaxMemory(ipConfig[0], Integer.valueOf(ipConfig[1])); | ||
225 | - if(maxmemory==0){ | ||
226 | - paramMonitor.append("最大内存为0"); | ||
227 | - }else{ | ||
228 | - BigDecimal byteDang=BigDecimal.valueOf(Long.valueOf(1024*1024)); | ||
229 | - BigDecimal maxMemoryMb = BigDecimal.valueOf(maxmemory).divide(byteDang,2,4); | ||
230 | - paramMonitor.append(maxMemoryMb+"M,"); | ||
231 | - | ||
232 | - BigDecimal used_memory=BigDecimal.valueOf(Long.valueOf(result.get("used_memory").toString())); | ||
233 | - BigDecimal useProportion = used_memory.divide(BigDecimal.valueOf(maxmemory),2,4); | ||
234 | - paramMonitor.append(useProportion.multiply(new BigDecimal(100))+"%,"); | ||
235 | - } | ||
236 | - }catch (Exception e){ | ||
237 | - log.error("计算Redis使用率错误",e); | ||
238 | - paramMonitor.append("0.00%,"); | ||
239 | - } | ||
240 | - paramMonitor.append(result.get("used_memory_human")+","); | ||
241 | - paramMonitor.append(result.get("instantaneous_ops_per_sec")+","); | ||
242 | - | ||
243 | - //计算命中率hitRate | ||
244 | - long keyspaceHits = Long.valueOf(result.get("keyspace_hits").toString()); | ||
245 | - long keyspaceMisses = Long.valueOf(result.get("keyspace_misses").toString()); | ||
246 | - double hitRate = keyspaceHits * 1.0 / (keyspaceMisses + keyspaceHits); | ||
247 | - paramMonitor.append(hitRate + ","); | ||
248 | - | ||
249 | - if("slave".equals(role)){ | ||
250 | - //测试主从是否是好的 | ||
251 | - log.info("test master and slave info {}",result); | ||
252 | - int flag = RedisCommonUtil.getRedisIsSlave(result.get("master_host").toString(), | ||
253 | - Integer.valueOf(result.get("master_port").toString()), | ||
254 | - ipConfig[0], Integer.valueOf(ipConfig[1])); | ||
255 | - log.info("slave test result",flag); | ||
256 | - String msg=""; | ||
257 | - | ||
258 | - switch (flag) { | ||
259 | - case RedisCommonUtil.NORMAL: | ||
260 | - msg = "OK"; | ||
261 | - break; | ||
262 | - case RedisCommonUtil.READ_EXCEPTION: | ||
263 | - slaveAlarmList.add("主"+result.get("master_host").toString()+":" + | ||
264 | - result.get("master_port").toString() + | ||
265 | - ",从" + ipConfig[0]+":"+ipConfig[1]+","); | ||
266 | - msg="ERROR"; | ||
267 | - break; | ||
268 | - case RedisCommonUtil.WRITE_EXCEPTION: | ||
269 | - // 主写数据失败 | ||
270 | - msg="ERROR"; | ||
271 | - slaveAlarmList.add("主"+result.get("master_host").toString()+":" + | ||
272 | - result.get("master_port").toString() + | ||
273 | - ",从" + ipConfig[0]+":"+ipConfig[1]+",主Redis异常,塞值"); | ||
274 | - break; | ||
275 | - } | ||
276 | - | ||
277 | -// if(RedisCommonUtil.NORMAL == flag){ | ||
278 | -// msg="OK"; | ||
279 | -// }else if (RedisCommonUtil.WRITE_EXCEPTION == flag){ | ||
280 | -// slaveAlarmList.add("主"+result.get("master_host").toString()+":"+result.get("master_port").toString()+",从"+ ipConfig[0]+":"+ipConfig[1]+","); | ||
281 | -// msg="ERROR"; | ||
282 | -// } | ||
283 | - redisInfoList.add(new RedisMonitor(ipConfig[0]+":" + ipConfig[1], | ||
284 | - result.get("master_host").toString()+":"+result.get("master_port").toString(), | ||
285 | - 2,isFailed,1,Integer.valueOf(ipConfig[2]),msg)); | ||
286 | - } | ||
287 | - } | ||
288 | - redisInfoList.add(new RedisMonitor(key,ipConfig[0]+":"+ipConfig[1],2,isFailed,0,Integer.valueOf(ipConfig[2]),paramMonitor.toString())); | ||
289 | - } | ||
290 | - } | ||
291 | - } | ||
292 | - } | ||
293 | - | ||
294 | - /********************************************************************** | ||
295 | - *3、存储监控RedisInfo信息数据 | ||
296 | - ***********************************************************************/ | ||
297 | - if(!CollectionUtils.isEmpty(redisInfoList)){ | ||
298 | - log.info("bachInsertRedisMonitor redis monitor:"+redisInfoList); | ||
299 | - redisMonitorMapper.deleteAllRedisMonitor(); | ||
300 | - redisMonitorMapper.bachInsertRedisMonitor(redisInfoList); | ||
301 | - } | ||
302 | - | ||
303 | - /********************************************************************** | ||
304 | - *4、告警 | ||
305 | - ***********************************************************************/ | ||
306 | - StringBuffer alrarmMsg=new StringBuffer(""); | ||
307 | - if(!CollectionUtils.isEmpty(twemproxyAlarmList)){ | ||
308 | - alrarmMsg.append("twemproxy:"); | ||
309 | - for(String str:twemproxyAlarmList){ | ||
310 | - alrarmMsg.append(str); | ||
311 | - } | ||
312 | - alrarmMsg.append("失败;"); | ||
313 | - } | ||
314 | - | ||
315 | - if(!CollectionUtils.isEmpty(redisAlarmList)){ | ||
316 | - alrarmMsg.append("redis:"); | ||
317 | - for(String str:redisAlarmList){ | ||
318 | - alrarmMsg.append(str); | ||
319 | - } | ||
320 | - alrarmMsg.append("失败;"); | ||
321 | - } | ||
322 | - | ||
323 | - if(!CollectionUtils.isEmpty(slaveAlarmList)){ | ||
324 | - alrarmMsg.append("主从:"); | ||
325 | - for(String str:slaveAlarmList){ | ||
326 | - alrarmMsg.append(str); | ||
327 | - } | ||
328 | - alrarmMsg.append("失败;"); | ||
329 | - } | ||
330 | - | ||
331 | - log.info("sms redis start info {}",alrarmMsg.toString()); | ||
332 | - if(StringUtils.isNotBlank(alrarmMsg.toString())) { | ||
333 | - alrarmMsg.append("请您及时查看!"); | ||
334 | - alarmMsgService.sendSms("redis", alrarmMsg.toString(), snsMobileConfig.getBaseMobile()); | ||
335 | - } | ||
336 | - } | ||
337 | -} |
-
Please register or login to post a comment