Showing
1 changed file
with
29 additions
and
14 deletions
@@ -17,35 +17,46 @@ DING = "https://oapi.dingtalk.com/robot/send?access_token=31cf1a526c98862b7945e | @@ -17,35 +17,46 @@ DING = "https://oapi.dingtalk.com/robot/send?access_token=31cf1a526c98862b7945e | ||
17 | ansible_url = "http://ansible.yohops.com/api/v2" | 17 | ansible_url = "http://ansible.yohops.com/api/v2" |
18 | ansible_user = "chunhua.zhang" | 18 | ansible_user = "chunhua.zhang" |
19 | ansible_password = "Zch@2019" | 19 | ansible_password = "Zch@2019" |
20 | - | 20 | +JENKINS_UPDATE_VAR_URL = "http://jenkins.yohops.com/job/yoho-ansible-roles/buildWithParameters" |
21 | +JENKINS_TOKEN = "607f13cd5f7ff5bdf5813f9745d7da49" | ||
21 | 22 | ||
22 | def main_handler(event, context): | 23 | def main_handler(event, context): |
24 | + if not event['queryString'] | ||
25 | + return | ||
26 | + print("start to process mysql alarm: %s" % event['queryString']) | ||
23 | return_str = "OK" | 27 | return_str = "OK" |
24 | - if event['queryStringParameters']['method'] != 'master_fail_over' : | ||
25 | - return_str = "Method Not Supported" | ||
26 | - elif event['queryStringParameters']['auth'] != 'yohomysqlfailover123456887': | 28 | + elif event['queryString']['auth'] != 'yohomysqlfailover123456887': |
27 | return_str = "Auth Failed" | 29 | return_str = "Auth Failed" |
28 | else: | 30 | else: |
29 | - fail_master = event['queryStringParameters']['failmaster'] | ||
30 | - new_master = event['queryStringParameters']['newmaster'] | ||
31 | - return_str = " Mysql Failover Proccess Successfully!" | ||
32 | - ret = ansible_tower_process(new_master, fail_master) | ||
33 | - alert_to_dingding(new_master, fail_master, ret) | 31 | + fail_host = event['queryString']['host'] |
32 | + alarm = event['queryString']['alarm'] # maybe: MySQL status on 3306 port is Down (0)' | ||
33 | + return_str = " Mysql host error Proccess Successfully!" | ||
34 | + ret = ansible_tower_process(fail_host, alarm) | ||
35 | + alert_to_dingding(fail_host, alarm, ret) | ||
34 | return { "isBase64Encoded": False, "statusCode": 200, "headers": {} , "body": "%s" %return_str } | 36 | return { "isBase64Encoded": False, "statusCode": 200, "headers": {} , "body": "%s" %return_str } |
35 | 37 | ||
36 | -def ansible_tower_process(down_host): | 38 | +def ansible_tower_process(down_host, alarm): |
37 | if not down_host: | 39 | if not down_host: |
38 | return | 40 | return |
39 | - | 41 | + if "mysql" not in alarm.lower() or "is down" not in alarm.lower() |
42 | + print("ignore this alarm :%s" % alarm) | ||
43 | + return | ||
40 | ## az1, az2, az3 | 44 | ## az1, az2, az3 |
45 | + | ||
41 | for inventory in [ 6, 5, 4]: | 46 | for inventory in [ 6, 5, 4]: |
42 | extra_var = {} | 47 | extra_var = {} |
48 | + ## for update git | ||
49 | + keys = [] | ||
50 | + values = [] | ||
43 | r = requests.get(ansible_url + "/inventories/%i/" %inventory, auth=(ansible_user, ansible_password)) | 51 | r = requests.get(ansible_url + "/inventories/%i/" %inventory, auth=(ansible_user, ansible_password)) |
44 | inventory_vars = json.loads( r.json()['variables']) | 52 | inventory_vars = json.loads( r.json()['variables']) |
45 | for key in inventory_vars: | 53 | for key in inventory_vars: |
46 | # find db read key which value == fail-mysql-slave-node | 54 | # find db read key which value == fail-mysql-slave-node |
47 | if key.startswith("db") and key.endswith("_read") and inventory_vars[key] == down_host: | 55 | if key.startswith("db") and key.endswith("_read") and inventory_vars[key] == down_host: |
48 | - extra_var[key] = inventory_vars[ key.replace("_read", "_write") ] | 56 | + new_read_host = inventory_vars[ key.replace("_read", "_write") ] # change read to write host |
57 | + extra_var[key] = new_read_host | ||
58 | + keys.append(key) | ||
59 | + values.append(new_read_host) | ||
49 | 60 | ||
50 | if not extra_var: | 61 | if not extra_var: |
51 | print("can not find any variables in inventory:%i with read mysql is %s" %(inventory, down_host)) | 62 | print("can not find any variables in inventory:%i with read mysql is %s" %(inventory, down_host)) |
@@ -72,6 +83,10 @@ def ansible_tower_process(down_host): | @@ -72,6 +83,10 @@ def ansible_tower_process(down_host): | ||
72 | # 2.1. fetch job status, make sure job success | 83 | # 2.1. fetch job status, make sure job success |
73 | job_id = r_cobar.json()['job'] | 84 | job_id = r_cobar.json()['job'] |
74 | make_sure_job_success(job_id) | 85 | make_sure_job_success(job_id) |
86 | + | ||
87 | + #update git | ||
88 | + jenkins_param = {"token": JENKINS_TOKEN, "MODIFY_ENV": INVENTORY_ENV[inventory], "key": ",".join(keys), "value": ",".join(values)} | ||
89 | + requests.get(JENKINS_UPDATE_VAR_URL, params = jenkins_param) | ||
75 | return 0 | 90 | return 0 |
76 | 91 | ||
77 | 92 | ||
@@ -94,13 +109,13 @@ def make_sure_job_success(job_id): | @@ -94,13 +109,13 @@ def make_sure_job_success(job_id): | ||
94 | retry = retry + 1 | 109 | retry = retry + 1 |
95 | return 0 | 110 | return 0 |
96 | 111 | ||
97 | -def alert_to_dingding(down_host, ret): | 112 | +def alert_to_dingding(down_host, alarm, ret): |
98 | # send text dingding message. see:https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.karFPe&treeId=257&articleId=105735&docType=1 | 113 | # send text dingding message. see:https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.karFPe&treeId=257&articleId=105735&docType=1 |
99 | ding_req = {} | 114 | ding_req = {} |
100 | ding_req["msgtype"]="markdown" | 115 | ding_req["msgtype"]="markdown" |
101 | ding_req["markdown"] = {} | 116 | ding_req["markdown"] = {} |
102 | ding_req["markdown"]['title']="Mysql Failover" | 117 | ding_req["markdown"]['title']="Mysql Failover" |
103 | - ding_req["markdown"]["text"]= "### Mysql Down. Down Host: %s , ret: %i" % (down_host, ret) | 118 | + ding_req["markdown"]["text"]= "### Mysql Down. Down Host: %s , alarm: %s, ret: %i" % (down_host, alarm, ret) |
104 | 119 | ||
105 | r = requests.post(url = DING, json = ding_req) | 120 | r = requests.post(url = DING, json = ding_req) |
106 | print("Send DingDing message result:%s, request: %s " %(r, ding_req)) | 121 | print("Send DingDing message result:%s, request: %s " %(r, ding_req)) |
-
Please register or login to post a comment