Authored by chunhua.zhang

add

@@ -17,35 +17,46 @@ DING = "https://oapi.dingtalk.com/robot/send?access_token=31cf1a526c98862b7945e @@ -17,35 +17,46 @@ DING = "https://oapi.dingtalk.com/robot/send?access_token=31cf1a526c98862b7945e
17 ansible_url = "http://ansible.yohops.com/api/v2" 17 ansible_url = "http://ansible.yohops.com/api/v2"
18 ansible_user = "chunhua.zhang" 18 ansible_user = "chunhua.zhang"
19 ansible_password = "Zch@2019" 19 ansible_password = "Zch@2019"
20 - 20 +JENKINS_UPDATE_VAR_URL = "http://jenkins.yohops.com/job/yoho-ansible-roles/buildWithParameters"
  21 +JENKINS_TOKEN = "607f13cd5f7ff5bdf5813f9745d7da49"
21 22
22 def main_handler(event, context): 23 def main_handler(event, context):
  24 + if not event['queryString']
  25 + return
  26 + print("start to process mysql alarm: %s" % event['queryString'])
23 return_str = "OK" 27 return_str = "OK"
24 - if event['queryStringParameters']['method'] != 'master_fail_over' :  
25 - return_str = "Method Not Supported"  
26 - elif event['queryStringParameters']['auth'] != 'yohomysqlfailover123456887': 28 + elif event['queryString']['auth'] != 'yohomysqlfailover123456887':
27 return_str = "Auth Failed" 29 return_str = "Auth Failed"
28 else: 30 else:
29 - fail_master = event['queryStringParameters']['failmaster']  
30 - new_master = event['queryStringParameters']['newmaster']  
31 - return_str = " Mysql Failover Proccess Successfully!"  
32 - ret = ansible_tower_process(new_master, fail_master)  
33 - alert_to_dingding(new_master, fail_master, ret) 31 + fail_host = event['queryString']['host']
  32 + alarm = event['queryString']['alarm'] # maybe: MySQL status on 3306 port is Down (0)'
  33 + return_str = " Mysql host error Proccess Successfully!"
  34 + ret = ansible_tower_process(fail_host, alarm)
  35 + alert_to_dingding(fail_host, alarm, ret)
34 return { "isBase64Encoded": False, "statusCode": 200, "headers": {} , "body": "%s" %return_str } 36 return { "isBase64Encoded": False, "statusCode": 200, "headers": {} , "body": "%s" %return_str }
35 37
36 -def ansible_tower_process(down_host): 38 +def ansible_tower_process(down_host, alarm):
37 if not down_host: 39 if not down_host:
38 return 40 return
39 - 41 + if "mysql" not in alarm.lower() or "is down" not in alarm.lower()
  42 + print("ignore this alarm :%s" % alarm)
  43 + return
40 ## az1, az2, az3 44 ## az1, az2, az3
  45 +
41 for inventory in [ 6, 5, 4]: 46 for inventory in [ 6, 5, 4]:
42 extra_var = {} 47 extra_var = {}
  48 + ## for update git
  49 + keys = []
  50 + values = []
43 r = requests.get(ansible_url + "/inventories/%i/" %inventory, auth=(ansible_user, ansible_password)) 51 r = requests.get(ansible_url + "/inventories/%i/" %inventory, auth=(ansible_user, ansible_password))
44 inventory_vars = json.loads( r.json()['variables']) 52 inventory_vars = json.loads( r.json()['variables'])
45 for key in inventory_vars: 53 for key in inventory_vars:
46 # find db read key which value == fail-mysql-slave-node 54 # find db read key which value == fail-mysql-slave-node
47 if key.startswith("db") and key.endswith("_read") and inventory_vars[key] == down_host: 55 if key.startswith("db") and key.endswith("_read") and inventory_vars[key] == down_host:
48 - extra_var[key] = inventory_vars[ key.replace("_read", "_write") ] 56 + new_read_host = inventory_vars[ key.replace("_read", "_write") ] # change read to write host
  57 + extra_var[key] = new_read_host
  58 + keys.append(key)
  59 + values.append(new_read_host)
49 60
50 if not extra_var: 61 if not extra_var:
51 print("can not find any variables in inventory:%i with read mysql is %s" %(inventory, down_host)) 62 print("can not find any variables in inventory:%i with read mysql is %s" %(inventory, down_host))
@@ -72,6 +83,10 @@ def ansible_tower_process(down_host): @@ -72,6 +83,10 @@ def ansible_tower_process(down_host):
72 # 2.1. fetch job status, make sure job success 83 # 2.1. fetch job status, make sure job success
73 job_id = r_cobar.json()['job'] 84 job_id = r_cobar.json()['job']
74 make_sure_job_success(job_id) 85 make_sure_job_success(job_id)
  86 +
  87 + #update git
  88 + jenkins_param = {"token": JENKINS_TOKEN, "MODIFY_ENV": INVENTORY_ENV[inventory], "key": ",".join(keys), "value": ",".join(values)}
  89 + requests.get(JENKINS_UPDATE_VAR_URL, params = jenkins_param)
75 return 0 90 return 0
76 91
77 92
@@ -94,13 +109,13 @@ def make_sure_job_success(job_id): @@ -94,13 +109,13 @@ def make_sure_job_success(job_id):
94 retry = retry + 1 109 retry = retry + 1
95 return 0 110 return 0
96 111
97 -def alert_to_dingding(down_host, ret): 112 +def alert_to_dingding(down_host, alarm, ret):
98 # send text dingding message. see:https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.karFPe&treeId=257&articleId=105735&docType=1 113 # send text dingding message. see:https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.karFPe&treeId=257&articleId=105735&docType=1
99 ding_req = {} 114 ding_req = {}
100 ding_req["msgtype"]="markdown" 115 ding_req["msgtype"]="markdown"
101 ding_req["markdown"] = {} 116 ding_req["markdown"] = {}
102 ding_req["markdown"]['title']="Mysql Failover" 117 ding_req["markdown"]['title']="Mysql Failover"
103 - ding_req["markdown"]["text"]= "### Mysql Down. Down Host: %s , ret: %i" % (down_host, ret) 118 + ding_req["markdown"]["text"]= "### Mysql Down. Down Host: %s , alarm: %s, ret: %i" % (down_host, alarm, ret)
104 119
105 r = requests.post(url = DING, json = ding_req) 120 r = requests.post(url = DING, json = ding_req)
106 print("Send DingDing message result:%s, request: %s " %(r, ding_req)) 121 print("Send DingDing message result:%s, request: %s " %(r, ding_req))