mysql slave down auto processing

+# -*- coding: utf8 -*-
+import requests
+from requests.auth import HTTPBasicAuth
+import json
+import time
+ 
+
+"""
+Author: chunhua.zhang@yoho.cn
+
+ 1. Called by Zabbix when mysql server down.
+ 2. Only process when downed mysql is slave/read
+ 3. Will change read dns to correspond master/write dns and reload cobar
+"""
+# dingding 
+DING =  "https://oapi.dingtalk.com/robot/send?access_token=31cf1a526c98862b7945e561c5a0d857a069350dbee0e807074b83b39fa1e1af"
+ansible_url = "http://ansible.yohops.com/api/v2"
+ansible_user = "chunhua.zhang"
+ansible_password = "Zch@2019"
+
+
+def main_handler(event, context):
+    return_str = "OK"
+    if event['queryStringParameters']['method'] != 'master_fail_over' :
+        return_str = "Method Not Supported"
+    elif event['queryStringParameters']['auth'] != 'yohomysqlfailover123456887':
+        return_str = "Auth Failed"
+    else:
+        fail_master = event['queryStringParameters']['failmaster']
+        new_master = event['queryStringParameters']['newmaster']
+        return_str = " Mysql Failover Proccess Successfully!"
+        ret = ansible_tower_process(new_master, fail_master)
+        alert_to_dingding(new_master, fail_master, ret)
+    return { "isBase64Encoded": False, "statusCode": 200,  "headers": {} , "body": "%s" %return_str }
+
+def ansible_tower_process(down_host):
+   if not down_host:
+       return
+
+   ## az1, az2, az3
+   for inventory in [ 6, 5, 4]:
+      extra_var = {}
+      r = requests.get(ansible_url + "/inventories/%i/" %inventory, auth=(ansible_user, ansible_password))
+      inventory_vars = json.loads( r.json()['variables'])
+      for key in inventory_vars:
+        # find db read key which value == fail-mysql-slave-node 
+        if key.startswith("db") and  key.endswith("_read") and inventory_vars[key] == down_host:
+            extra_var[key] = inventory_vars[ key.replace("_read", "_write") ]
+        
+      if not extra_var:
+         print("can not find any variables in inventory:%i with read mysql is %s" %(inventory, down_host))
+         continue
+    
+      # 1. update dns
+      extra_var['confirmation'] = 'DNS'
+      dns_payload = {}
+      dns_payload['inventory'] = inventory
+      dns_payload['extra_vars'] = json.dumps(extra_var)
+      print("start to update dns for inventory: %i , params is: %s" %(inventory, dns_payload))
+      r_dns = requests.post(ansible_url + "/job_templates/36/launch/", json = dns_payload, auth=(ansible_user, ansible_password))
+     
+      # 1.1. fetch job status, make sure dns reload success
+      job_id = r_dns.json()['job']
+      make_sure_job_success(job_id) 
+    
+      # 2. reload cobar
+      cobar_payload = {}
+      cobar_payload['inventory'] = inventory
+      print("start to reload cobar for inventory: %i , params is: %s" %(inventory, cobar_payload))
+      r_cobar = requests.post(ansible_url + "/job_templates/35/launch/", json = cobar_payload, auth=(ansible_user, ansible_password))
+     
+      # 2.1. fetch job status, make sure job success
+      job_id = r_cobar.json()['job']
+      make_sure_job_success(job_id) 
+   return 0 
+    
+
+ # 2. fetch job status, make sure job success
+def make_sure_job_success(job_id):
+      max_retry = 100
+      job_success = False
+      retry = 0
+      while not job_success:
+       if retry == max_retry:
+         print("can not finish job : %i at %i times" %(job_id, max_retry))
+         return -1
+       r_check_job_status =  requests.get(ansible_url + "/jobs/%i/" %job_id, auth=(ansible_user, ansible_password))
+       if r_check_job_status.json()['status'] == 'successful':
+         print("job: %i is success" %job_id)
+         job_success = True
+       else:
+         print("job:%i  status is : %s" %(job_id, r_check_job_status.json()['status'] ))
+         time.sleep(3)
+         retry = retry + 1
+      return 0
+
+def alert_to_dingding(down_host, ret):
+    # send text dingding message. see:https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.karFPe&treeId=257&articleId=105735&docType=1
+    ding_req = {}
+    ding_req["msgtype"]="markdown"
+    ding_req["markdown"] = {}
+    ding_req["markdown"]['title']="Mysql Failover"
+    ding_req["markdown"]["text"]= "### Mysql Down. Down Host: %s , ret: %i" % (down_host, ret)
+    
+    r = requests.post(url = DING, json = ding_req) 
+    print("Send DingDing message result:%s, request: %s "  %(r, ding_req)) 
+    return("SUCCESS")
+  
\ No newline at end of file