mysql_master_failover.py
4.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# -*- coding: utf8 -*-
import requests
from requests.auth import HTTPBasicAuth
import json
import time
import os
# dingding
DING = "https://oapi.dingtalk.com/robot/send?access_token=31cf1a526c98862b7945e561c5a0d857a069350dbee0e807074b83b39fa1e1af"
ansible_url = "http://ansible.yohops.com/api/v2"
ansible_user = "chunhua.zhang"
ansible_password = os.environ.get('ansible_password')
JENKINS_UPDATE_VAR_URL = "http://jenkins.yohops.com/job/yoho-ansible-roles/buildWithParameters"
JENKINS_TOKEN = "607f13cd5f7ff5bdf5813f9745d7da49"
def main_handler(event, context):
return_str = "OK"
if event['queryStringParameters']['method'] != 'master_fail_over' :
return_str = "Method Not Supported"
elif event['queryStringParameters']['auth'] != 'yohomysqlfailover123456887':
return_str = "Auth Failed"
else:
fail_master = event['queryStringParameters']['failmaster']
new_master = event['queryStringParameters']['newmaster']
return_str = " Mysql Failover Proccess Successfully!"
ret = ansible_tower_process(new_master, fail_master)
alert_to_dingding(new_master, fail_master, ret)
return { "isBase64Encoded": False, "statusCode": 200, "headers": {} , "body": "%s" %return_str }
def ansible_tower_process(newmaster, failmaster):
## az1, az2, az3
INVENTORY_ENV = {6: "az1", 5: "az2", 4: "az3"}
keys = []
values = []
for inventory in [ 6, 5, 4]:
extra_var = {}
r = requests.get(ansible_url + "/inventories/%i/" %inventory, auth=(ansible_user, ansible_password))
inventory_vars = json.loads( r.json()['variables'])
for key in inventory_vars:
# only proccess variable's name starts with db and variable's value equals [failmaster]
if key.startswith("db") and inventory_vars[key] == failmaster:
extra_var[key] = newmaster
keys.append(key)
values.append(newmaster)
if not extra_var:
print("can not find any variables in inventory:%i starts with db and value is %s" %(inventory,failmaster))
continue
# 1. update dns
extra_var['confirmation'] = 'DNS'
dns_payload = {}
dns_payload['inventory'] = inventory
dns_payload['extra_vars'] = json.dumps(extra_var)
print("start to update dns for inventory: %i , params is: %s" %(inventory, dns_payload))
r_dns = requests.post(ansible_url + "/job_templates/36/launch/", json = dns_payload, auth=(ansible_user, ansible_password))
# 1.1. fetch job status, make sure dns reload success
job_id = r_dns.json()['job']
make_sure_job_success(job_id)
# 2. reload cobar
cobar_payload = {}
cobar_payload['inventory'] = inventory
print("start to reload cobar for inventory: %i , params is: %s" %(inventory, cobar_payload))
r_cobar = requests.post(ansible_url + "/job_templates/35/launch/", json = cobar_payload, auth=(ansible_user, ansible_password))
# 2.1. fetch job status, make sure job success
job_id = r_cobar.json()['job']
make_sure_job_success(job_id)
#update git
if keys and values:
jenkins_param = {"token": JENKINS_TOKEN, "MODIFY_ENV": INVENTORY_ENV[inventory], "keys": ",".join(keys), "values": ",".join(values)}
requests.get(JENKINS_UPDATE_VAR_URL, params = jenkins_param)
return 0
# 2. fetch job status, make sure job success
def make_sure_job_success(job_id):
max_retry = 100
job_success = False
retry = 0
while not job_success:
if retry == max_retry:
print("can not finish job : %i at %i times" %(job_id, max_retry))
return -1
r_check_job_status = requests.get(ansible_url + "/jobs/%i/" %job_id, auth=(ansible_user, ansible_password))
if r_check_job_status.json()['status'] == 'successful':
print("job: %i is success" %job_id)
job_success = True
else:
print("job:%i status is : %s" %(job_id, r_check_job_status.json()['status'] ))
time.sleep(3)
retry = retry + 1
return 0
def alert_to_dingding(newmaster, failmaster, ret):
# send text dingding message. see:https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.karFPe&treeId=257&articleId=105735&docType=1
ding_req = {}
ding_req["msgtype"]="markdown"
ding_req["markdown"] = {}
ding_req["markdown"]['title']="Mysql Failover"
ding_req["markdown"]["text"]= "### Mysql Master Failover. Fail Master: %s , New Master: %s, ret: %i" % (failmaster, newmaster, ret)
r = requests.post(url = DING, json = ding_req)
print("Send DingDing message result:%s, request: %s " %(r, ding_req))
return("SUCCESS")