Authored by root

Fix error handler bug

... ... @@ -11,10 +11,16 @@ import os,shutil
import time
import re
"""
作用:由于EMR集群不定时会发生伸缩变化,该脚本可以定时维护inventories/bigdata/hosts文件,如果发生变更会同步更新该文件并推送到git.yoho.cn,并执行resolv的playbook同步DNS
依赖:zabbix.py(注销zabbix host)
Demo:
python server_decomission.py 1.2.3.4
"""
class EMRClusterScanner(object):
def __init__(self,interval,secretId='',secretKey=''):
self.interval = int(interval) if int(interval)> 10 else 10 #扫描时间间隔须大于10秒,避免浪费cpu资源以及api访问过于频繁 单位:秒
self.interval = int(interval) #扫描时间间隔 若为0则只扫描一次 单位:秒
if not secretId or not secretKey:
self.secretId,self.secretKey = open('/home/ansible/.qcloud_config').read().strip('\n').split('\n')
else:
... ... @@ -24,7 +30,7 @@ class EMRClusterScanner(object):
self.inventoryFile = '{0}inventories/bigdata/hosts'.format(self.workDir)
self.playbook_resolv = '{0}playbooks/resolv.yml'.format(self.workDir)
self.mailman = mailman()
self.receivers = ['tiexin.yang@yoho.cn']#,'chunhua.zhang@yoho.cn']
self.receivers = ['tiexin.yang@yoho.cn','chunhua.zhang@yoho.cn']
self.emrNameMap = {
"emr-rt": "emr-r6bhtb5v",
"emr-ops": "emr-iaeloyc2",
... ... @@ -122,7 +128,8 @@ class EMRClusterScanner(object):
def syncDNS(self,target='emr-recom'):
if target == 'emr-recom': #目前只负责同步emr-recom的DNS
cmd = 'ansible-playbook -i {0} {1}'.format(self.inventoryFile,self.playbook_resolv)
if os.system(cmd) != 0:
output = os.popen(cmd).read()
if 'changed=1' not in output or 'unreachable=1' in output:
return False
else:
return True
... ... @@ -139,7 +146,7 @@ class EMRClusterScanner(object):
print 'Inventory already up to date'
return 'Passed'
else:
print '\n'.join(currentNodes)
#print '\n'.join(currentNodes)
N_toAdd = len(set(currentNodes) - set(hostsInFile))
N_toDel = len(set(hostsInFile) - set(currentNodes))
print '{0} to add,{1} to del'.format(N_toAdd,N_toDel)
... ... @@ -178,10 +185,13 @@ class EMRClusterScanner(object):
result = self.updateEMRInventory(emrName)
if result:
if result.upper() == 'PASSED':
print 'Nothing changed, passed'
continue
else:
print 'Sending mail'
self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory [{0}] 已更新\n{1}\n详见 http://git.yoho.cn/ops/yoho-ansible-roles/blob/master/inventories/bigdata/hosts'.format(emrName,result),Title='EMR Inventory 更新结果')
else:
print 'Sending mail'
self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory更新失败\n请检查本地git缓存和ansible连接\n5分钟后重试',Title='EMR Inventory 更新结果')
time.sleep(300)
continue
... ... @@ -190,7 +200,10 @@ class EMRClusterScanner(object):
self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory更新失败\n{0}\n5分钟后重试'.format(str(e)),Title='EMR Inventory 更新结果')
time.sleep(300)
pass
time.sleep(self.interval)
if self.interval > 0:
time.sleep(self.interval)
else:
break
if __name__ == '__main__':
... ...