|
|
#!/usr/bin/python
|
|
|
# -*- coding: UTF-8 -*-
|
|
|
# author tiexin.yang@yoho.cn
|
|
|
|
|
|
from qcloud_api import QcloudApi
|
|
|
import re
|
|
|
|
|
|
|
|
|
secretId = "************"
|
|
|
secretKey = "*****************"
|
|
|
|
|
|
def EmrDescribeCluster(PageNo=1):
|
|
|
client = QcloudApi(secretId,secretKey)
|
|
|
params = {
|
|
|
"Action":"EmrDescribeCluster",
|
|
|
"PageNo": PageNo,
|
|
|
"PageSize": 20
|
|
|
}
|
|
|
result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
|
|
|
return result
|
|
|
|
|
|
|
|
|
def EmrDescribeClusterNode(ClusterId, NodeFlag, PageNo=1):
|
|
|
client = QcloudApi(secretId,secretKey)
|
|
|
params = {
|
|
|
"Action":"EmrDescribeClusterNode",
|
|
|
"ClusterId":ClusterId,
|
|
|
"NodeFlag":NodeFlag, #节点名称, 取值为:master,core,task,common,all
|
|
|
"PageNo": PageNo,
|
|
|
"PageSize": 20
|
|
|
}
|
|
|
result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
|
|
|
return result
|
|
|
|
|
|
def getCurrentNodes():
|
|
|
clusterList = getCurrentEmrs()
|
|
|
emrNodeIps = []
|
|
|
nodesCnt = 0
|
|
|
PageNo = 1
|
|
|
for clusterId in clusterList:
|
|
|
while True:
|
|
|
print 'Getting nodes from',clusterId,'page',PageNo
|
|
|
result = EmrDescribeClusterNode(clusterId,"all",PageNo)
|
|
|
nodeList = result['data']['nodeList']
|
|
|
for node in nodeList:
|
|
|
emrNodeIps.append(node['ip'])
|
|
|
nodesCnt += len(nodeList)
|
|
|
totalCnt = result['data']['totalCnt']
|
|
|
if nodesCnt < totalCnt:
|
|
|
PageNo+=1
|
|
|
continue
|
|
|
else:
|
|
|
break
|
|
|
PageNo = 1
|
|
|
return emrNodeIps
|
|
|
|
|
|
|
|
|
def getCurrentEmrs():
|
|
|
emrClusterList = []
|
|
|
emrCnt = 0
|
|
|
PageNo = 1
|
|
|
while True:
|
|
|
result = EmrDescribeCluster(PageNo)
|
|
|
clusterList = result['data']['clusterList']
|
|
|
for cluster in clusterList:
|
|
|
emrClusterList.append(cluster['clusterId'])
|
|
|
emrCnt += len(clusterList)
|
|
|
totalCnt = result['data']['totalCnt']
|
|
|
if emrCnt < totalCnt:
|
|
|
PageNo+=1
|
|
|
continue
|
|
|
else:
|
|
|
break
|
|
|
print emrClusterList
|
|
|
return emrClusterList
|
|
|
|
|
|
|
|
|
def getHostsInFile(data):
|
|
|
rex = re.compile(r'\[emr-recom\]([\d+\.\d+\.\d+\.\d+\s*]*)',re.S)
|
|
|
hostsEntry = rex.findall(data)[0].strip('\n')
|
|
|
hostsInFile = hostsEntry.split('\n')
|
|
|
return hostsEntry,hostsInFile
|
|
|
|
|
|
def updateEMRInventory():
|
|
|
hostsData = open('/home/ansible/yoho-ansible-roles/inventories/bigdata/hosts').read()
|
|
|
hostsEntry,hostsInFile = getHostsInFile(hostsData)
|
|
|
currentNodes = getCurrentNodes()
|
|
|
if set(currentNodes) == set(hostsInFile):
|
|
|
print 'Inventory already up to date'
|
|
|
return 0
|
|
|
else:
|
|
|
print 'Replacing inventory file content...'
|
|
|
newInventoryContent = hostsData.replace(hostsEntry,'\n'.join(list(set(currentNodes))))
|
|
|
with open('new.hosts','w') as f:
|
|
|
f.write(newInventoryContent)
|
|
|
f.close()
|
|
|
print 'Inventory file updated success'
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
updateEMRInventory()
|
|
|
#getCurrentEmrs() |
...
|
...
|
|