Authored by txyang

Add emr query

#!/usr/bin/python
# -*- coding: UTF-8 -*-
# author tiexin.yang@yoho.cn
from qcloud_api import QcloudApi
import re
secretId = "************"
secretKey = "*****************"
def EmrDescribeCluster(PageNo=1):
client = QcloudApi(secretId,secretKey)
params = {
"Action":"EmrDescribeCluster",
"PageNo": PageNo,
"PageSize": 20
}
result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
return result
def EmrDescribeClusterNode(ClusterId, NodeFlag, PageNo=1):
client = QcloudApi(secretId,secretKey)
params = {
"Action":"EmrDescribeClusterNode",
"ClusterId":ClusterId,
"NodeFlag":NodeFlag, #节点名称, 取值为:master,core,task,common,all
"PageNo": PageNo,
"PageSize": 20
}
result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
return result
def getCurrentNodes():
clusterList = getCurrentEmrs()
emrNodeIps = []
nodesCnt = 0
PageNo = 1
for clusterId in clusterList:
while True:
print 'Getting nodes from',clusterId,'page',PageNo
result = EmrDescribeClusterNode(clusterId,"all",PageNo)
nodeList = result['data']['nodeList']
for node in nodeList:
emrNodeIps.append(node['ip'])
nodesCnt += len(nodeList)
totalCnt = result['data']['totalCnt']
if nodesCnt < totalCnt:
PageNo+=1
continue
else:
break
PageNo = 1
return emrNodeIps
def getCurrentEmrs():
emrClusterList = []
emrCnt = 0
PageNo = 1
while True:
result = EmrDescribeCluster(PageNo)
clusterList = result['data']['clusterList']
for cluster in clusterList:
emrClusterList.append(cluster['clusterId'])
emrCnt += len(clusterList)
totalCnt = result['data']['totalCnt']
if emrCnt < totalCnt:
PageNo+=1
continue
else:
break
print emrClusterList
return emrClusterList
def getHostsInFile(data):
rex = re.compile(r'\[emr-recom\]([\d+\.\d+\.\d+\.\d+\s*]*)',re.S)
hostsEntry = rex.findall(data)[0].strip('\n')
hostsInFile = hostsEntry.split('\n')
return hostsEntry,hostsInFile
def updateEMRInventory():
hostsData = open('/home/ansible/yoho-ansible-roles/inventories/bigdata/hosts').read()
hostsEntry,hostsInFile = getHostsInFile(hostsData)
currentNodes = getCurrentNodes()
if set(currentNodes) == set(hostsInFile):
print 'Inventory already up to date'
return 0
else:
print 'Replacing inventory file content...'
newInventoryContent = hostsData.replace(hostsEntry,'\n'.join(list(set(currentNodes))))
with open('new.hosts','w') as f:
f.write(newInventoryContent)
f.close()
print 'Inventory file updated success'
return 0
if __name__ == '__main__':
updateEMRInventory()
#getCurrentEmrs()
... ...