Showing
1 changed file
with
104 additions
and
0 deletions
scripts/qcloud/emr_query.py
0 → 100644
1 | +#!/usr/bin/python | ||
2 | +# -*- coding: UTF-8 -*- | ||
3 | +# author tiexin.yang@yoho.cn | ||
4 | + | ||
5 | +from qcloud_api import QcloudApi | ||
6 | +import re | ||
7 | + | ||
8 | + | ||
9 | +secretId = "************" | ||
10 | +secretKey = "*****************" | ||
11 | + | ||
12 | +def EmrDescribeCluster(PageNo=1): | ||
13 | + client = QcloudApi(secretId,secretKey) | ||
14 | + params = { | ||
15 | + "Action":"EmrDescribeCluster", | ||
16 | + "PageNo": PageNo, | ||
17 | + "PageSize": 20 | ||
18 | + } | ||
19 | + result = client.do_query(params,"emr.api.qcloud.com/v2/index.php") | ||
20 | + return result | ||
21 | + | ||
22 | + | ||
23 | +def EmrDescribeClusterNode(ClusterId, NodeFlag, PageNo=1): | ||
24 | + client = QcloudApi(secretId,secretKey) | ||
25 | + params = { | ||
26 | + "Action":"EmrDescribeClusterNode", | ||
27 | + "ClusterId":ClusterId, | ||
28 | + "NodeFlag":NodeFlag, #节点名称, 取值为:master,core,task,common,all | ||
29 | + "PageNo": PageNo, | ||
30 | + "PageSize": 20 | ||
31 | + } | ||
32 | + result = client.do_query(params,"emr.api.qcloud.com/v2/index.php") | ||
33 | + return result | ||
34 | + | ||
35 | +def getCurrentNodes(): | ||
36 | + clusterList = getCurrentEmrs() | ||
37 | + emrNodeIps = [] | ||
38 | + nodesCnt = 0 | ||
39 | + PageNo = 1 | ||
40 | + for clusterId in clusterList: | ||
41 | + while True: | ||
42 | + print 'Getting nodes from',clusterId,'page',PageNo | ||
43 | + result = EmrDescribeClusterNode(clusterId,"all",PageNo) | ||
44 | + nodeList = result['data']['nodeList'] | ||
45 | + for node in nodeList: | ||
46 | + emrNodeIps.append(node['ip']) | ||
47 | + nodesCnt += len(nodeList) | ||
48 | + totalCnt = result['data']['totalCnt'] | ||
49 | + if nodesCnt < totalCnt: | ||
50 | + PageNo+=1 | ||
51 | + continue | ||
52 | + else: | ||
53 | + break | ||
54 | + PageNo = 1 | ||
55 | + return emrNodeIps | ||
56 | + | ||
57 | + | ||
58 | +def getCurrentEmrs(): | ||
59 | + emrClusterList = [] | ||
60 | + emrCnt = 0 | ||
61 | + PageNo = 1 | ||
62 | + while True: | ||
63 | + result = EmrDescribeCluster(PageNo) | ||
64 | + clusterList = result['data']['clusterList'] | ||
65 | + for cluster in clusterList: | ||
66 | + emrClusterList.append(cluster['clusterId']) | ||
67 | + emrCnt += len(clusterList) | ||
68 | + totalCnt = result['data']['totalCnt'] | ||
69 | + if emrCnt < totalCnt: | ||
70 | + PageNo+=1 | ||
71 | + continue | ||
72 | + else: | ||
73 | + break | ||
74 | + print emrClusterList | ||
75 | + return emrClusterList | ||
76 | + | ||
77 | + | ||
78 | +def getHostsInFile(data): | ||
79 | + rex = re.compile(r'\[emr-recom\]([\d+\.\d+\.\d+\.\d+\s*]*)',re.S) | ||
80 | + hostsEntry = rex.findall(data)[0].strip('\n') | ||
81 | + hostsInFile = hostsEntry.split('\n') | ||
82 | + return hostsEntry,hostsInFile | ||
83 | + | ||
84 | +def updateEMRInventory(): | ||
85 | + hostsData = open('/home/ansible/yoho-ansible-roles/inventories/bigdata/hosts').read() | ||
86 | + hostsEntry,hostsInFile = getHostsInFile(hostsData) | ||
87 | + currentNodes = getCurrentNodes() | ||
88 | + if set(currentNodes) == set(hostsInFile): | ||
89 | + print 'Inventory already up to date' | ||
90 | + return 0 | ||
91 | + else: | ||
92 | + print 'Replacing inventory file content...' | ||
93 | + newInventoryContent = hostsData.replace(hostsEntry,'\n'.join(list(set(currentNodes)))) | ||
94 | + with open('new.hosts','w') as f: | ||
95 | + f.write(newInventoryContent) | ||
96 | + f.close() | ||
97 | + print 'Inventory file updated success' | ||
98 | + return 0 | ||
99 | + | ||
100 | + | ||
101 | + | ||
102 | +if __name__ == '__main__': | ||
103 | + updateEMRInventory() | ||
104 | + #getCurrentEmrs() |
-
Please register or login to post a comment