Authored by txyang

Add emr query

  1 +#!/usr/bin/python
  2 +# -*- coding: UTF-8 -*-
  3 +# author tiexin.yang@yoho.cn
  4 +
  5 +from qcloud_api import QcloudApi
  6 +import re
  7 +
  8 +
  9 +secretId = "************"
  10 +secretKey = "*****************"
  11 +
  12 +def EmrDescribeCluster(PageNo=1):
  13 + client = QcloudApi(secretId,secretKey)
  14 + params = {
  15 + "Action":"EmrDescribeCluster",
  16 + "PageNo": PageNo,
  17 + "PageSize": 20
  18 + }
  19 + result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
  20 + return result
  21 +
  22 +
  23 +def EmrDescribeClusterNode(ClusterId, NodeFlag, PageNo=1):
  24 + client = QcloudApi(secretId,secretKey)
  25 + params = {
  26 + "Action":"EmrDescribeClusterNode",
  27 + "ClusterId":ClusterId,
  28 + "NodeFlag":NodeFlag, #节点名称, 取值为:master,core,task,common,all
  29 + "PageNo": PageNo,
  30 + "PageSize": 20
  31 + }
  32 + result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
  33 + return result
  34 +
  35 +def getCurrentNodes():
  36 + clusterList = getCurrentEmrs()
  37 + emrNodeIps = []
  38 + nodesCnt = 0
  39 + PageNo = 1
  40 + for clusterId in clusterList:
  41 + while True:
  42 + print 'Getting nodes from',clusterId,'page',PageNo
  43 + result = EmrDescribeClusterNode(clusterId,"all",PageNo)
  44 + nodeList = result['data']['nodeList']
  45 + for node in nodeList:
  46 + emrNodeIps.append(node['ip'])
  47 + nodesCnt += len(nodeList)
  48 + totalCnt = result['data']['totalCnt']
  49 + if nodesCnt < totalCnt:
  50 + PageNo+=1
  51 + continue
  52 + else:
  53 + break
  54 + PageNo = 1
  55 + return emrNodeIps
  56 +
  57 +
  58 +def getCurrentEmrs():
  59 + emrClusterList = []
  60 + emrCnt = 0
  61 + PageNo = 1
  62 + while True:
  63 + result = EmrDescribeCluster(PageNo)
  64 + clusterList = result['data']['clusterList']
  65 + for cluster in clusterList:
  66 + emrClusterList.append(cluster['clusterId'])
  67 + emrCnt += len(clusterList)
  68 + totalCnt = result['data']['totalCnt']
  69 + if emrCnt < totalCnt:
  70 + PageNo+=1
  71 + continue
  72 + else:
  73 + break
  74 + print emrClusterList
  75 + return emrClusterList
  76 +
  77 +
  78 +def getHostsInFile(data):
  79 + rex = re.compile(r'\[emr-recom\]([\d+\.\d+\.\d+\.\d+\s*]*)',re.S)
  80 + hostsEntry = rex.findall(data)[0].strip('\n')
  81 + hostsInFile = hostsEntry.split('\n')
  82 + return hostsEntry,hostsInFile
  83 +
  84 +def updateEMRInventory():
  85 + hostsData = open('/home/ansible/yoho-ansible-roles/inventories/bigdata/hosts').read()
  86 + hostsEntry,hostsInFile = getHostsInFile(hostsData)
  87 + currentNodes = getCurrentNodes()
  88 + if set(currentNodes) == set(hostsInFile):
  89 + print 'Inventory already up to date'
  90 + return 0
  91 + else:
  92 + print 'Replacing inventory file content...'
  93 + newInventoryContent = hostsData.replace(hostsEntry,'\n'.join(list(set(currentNodes))))
  94 + with open('new.hosts','w') as f:
  95 + f.write(newInventoryContent)
  96 + f.close()
  97 + print 'Inventory file updated success'
  98 + return 0
  99 +
  100 +
  101 +
  102 +if __name__ == '__main__':
  103 + updateEMRInventory()
  104 + #getCurrentEmrs()