Add emr scanner

root
Commit ca09cd561def09a01e98f6c9b095ae103d20923d 1 parent ccf2ef4b
Showing 1 changed file with 47 additions and 37 deletions
scripts/qcloud/emr_query.py
--- a/scripts/qcloud/emr_query.py
View file @ca09cd5
+++ b/scripts/qcloud/emr_query.py
View file @ca09cd5
@@ -21,7 +21,13 @@ class EMRClusterScanner(object):
         self.inventoryFile = '/opt/projects/yoho-ansible-roles/inventories/bigdata/hosts'
         self.mailman = mailman()
-        self.receivers = ['tiexin.yang@yoho.cn']
+        self.receivers = ['tiexin.yang@yoho.cn','chunhua.zhang@yoho.cn']
+        self.emrNameMap = {
+            "emr-rt": "emr-r6bhtb5v",
+            "emr-ops": "emr-iaeloyc2",
+            "emr-dw": "emr-r6bhtb5v",
+            "emr-recom": "emr-ilj72ynu"
+        }
     def EmrDescribeCluster(self,PageNo=1):
@@ -50,27 +56,25 @@ class EMRClusterScanner(object):
         result = client.do_query(params,"emr.api.qcloud.com/v2/index.php")
         return result
-    def getCurrentNodes(self):
+    def getCurrentNodes(self,clusterId):
         #获取当前所有EMR集群下的所有节点（不区分节点名称）
-        clusterList = self.getCurrentEmrs()
+        #clusterList = self.getCurrentEmrs()
         emrNodeIps = []
         nodesCnt = 0
         PageNo = 1
-        for clusterId in clusterList:
-            while True:
-                print 'Getting nodes from',clusterId,'page',PageNo
-                result = self.EmrDescribeClusterNode(clusterId,"all",PageNo)
-                nodeList = result['data']['nodeList']
-                for node in nodeList:
-                    emrNodeIps.append(node['ip'])
-                nodesCnt += len(nodeList)
-                totalCnt = result['data']['totalCnt']
-                if nodesCnt < totalCnt:
-                    PageNo+=1
-                    continue
-                else:
-                    break
-            PageNo = 1
+        while True:
+            print 'Getting nodes from',clusterId,'page',PageNo
+            result = self.EmrDescribeClusterNode(clusterId,"all",PageNo)
+            nodeList = result['data']['nodeList']
+            for node in nodeList:
+                emrNodeIps.append(node['ip'])
+            nodesCnt += len(nodeList)
+            totalCnt = result['data']['totalCnt']
+            if nodesCnt < totalCnt:
+                PageNo+=1
+                continue
+            else:
+                break
         return emrNodeIps
@@ -94,11 +98,14 @@ class EMRClusterScanner(object):
         return emrClusterList
-    def getHostsInFile(self,data):
+    def getHostsInFile(self,data,prefix):
         #读取当前的Inventory信息，截取emr-recom下的所有ip数据用于和最新采集的ip列表进行匹配
-        rex = re.compile(r'\[emr-recom\]([\d+\.\d+\.\d+\.\d+\s*]*)',re.S)
-        hostsEntry = rex.findall(data)[0].strip('\n')
-        hostsInFile = hostsEntry.split('\n')
+        print 'prefix:',prefix
+        rexContent = re.compile(r'(\[{0}\][\d+\.\d+\.\d+\.\d+\s*]*)'.format(prefix),re.S)
+        rexIp = re.compile(r'(\d+\.\d+\.\d+\.\d+)')
+        content = rexContent.findall(data)
+        hostsEntry = content[0].strip('\n')
+        hostsInFile = rexIp.findall(hostsEntry)
         return hostsEntry,hostsInFile
     def gitUpdate(self):
@@ -106,22 +113,24 @@ class EMRClusterScanner(object):
         os.popen(cmd)
         return True
-    def updateEMRInventory(self):
+    def updateEMRInventory(self, emrName):
         #比对当前与最新的Inventory ip列表，相同则pass不同则更新
         today = datetime.date.today().strftime('%Y%m%d')
         hostsData = open(self.inventoryFile).read()
-        hostsEntry,hostsInFile = self.getHostsInFile(hostsData)
-        currentNodes = self.getCurrentNodes()
+        hostsEntry,hostsInFile = self.getHostsInFile(hostsData,prefix=emrName)
+        currentNodes = self.getCurrentNodes(clusterId=self.emrNameMap[emrName])
         if set(currentNodes) == set(hostsInFile):
             print 'Inventory already up to date'
             return 0
         else:
+            print '\n'.join(currentNodes)
             N_toAdd = len(set(currentNodes) - set(hostsInFile))
             N_toDel = len(set(hostsInFile) - set(currentNodes))
+            print '{0} to add,{1} to del'.format(N_toAdd,N_toDel)
             print 'Taking backup for current inventory file...'
             shutil.copyfile(self.inventoryFile,self.inventoryFile+'.'+today)
             print 'Replacing inventory file content...'
-            newInventoryContent = hostsData.replace(hostsEntry,'\n'.join(list(set(currentNodes))))
+            newInventoryContent = hostsData.replace(hostsEntry,'[{0}]\n'.format(emrName)+'\n'.join(list(set(currentNodes))))
             with open(self.inventoryFile,'w') as f:
                 f.write(newInventoryContent)
                 f.close()
@@ -133,17 +142,18 @@ class EMRClusterScanner(object):
     def start_daemon(self):
         #以守护进程的状态运行此EMR扫描程序
         while True:
-            try:
-                result = self.updateEMRInventory()
-                if result:
-                    self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory已更新\n{0}\n详见 http://git.yoho.cn/ops/yoho-ansible-roles/blob/master/inventories/bigdata/hosts'.format(result),Title='EMR Inventory 更新结果')
-                else:
-                    pass
-            except Exception as e:
-                print e
-                self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory更新失败\n{0}\n5分钟后重试'.format(str(e)),Title='EMR Inventory 更新结果')
-                time.sleep(300)
-                pass
+            for emrName in self.emrNameMap:
+                #try:
+                    result = self.updateEMRInventory(emrName)
+                    if result:
+                        self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory [{0}] 已更新\n{1}\n详见 http://git.yoho.cn/ops/yoho-ansible-roles/blob/master/inventories/bigdata/hosts'.format(emrName,result),Title='EMR Inventory 更新结果')
+                    else:
+                        pass
+                #except Exception as e:
+                #    print e
+                #    self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory更新失败\n{0}\n5分钟后重试'.format(str(e)),Title='EMR Inventory 更新结果')
+                #    time.sleep(300)
+                #    pass
             time.sleep(self.interval)