Sync test

@@ -4,6 +4,7 @@
 
 from qcloud.qcloud_api import QcloudApi
 from qcloud.mailer import mailman
+import json
 import datetime
 import argparse
 import os,shutil
@@ -23,7 +24,7 @@ class EMRClusterScanner(object):
         self.inventoryFile = '{0}inventories/bigdata/hosts'.format(self.workDir)
         self.playbook_resolv = '{0}playbooks/resolv.yml'.format(self.workDir)
         self.mailman = mailman()
-        self.receivers = ['tiexin.yang@yoho.cn','chunhua.zhang@yoho.cn']
+        self.receivers = ['tiexin.yang@yoho.cn']#,'chunhua.zhang@yoho.cn']
         self.emrNameMap = {
             "emr-rt": "emr-r6bhtb5v",
             "emr-ops": "emr-iaeloyc2",
@@ -70,6 +71,7 @@ class EMRClusterScanner(object):
             nodeList = result['data']['nodeList']
             for node in nodeList:
                 emrNodeIps.append(node['ip'])
+                print node['ip']
             nodesCnt += len(nodeList)
             totalCnt = result['data']['totalCnt']
             if nodesCnt < totalCnt:
@@ -112,16 +114,20 @@ class EMRClusterScanner(object):
 
     def gitUpdate(self):
         cmd = 'cd {0};git pull;git add {1};git commit -m "Update bigdata inventory file"; git push'.format(self.workDir,self.inventoryFile)
-        os.popen(cmd)
-        return True
+        if os.system(cmd) != 0:
+            return False
+        else:
+            return True
 
     def syncDNS(self,target='emr-recom'):
         if target == 'emr-recom': #目前只负责同步emr-recom的DNS
             cmd = 'ansible-playbook -i {0} {1}'.format(self.inventoryFile,self.playbook_resolv)
-            os.popen(cmd)
-            return True
+            if os.system(cmd) != 0:
+                return False
+            else:
+                return True
         else:
-            return False
+            return True
     
     def updateEMRInventory(self, emrName):
         #比对当前与最新的Inventory ip列表，相同则pass不同则更新
@@ -131,7 +137,7 @@ class EMRClusterScanner(object):
         currentNodes = self.getCurrentNodes(clusterId=self.emrNameMap[emrName])
         if set(currentNodes) == set(hostsInFile):
             print 'Inventory already up to date'
-            return 0
+            return 'Passed'
         else:
             print '\n'.join(currentNodes)
             N_toAdd = len(set(currentNodes) - set(hostsInFile))
@@ -145,12 +151,22 @@ class EMRClusterScanner(object):
                 f.write(newInventoryContent)
                 f.close()
             print 'Pushing changes to git server...'
-            self.gitUpdate()
+            updateResult = self.gitUpdate()
+            if updateResult:
+                pass
+            else:
+                print 'Inventory file updated failed'
+                return False
+            
             print 'Inventory file updated success'
             if N_toAdd > 0:
                 print 'Running playbook to sync DNS...'
-                self.syncDNS(emrName)
-                print 'DNS synced'
+                syncResult = self.syncDNS(emrName)
+                if syncResult:
+                    print 'DNS synced'
+                else:
+                    print 'DNS sync failed'
+                    return False
 
             return 'Inventory 新增{0}条，删除{1}条'.format(N_toAdd,N_toDel)
 
@@ -161,9 +177,14 @@ class EMRClusterScanner(object):
                 try:
                     result = self.updateEMRInventory(emrName)
                     if result:
-                        self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory [{0}] 已更新\n{1}\n详见 http://git.yoho.cn/ops/yoho-ansible-roles/blob/master/inventories/bigdata/hosts'.format(emrName,result),Title='EMR Inventory 更新结果')
+                        if result.upper() == 'PASSED':
+                            continue
+                        else:
+                            self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory [{0}] 已更新\n{1}\n详见 http://git.yoho.cn/ops/yoho-ansible-roles/blob/master/inventories/bigdata/hosts'.format(emrName,result),Title='EMR Inventory 更新结果')
                     else:
-                        pass
+                        self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory更新失败\n请检查本地git缓存和ansible连接\n5分钟后重试',Title='EMR Inventory 更新结果')
+                        time.sleep(300)
+                        continue
                 except Exception as e:
                     print e
                     self.mailman.mail(receivers=self.receivers,Content='EMR大数据Inventory更新失败\n{0}\n5分钟后重试'.format(str(e)),Title='EMR Inventory 更新结果')