cos_archive_retrieve.py
3.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#!/usr/bin/python
# -*- coding: UTF-8 -*-
# author tiexin.yang@yoho.cn
from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import argparse
import sys
import time
import json
import re
"""
作用：恢复指定cos路径下的所有归档类型对象并设置临时副本的过期时间
依赖：无
Demo:
python cos_archive_retrieve.py --bucket=yohobd --appid=1251943603 --prefix=hive/warehouse/fact.db/fact_mobile_click/2016/ --days=60
"""
class consoleRobot(object):
def __init__(self,secretId='',secretKey='',region='ap-beijing-1',bucket='yohobd',appid='1251943603'):
self.Region = region
if not secretId or not secretKey:
self.secretId,self.secretKey = open('/home/ansible/.qcloud_config').read().strip('\n').split('\n')
else:
self.secretId,self.secretKey = secretId,secretKey
config = CosConfig(Secret_id=self.secretId, Secret_key=self.secretKey, Region=self.Region, Appid=appid)
self.bucket = bucket
self.client = CosS3Client(config)
self.target_objects = []
def list_objects(self,prefix='',marker=''):
print 'prefix',prefix
response = self.client.list_objects(
Bucket=self.bucket,
MaxKeys=1000,
Prefix=prefix,
Marker=marker
)
for item in response['Contents']:
marker = item['Key']
if item['StorageClass'] == 'ARCHIVE':
self.target_objects.append(item['Key'])
else:
continue
if response['IsTruncated'].upper() == 'TRUE':
print 'Objects more than 1000 has been truncated, Going next page...'
self.list_objects(prefix=prefix,marker=marker)
return True
def restore_object(self,key,days=60):
try:
print 'Restoring object',key
response = self.client.restore_object(
Bucket=self.bucket,
Key=key,
RestoreRequest={
'Days': days
}
)
time.sleep(0.3)
except Exception as e:
if 'Error' in e.message:
rex_errorMsg = re.compile(r'<Message>(.*?)</Message>')
errorMsg = rex_errorMsg.findall(e.message)[0]
print 'Error Message:',errorMsg
time.sleep(1)
else:
print e
pass
return True
def restore_objects_by_prefix(self,prefix='',days=60):
self.list_objects(prefix=prefix)
for key in list(set(self.target_objects)):
self.restore_object(key,days=days)
print 'All objects restored!'
return True
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--bucket',type=str,help='Bucke Name')
parser.add_argument('--appid',type=str,help='Appid, The bucket name string is named as the format: bucketname-appid, like yohobd-1251943603, yohobd as bucketname and 1251943603 is appid here')
parser.add_argument('--prefix',type=str,help='Prefix of object keys')
parser.add_argument('--days',type=int,help='Temp file expire days')
args = parser.parse_args()
demo = consoleRobot(bucket=args.bucket,appid=args.appid)
demo.restore_objects_by_prefix(prefix=args.prefix,days=args.days)