LinuxEye - Linux系统教程
TAG标签RSS订阅[设为首页] [加入收藏]

LinuxEye - Linux系统教程

热门标签:

当前位置: 主页 > 脚本编程 >

腾讯云COS上传、批量删除工具(Python)

时间:2017-04-01 11:21来源:Linux运维笔记 编辑:yeho 点击: 次
腾讯云对象存储COS是类似于阿里云OSS，相比OSS，COS提供每月免费额度：存储空间50G、外网访问流量10G（内网免费）、免费读请求100万次、写请求10万次。对网站备份来说不错，但是，腾

腾讯云对象存储COS是类似于阿里云OSS，相比OSS，COS提供每月免费额度：存储空间50G、外网访问流量10G（内网免费）、免费读请求100万次、写请求10万次。对网站备份来说不错，但是，腾讯云提供的工具太low，参考阿里云OSS，写了一个cos信息配置、创建目录、上传、批量删除工具（coscmd）

安装Python2.7

此工具在Python2.7上测试通过，建议用《OneinStack》安装Python2.7，安装路径为:/usr/local/python，命令如下：

 
  1. cd ~/oneinstack
  2. ./addons.sh #选择7，install Let's Encrypt client
配置COS

登陆腾讯云管理后台https://console.qcloud.com/cos/bucket创建bucket，并获取API密钥，在下面配置中一一对应。

 
  1. cd oneinstack/tools #必须进入该目录执行
  2. wget http://mirrors.linuxeye.com/oneinstack/tools/cos.tgz
  3. tar xzf cos.tgz
  4. /usr/local/python/bin/python ./coscmd config --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket]

执行后会将相关信息写到~/.coscredentials，执行其它动作会自动加在改配置。

创建目录
上传文件
批量删除
coscmd代码(coscmd)
 
  1. #!/usr/bin/env python
  2. #coding:utf-8
  3.  
  4. import sys,os
  5. import datetime
  6. import random
  7. import threading
  8. import time
  9. import datetime
  10. import logging
  11. import ConfigParser
  12. from optparse import OptionParser
  13. from logging.handlers import RotatingFileHandler
  14. from time import strftime, localtime
  15. from time import sleep
  16. from datetime import date
  17. from datetime import timedelta
  18. from cos import CosClient
  19. from cos import UploadFileRequest
  20. from cos import CreateFolderRequest
  21. from cos import DelFileRequest
  22. from cos import DelFolderRequest
  23. from cos import ListFolderRequest
  24. from cos import threadpool
  25.  
  26. MAX_RETRY_TIMES = 3
  27. LOG_SAVE_EVERY_NUM = 1024
  28. ONE_TASK_DEL_FILE_NUMS = 50
  29. log_level=1
  30. log_file_name="del_file.log"
  31. dir_thread_num=2
  32. file_thread_num=5
  33. delete_folder_fail_exist=0
  34.  
  35. CONFIGFILE = "%s/.coscredentials" % os.path.expanduser('~')
  36. CONFIGSECTION = 'COSCredentials'
  37.  
  38. HELP = \
  39. '''coscmd:
  40.     config       --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket] 
  41.     ls             cosdir
  42.     mkdir       dirname
  43.     put          localfile  cosdir 
  44.     rm(delete,del) object
  45.     '''
  46.  
  47. CMD_LIST = {}
  48. def cmd_configure(args, options):
  49.     if options.appid is None or options.secret_id is None or options.secret_key is None or options.bucket is None:
  50.         print("%s miss parameters, use --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket] to specify appid/id/key/bucket pair" % args[0])
  51.         sys.exit(-1)
  52.     config = ConfigParser.RawConfigParser()
  53.     config.add_section(CONFIGSECTION)
  54.     config.set(CONFIGSECTION, 'appid', options.appid)
  55.     config.set(CONFIGSECTION, 'secret_id', options.secret_id)
  56.     config.set(CONFIGSECTION, 'secret_key', options.secret_key)
  57.     config.set(CONFIGSECTION, 'bucket', options.bucket)
  58.     cfgfile = open(CONFIGFILE, 'w+')
  59.     config.write(cfgfile)
  60.     print("Your configuration is saved into %s ." % CONFIGFILE)
  61.     cfgfile.close()
  62.     import stat
  63.     os.chmod(CONFIGFILE, stat.S_IREAD | stat.S_IWRITE)
  64.  
  65. def cmd_loadconfigure():
  66.     config = ConfigParser.ConfigParser()
  67.     config.read(CONFIGFILE)
  68.     global appid
  69.     global secret_id
  70.     global secret_key
  71.     global bucket
  72.     appid = int(config.get(CONFIGSECTION, 'appid'))
  73.     secret_id = config.get(CONFIGSECTION, 'secret_id').decode('utf-8')
  74.     secret_key = config.get(CONFIGSECTION, 'secret_key').decode('utf-8')
  75.     bucket = config.get(CONFIGSECTION, 'bucket').decode('utf-8')
  76.     if len(secret_id) == 0 or len(secret_key) == 0 or len(bucket) == 0:
  77.         print("can't get appid/secret_id/secret_key/bucket, setup use : config --appid=[appid] --id=[secret_id] --key=[secret_key] --bucket=[bucket]")
  78.         sys.exit(1)
  79.  
  80. def cmd_lsdir(COSDIR):
  81.     cosdir = COSDIR.decode('utf-8')
  82.     request = ListFolderRequest(bucket, cosdir)
  83.     list_folder_ret = cos_client.list_folder(request)
  84.     if list_folder_ret[u'code'] == 0:
  85.         print(True)
  86.     else:
  87.         print("%s, appid/secret_id/secret_key/bucket invalid"% list_folder_ret[u'message'])
  88.  
  89. def cmd_mkdir(COSDIR):
  90.     cosdir = COSDIR.decode('utf-8')
  91.     request = CreateFolderRequest(bucket, cosdir)
  92.     create_folder_ret = cos_client.create_folder(request)
  93.     if create_folder_ret[u'code'] == 0:
  94.         print("mkdir cos://%s%s OK" % (bucket,COSDIR))
  95.     else:
  96.         print(create_folder_ret[u'message'])
  97.  
  98. def cmd_put(LOCALFILE,COSFILE):
  99.     localfile = LOCALFILE.decode('utf-8')
  100.     cosfile = COSFILE.decode('utf-8')
  101.     request = UploadFileRequest(bucket, cosfile, localfile)
  102.     request.set_insert_only(0)
  103.     upload_file_ret = cos_client.upload_file(request)
  104.     if upload_file_ret[u'code'] == 0:
  105.         print("put cos://%s%s OK" % (bucket,COSFILE))
  106.     else:
  107.         print(upload_file_ret[u'message'])
  108.  
  109. def loginit():
  110.     if (log_file_name == ""):
  111.         return
  112.     log_level = logging.ERROR
  113.     if log_level == 0:
  114.         log_level = logging.DEBUG
  115.     if log_level == 1:
  116.         log_level = logging.INFO
  117.     if log_level == 2:
  118.         log_level = logging.WARNING
  119.  
  120.     #定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大20M
  121.     logger = logging.getLogger("")
  122.     Rthandler = RotatingFileHandler(log_file_name, maxBytes= 20*1024*1024,backupCount=5)
  123.     Rthandler.setLevel(log_level)
  124.     formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
  125.     Rthandler.setFormatter(formatter)
  126.     logger.addHandler(Rthandler)
  127.     logger.setLevel(log_level)
  128.     return logger
  129.  
  130. #日期相关操作
  131. class Dateop():
  132.     @staticmethod
  133.     def isValidDate(str):
  134.         try:
  135.             time.strptime(str, "%Y""%m""%d")
  136.             return True
  137.         except:
  138.             return False
  139.  
  140.     @staticmethod
  141.     def getdaystr(n=0):
  142.         dt = date.today()-timedelta(days=n)
  143.         tt = dt.timetuple()
  144.         daystr = strftime("%Y""%m""%d",tt)
  145.         return daystr
  146.  
  147.     @staticmethod
  148.     def cmpDateAgo(t1,t2):
  149.         if (Dateop.isValidDate(t1)==False or Dateop.isValidDate(t2)==False):
  150.             return False
  151.         if (int(t1) <= int (t2)):
  152.             return True
  153.         return False
  154.  
  155.     @staticmethod
  156.     def isNeedDeleteDir(dirname, n=0):
  157.         if (len(dirname) != 8):
  158.             return False
  159.         if Dateop.isValidDate(dirname) == False:
  160.             return False
  161.         d2 = Dateop.getdaystr(n);
  162.         if Dateop.cmpDateAgo(dirname, d2):
  163.             return True
  164.         return False
  165. #删除文件统计
  166. class FileStat():
  167.     global cos_log
  168.     def __init__(self):
  169.         self.delfilesuccnum = 0
  170.         self.deldirsuccnum = 0
  171.         self.delfilefailnum = 0
  172.         self.deldirfailnum = 0
  173.         self.lock = threading.Lock()
  174.  
  175.     def addDelFileFailNum(self,num=1):
  176.         self.lock.acquire(1)
  177.         self.delfilefailnum += num
  178.         self.lock.release()
  179.     def addDelDirFailNum(self,num=1):
  180.         self.lock.acquire(1)
  181.         self.deldirfailnum += num
  182.         self.lock.release()
  183.     def addDelDirSuccNum(self, num=1):
  184.         self.lock.acquire(1)
  185.         self.deldirsuccnum += num
  186.         self.lock.release()
  187.     def addDelFileSuccNum(self, num=1):
  188.         self.lock.acquire(1)
  189.         self.delfilesuccnum += num
  190.         self.lock.release()
  191.     def printStat(self):
  192.         msg ="".join(["delfilesuccnum=",str(self.delfilesuccnum),
  193.                 ",delfilefailnum=",str(self.delfilefailnum),
  194.                 ",deldirsuccnum=",str(self.deldirsuccnum),
  195.                 ",deldirfailnum=",str(self.deldirfailnum)])
  196.         print(msg)
  197.     def logStat(self):
  198.         curtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
  199.         log = ''.join(["delfilenum=",str(self.delfilesuccnum),
  200.             ",deldirnum=",str(self.deldirsuccnum),",delfilefailnum=",
  201.             str(self.delfilefailnum),",deldirfailnum=",str(self.deldirfailnum)])
  202.         cos_log.info(log)
  203.         print(log)
  204.  
  205. #执行时间统计
  206. class TimeStat(object):
  207.     global cos_log
  208.     def __init__(self):
  209.         self.start()
  210.     def start(self):
  211.         self.start = datetime.datetime.now()
  212.         self.t1 = time.time()
  213.         msg = "delete task started  ..........."
  214.         print(msg)
  215.         cos_log.info(msg)
  216.     def end(self):
  217.         self.end = datetime.datetime.now()
  218.         self.t2 = time.time()
  219.         msg = "delete task ended\n\nrm task finished,\ntimecost:"+str(self.t2-self.t1) + " (s)"
  220.         print(msg)
  221.         cos_log.info(msg)
  222.  
  223. #删除文件列表中的文件
  224. def delfiles(cos_client, bucket, filelist):
  225.     for f in filelist:
  226.         delfile(cos_client, bucket, f)
  227.  
  228. #文件夹删除
  229. def delfolder(cos_client, bucket, folder):
  230.     global stat
  231.     global cos_log
  232.     if not folder:
  233.         return 0
  234.     delfolderreq = DelFolderRequest(bucket, folder)
  235.     retry = 0
  236.     while (retry < MAX_RETRY_TIMES):
  237.         ret = cos_client.del_folder(delfolderreq)
  238.         msg = "delfolder fail, bucket="+bucket+",folder="+folder+ret['message']
  239.         if (ret['code'] == 0):
  240.             break
  241.         elif (ret['code'] == -166):
  242.             cos_log.warning(msg)
  243.             break
  244.         #操作太频繁,频控
  245.         elif (ret['code'] == -71):
  246.             sleep(random.randint(1,5))
  247.             cos_log.warning(msg)
  248.             print(msg)
  249.             retry += 1
  250.             continue
  251.         #文件夹非空
  252.         elif (ret['code'] == -173):
  253.             break
  254.         else:
  255.             cos_log.warning(msg)
  256.             retry += 1
  257.     if (ret['code'] != 0 and  ret['code'] != -166):
  258.         stat.addDelDirFailNum()
  259.         cos_log.error("delfolder fail, bucket="+bucket+",folder="+folder+ret['message'])
  260.         return ret['code']
  261.     if (ret['code'] == 0):
  262.         stat.addDelDirSuccNum()
  263.         msg = "delfolder success, bucket="+bucket+",folder="+folder
  264.         cos_log.info(msg)
  265.         print(msg)
  266.     return 0
  267.  
  268. #文件删除
  269. def delfile(cos_client, bucket, filepath):
  270.     global stat
  271.     global cos_log
  272.     delfilereq = DelFileRequest(bucket, filepath)
  273.     retry = 0
  274.     while (retry < MAX_RETRY_TIMES):
  275.         ret = cos_client.del_file(delfilereq)
  276.         msg = "delfile fail bucket="+bucket+",file="+filepath+ret['message']
  277.         if (ret['code'] == 0):
  278.             break
  279.         #文件不存在
  280.         elif (ret['code'] == -166):
  281.             cos_log.warning(msg)
  282.             break
  283.         #单目录写操作过快
  284.         elif (ret['code'] == -143):
  285.             sleep(random.randint(1,5))
  286.             cos_log.warning(msg)
  287.             print(msg)
  288.             retry += 1
  289.             continue
  290.         #操作太频繁,频控
  291.         elif (ret['code'] == -71):
  292.             sleep(random.randint(1,5))
  293.             cos_log.warning(msg)
  294.             print(msg)
  295.             retry += 1
  296.             continue
  297.         else:
  298.             cos_log.warning(msg)
  299.             retry += 1
  300.             continue
  301.     if (ret['code'] != 0 and  ret['code'] != -166):
  302.         stat.addDelFileFailNum()
  303.         cos_log.error("delfile fail, bucket="+bucket+",file="+filepath+ret['message'])
  304.         return ret['code']
  305.     if (ret['code'] == 0):
  306.         stat.addDelFileSuccNum()
  307.         msg = "delfile success, bucket="+bucket+",file="+filepath
  308.         cos_log.info(msg)
  309.         print(msg)
  310.     return 0
  311.  
  312. #递归文件夹进行文件删除
  313. def delete_r(cos_client, bucket, path, thread_pool_file):
  314.     global stat
  315.     global cos_log
  316.     cos_log.debug("delete_r bucket:"+bucket+",path:"+path)
  317.     context = ""
  318.     #递归文件夹
  319.     while True:
  320.         listfolderreq = ListFolderRequest(bucket, path,199,u'eListDirOnly')
  321.         retry = 0
  322.         while (retry < MAX_RETRY_TIMES):
  323.             listret = cos_client.list_folder(listfolderreq)
  324.             if listret['code'] != 0 :
  325.                 retry += 1
  326.                 sleep(random.randint(1,5))
  327.                 continue
  328.             else:
  329.                 break
  330.         if (listret['code'] != 0):
  331.             cos_log.error("delete_r: list folder fail:"+path +",return msg:"+ listret['message'])
  332.             return listret['code']
  333.         if (len(listret['data']['infos']) == 0):
  334.             break;
  335.         for info in listret['data']['infos']:
  336.             fullname = path + info['name'] + u'/'
  337.             #当指示为False的时候不用比较,否则该标识一直递归传递下去
  338.             ret = delete_r(cos_client, bucket, fullname, thread_pool_file)
  339.             #当设置"删除失败不退出标志时",程序继续进行删除操作
  340.             if (ret != 0 and delete_fail_exist == 1):
  341.                 return ret
  342.         if (listret['data']['has_more'] == True):
  343.             context = listret['data']['context']
  344.             continue
  345.         else:
  346.             break
  347.     #本文件夹不删则不用list本文件夹的文件列表
  348.     while True:
  349.         listfilereq = ListFolderRequest(bucket, path,199,u'eListFileOnly')
  350.         retry = 0
  351.         while (retry < MAX_RETRY_TIMES):
  352.             listret = cos_client.list_folder(listfilereq)
  353.             if listret['code'] != 0 :
  354.                 retry += 1
  355.                 sleep(random.randint(1,5))
  356.                 continue
  357.             else:
  358.                 break
  359.         if (listret['code'] != 0):
  360.             cos_log.error("delete_r: list file fail:"+path +",return msg:"+ listret['message'])
  361.             return listret['code']
  362.  
  363.         if (len(listret['data']['infos']) == 0):
  364.             break;
  365.  
  366.         filelist = []
  367.         for info in listret['data']['infos']:
  368.             fullname = path + info['name']
  369.             filelist.append(fullname)
  370.             if (len(filelist) == ONE_TASK_DEL_FILE_NUMS):
  371.                 args = [cos_client, bucket, filelist]
  372.                 args_tuple = (args,None)
  373.                 args_list = [args_tuple]
  374.                 requests = threadpool.makeRequests(delfiles, args_list)
  375.                 for req in requests:
  376.                     thread_pool_file.putRequest(req)
  377.                 filelist = []
  378.                 continue
  379.         if (len(filelist) > 0):
  380.             args = [cos_client, bucket, filelist]
  381.             args_tuple = (args,None)
  382.             args_list = [args_tuple]
  383.             requests = threadpool.makeRequests(delfiles, args_list)
  384.             for req in requests:
  385.                 thread_pool_file.putRequest(req)
  386.             filelist = []
  387.  
  388.         cos_log.debug("delete_r thread pool file waiting\n")
  389.         thread_pool_file.wait()
  390.         cos_log.debug("delete_r thread pool file waiting end\n")
  391.  
  392.         if (listret['data']['has_more'] == True):
  393.             continue
  394.         else:
  395.             break
  396.     #删除本文件夹(上面已经删除子文件夹及子文件)
  397.     if (path != u"/"):
  398.         ret = delfolder(cos_client,bucket, path)
  399.         #当设置"删除失败不退出标志时",程序继续进行删除操作
  400.         if (ret != 0 and delete_folder_fail_exist == 1):
  401.             return ret
  402.     stat.logStat()
  403.     return 0
  404.  
  405. def cmd_rm(COSDIR):
  406.     global thread_pool
  407.     global cos_log
  408.     global stat
  409.     cos_log = loginit()
  410.     stat = FileStat()
  411.     timestat = TimeStat()
  412.     path = COSDIR.decode('utf-8')
  413.     thread_pool_dir = threadpool.ThreadPool(dir_thread_num)
  414.     thread_pool_file = threadpool.ThreadPool(file_thread_num)
  415.     cos_log.debug("bucket:"+bucket +",path:"+path)
  416.     args = [cos_client, bucket, path, thread_pool_file]
  417.     args_tuple = (args, None)
  418.     args_list = [args_tuple]
  419.     requests = threadpool.makeRequests(delete_r, args_list)
  420.     for req in requests:
  421.         thread_pool_dir.putRequest(req)
  422.  
  423.     cos_log.debug("thread_pool_dir waiting.....\n")
  424.     thread_pool_dir.wait()
  425.     thread_pool_dir.dismissWorkers(dir_thread_num, True)
  426.     cos_log.debug("thread_pool_dir wait end.....\n")
  427.  
  428.     timestat.end()
  429.     stat.logStat()
  430.  
  431. if sys.argv[1] in ['config','ls','mkdir','put','rm','delete','del'] and len(sys.argv) >= 3:
  432.     if sys.argv[1] == 'config':
  433.         parser = OptionParser()
  434.         parser.add_option("-a", "--appid", dest="appid", help="specify appid")
  435.         parser.add_option("-i", "--id", dest="secret_id", help="specify secret id")
  436.         parser.add_option("-k", "--key", dest="secret_key", help="specify secret key")
  437.         parser.add_option("-b", "--bucket", dest="bucket", help="specify bucket")
  438.         (options, args) = parser.parse_args()
  439.         CMD_LIST['config'] = cmd_configure
  440.         CMD_LIST['config'](args, options)
  441.     if sys.argv[1] == 'ls':
  442.         cmd_loadconfigure()
  443.         cos_client = CosClient(appid, secret_id, secret_key)
  444.         COSDIR = sys.argv[2]
  445.         cmd_lsdir(COSDIR)
  446.     if sys.argv[1] == 'mkdir':
  447.         cmd_loadconfigure()
  448.         cos_client = CosClient(appid, secret_id, secret_key)
  449.         COSDIR = sys.argv[2]
  450.         cmd_mkdir(COSDIR)
  451.     if sys.argv[1] == 'put' and len(sys.argv) == 4:
  452.         cmd_loadconfigure()
  453.         cos_client = CosClient(appid, secret_id, secret_key)
  454.         LOCALFILE = sys.argv[2]
  455.         COSFILE = sys.argv[3]
  456.         cmd_put(LOCALFILE,COSFILE)
  457.     if sys.argv[1] in ('rm','delete','del'):
  458.         cmd_loadconfigure()
  459.         cos_client = CosClient(appid, secret_id, secret_key)
  460.         COSDIR = sys.argv[2]
  461.         path = COSDIR.decode('utf-8')
  462.         cmd_rm(path)
  463. else:
  464.     print(HELP)
  465.     exit()

转载请保留固定链接： https://linuxeye.com/program/coscmd.html
------分隔线----------------------------
标签：
栏目列表
推荐内容
热点内容

Copyright © 2021 LINUXEYE
沪ICP备2021018617号-2