From d9629fbf969d94e9650929341e8c3ce616eba0b6 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 24 Jul 2020 16:38:53 +0800 Subject: [PATCH 1/2] support fetch --- README.rst | 2 +- demo/demo.py | 136 +++++++++ demo/tce_demo.py | 611 +++++++++++++++++++++++++++++++++++++++ qcloud_cos/cos_client.py | 79 ++++- 4 files changed, 826 insertions(+), 2 deletions(-) create mode 100644 demo/demo.py create mode 100644 demo/tce_demo.py diff --git a/README.rst b/README.rst index 6a543c28..dc9f926e 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ __________ 使用方法 __________ -使用python sdk，参照 https://github.com/tencentyun/cos-python-sdk-v5/blob/master/qcloud_cos/demo.py +使用python sdk，参照 https://github.com/tencentyun/cos-python-sdk-v5/blob/master/demo/demo.py cos最新可用地域，参照 https://cloud.tencent.com/document/product/436/6224 diff --git a/demo/demo.py b/demo/demo.py new file mode 100644 index 00000000..31195748 --- /dev/null +++ b/demo/demo.py @@ -0,0 +1,136 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos import CosServiceError +from qcloud_cos import CosClientError + +import sys +import logging + +# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7以及Python3.x + +# pip安装指南:pip install -U cos-python-sdk-v5 + +# cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 设置用户属性, 包括secret_id, secret_key, region +# appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 +secret_id = 'AKID15IsskiBQACGbAo6WhgcQbVls7HmuG00' # 替换为用户的secret_id +secret_key = 'csivKvxxrMvSvQpMWHuIz12pThQQlWRW' # 替换为用户的secret_key +region = 'ap-beijing-1' # 替换为用户的region +token = None # 使用临时密钥需要传入Token，默认为空,可不填 +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 +client = CosS3Client(config) + +# 文件流 简单上传 +file_name = 'test.txt' +with open('test.txt', 'rb') as fp: + response = client.put_object( + Bucket='test04-123456789', # Bucket由bucketname-appid组成 + Body=fp, + Key=file_name, + StorageClass='STANDARD', + ContentType='text/html; charset=utf-8' + ) + print(response['ETag']) + +# 字节流 简单上传 +response = client.put_object( + Bucket='test04-123456789', + Body=b'abcdefg', + Key=file_name +) +print(response['ETag']) + +# 本地路径 简单上传 +response = client.put_object_from_local_file( + Bucket='test04-123456789', + LocalFilePath='local.txt', + Key=file_name, +) +print(response['ETag']) + +# 设置HTTP头部 简单上传 +response = client.put_object( + Bucket='test04-123456789', + Body=b'test', + Key=file_name, + ContentType='text/html; charset=utf-8' +) +print(response['ETag']) + +# 设置自定义头部 简单上传 +response = client.put_object( + Bucket='test04-123456789', + Body=b'test', + Key=file_name, + Metadata={ + 'x-cos-meta-key1': 'value1', + 'x-cos-meta-key2': 'value2' + } +) +print(response['ETag']) + +# 高级上传接口(推荐) +response = client.upload_file( + Bucket='test04-123456789', + LocalFilePath='local.txt', + Key=file_name, + PartSize=10, + MAXThread=10 +) +print(response['ETag']) + +# 文件下载 获取文件到本地 +response = client.get_object( + Bucket='test04-123456789', + Key=file_name, +) +response['Body'].get_stream_to_file('output.txt') + +# 文件下载 获取文件流 +response = client.get_object( + Bucket='test04-123456789', + Key=file_name, +) +fp = response['Body'].get_raw_stream() +print(fp.read(2)) + +# 文件下载 设置Response HTTP 头部 +response = client.get_object( + Bucket='test04-123456789', + Key=file_name, + ResponseContentType='text/html; charset=utf-8' +) +print(response['Content-Type']) +fp = response['Body'].get_raw_stream() +print(fp.read(2)) + +# 文件下载 指定下载范围 +response = client.get_object( + Bucket='test04-123456789', + Key=file_name, + Range='bytes=0-10' +) +fp = response['Body'].get_raw_stream() +print(fp.read()) + +# 文件下载 捕获异常 +try: + response = client.get_object( + Bucket='test04-123456789', + Key='not_exist.txt', + ) + fp = response['Body'].get_raw_stream() + print(fp.read(2)) +except CosServiceError as e: + print(e.get_origin_msg()) + print(e.get_digest_msg()) + print(e.get_status_code()) + print(e.get_error_code()) + print(e.get_error_msg()) + print(e.get_resource_location()) + print(e.get_trace_id()) + print(e.get_request_id()) diff --git a/demo/tce_demo.py b/demo/tce_demo.py new file mode 100644 index 00000000..464677a2 --- /dev/null +++ b/demo/tce_demo.py @@ -0,0 +1,611 @@ +# coding=utf-8 + +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos import CosServiceError +from qcloud_cos import CosClientError +import sys +import random +import string + + +class CosV5PythonSDKTest(object): + def __init__(self, region, appid, secret_id, secret_key, end_point, + scheme): + self.region = region + self.appid = appid + self.secret_id = secret_id + self.secret_key = secret_key + self.end_point = end_point + self.scheme = scheme + self.cos_config = CosConfig( + Secret_id=self.secret_id, + Secret_key=self.secret_key, + Endpoint=self.end_point % region, + Scheme=self.scheme) + self.client = CosS3Client(self.cos_config) + + def create_bucket(self, bucket_name): + # 创建存储桶 + self.client.create_bucket(Bucket=bucket_name + '-' + self.appid) + return 0 + + def create_bucket_with_ACL(self, bucket_name, acl): + # 创建存储桶 ACL + self.client.create_bucket( + Bucket=bucket_name + '-' + self.appid, ACL=acl) + return 0 + + def create_bucket_with_GrantFullControl(self, bucket_name, owner_uin, + sub_uin): + # 创建存储桶 GrantFullControl + grant_full_control = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, + sub_uin) + self.client.create_bucket( + Bucket=bucket_name + '-' + self.appid, + GrantFullControl=grant_full_control) + return 0 + + def create_bucket_with_GrantRead(self, bucket_name, owner_uin, sub_uin): + # 创建存储桶 GrantRead + grant_read = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, sub_uin) + self.client.create_bucket( + Bucket=bucket_name + '-' + self.appid, GrantRead=grant_read) + return 0 + + def create_bucket_with_GrantWrite(self, bucket_name, owner_uin, sub_uin): + # 创建存储桶 GrantRead + grant_write = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, sub_uin) + self.client.create_bucket( + Bucket=bucket_name + '-' + self.appid, GrantRead=grant_write) + return 0 + + def delete_bucket(self, bucket_name): + # 删除存储桶 + self.client.delete_bucket(Bucket=bucket_name + '-' + self.appid) + return 0 + + def head_bucket(self, bucket_name): + # 判断存储桶是否存在 + self.client.head_bucket(Bucket=bucket_name + '-' + self.appid) + return 0 + + def get_bucket_location(self, bucket_name): + # 获取存储桶地域信息 + resp = self.client.get_bucket_location(Bucket=bucket_name + '-' + + self.appid) + return resp["LocationConstraint"] + + def list_objects(self, bucket_name): + # 获取存储桶下文件列表 + resp = self.client.list_objects(Bucket=bucket_name + '-' + self.appid) + return resp + + def list_objects_with_prefix(self, bucket_name, prefix): + # 获取存储桶下有prefix前缀的文件列表 + resp = self.client.list_objects( + Bucket=bucket_name + '-' + self.appid, Prefix=prefix) + return resp + + def list_objects_with_delimiter(self, bucket_name, delimiter): + # 模拟文件夹结构获取存储桶下 的文件列表 + resp = self.client.list_objects( + Bucket=bucket_name + '-' + self.appid, Delimiter=delimiter) + return resp + + def list_objects_with_Marker(self, bucket_name, marker): + # 获取存储桶下文件名包含marker的文件列表 + resp = self.client.list_objects( + Bucket=bucket_name + '-' + self.appid, Marker=marker) + return resp + + def list_objects_with_Maxkeys(self, bucket_name, maxkeys): + # 获取存储桶下最多maxkeys个文件列表 + resp = self.client.list_objects( + Bucket=bucket_name + '-' + self.appid, MaxKeys=maxkeys) + return resp + + def put_bucket_acl(self, bucket_name, acl): + # 设置存储桶访问控制权限 ACL + self.client.put_bucket_acl( + Bucket=bucket_name + '-' + self.appid, ACL=acl) + return 0 + + def put_bucket_acl_with_GrantFullControl(self, bucket_name, owner_uin, + sub_uin): + # 设置存储桶访问控制权限 GrantFullControl + grant_full_control = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, + sub_uin) + self.client.put_bucket_acl( + Bucket=bucket_name + '-' + self.appid, + GrantFullControl=grant_full_control) + + def put_bucket_acl_with_GrantRead(self, bucket_name, owner_uin, sub_uin): + # 设置存储桶访问控制权限 GrantRead + grant_read = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, sub_uin) + self.client.put_bucket_acl( + Bucket=bucket_name + '-' + self.appid, GrantRead=grant_read) + return 0 + + def put_bucket_acl_with_GrantWrite(self, bucket_name, owner_uin, sub_uin): + # 设置存储桶访问控制权限 GrantWrite + grant_write = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, sub_uin) + self.client.put_bucket_acl( + Bucket=bucket_name + '-' + self.appid, GrantRead=grant_write) + + def get_bucket_acl(self, bucket_name): + # 获取存储桶访问控制权限 + resp = self.client.get_bucket_acl(Bucket=bucket_name + "-" + + self.appid) + return resp + + def put_bucket_cors(self, bucket_name, max_age_seconds=0): + # 设置一条存储桶跨域访问规则 + corsconfig = { + "CORSRule": [{ + "ID": "CORSRule 1", + "MaxAgeSeconds": max_age_seconds, + "AllowedOrigin": ["http://cloud.tencent.com"], + "AllowedMethod": ["GET"] + }] + } + self.client.put_bucket_cors( + Bucket=bucket_name + '-' + self.appid, + CORSConfiguration=corsconfig) + return 0 + + def put_bucket_multi_cors(self, bucket_name): + # 设置多条存储桶跨域访问规则 + corsconfig = { + "CORSRule": [{ + "ID": "CORSRule 1", + "AllowedOrigin": ["http://cloud.tencent.com"], + "AllowedMethod": ["GET"] + }, { + "ID": "CORSRule 2", + "AllowedOrigin": ["http://cloud.tencent.com"], + "AllowedMethod": ["POST"] + }, { + "ID": "CORSRule 3", + "AllowedOrigin": ["http://cloud.tencent.com"], + "AllowedMethod": ["PUT"] + }] + } + self.client.put_bucket_cors( + Bucket=bucket_name + '-' + self.appid, + CORSConfiguration=corsconfig) + return 0 + + def get_bucket_cors(self, bucket_name): + # 获取存储桶跨域访问规则 + resp = self.client.get_bucket_cors(Bucket=bucket_name + '-' + + self.appid) + return resp + + def delete_bucket_cors(self, bucket_name): + # 删除存储桶跨域访问规则 + self.client.delete_bucket_cors(Bucket=bucket_name + '-' + self.appid) + return 0 + + def put_object_str(self, bucket_name, obj_name, str_len): + # 上传字符串到对象 + self.client.put_object( + Bucket=bucket_name + '-' + self.appid, + Body="".join( + random.choice(string.ascii_letters + string.digits) + for i in range(str_len)), + Key=obj_name) + return 0 + + def put_object_file(self, bucket_name, obj_name, file_name): + # 上传文件到对象 + self.client.put_object( + Bucket=bucket_name + '-' + self.appid, + Body=open(file_name, "rb").read(), + Key=obj_name) + return 0 + + def head_object(self, bucket_name, obj_name): + # 对象是否存在，获取对象属性 + resp = self.client.head_object( + Bucket=bucket_name + '-' + self.appid, Key=obj_name) + return resp + + def get_object(self, bucket_name, obj_name, file_name): + # 下载对象 + resp = self.client.get_object( + Bucket=bucket_name + '-' + self.appid, Key=obj_name) + resp["Body"].get_stream_to_file(file_name) + return 0 + + def delete_object(self, bucket_name, obj_name): + # 删除对象 + self.client.delete_object( + Bucket=bucket_name + '-' + self.appid, Key=obj_name) + return 0 + + def delete_objects(self, bucket_name, obj_list): + # 批量删除对象 + _obj = [] + for obj in obj_list: + _obj.append({"Key": obj}) + delete = { + "Object": _obj, + } + resp = self.client.delete_objects( + Bucket=bucket_name + '-' + self.appid, Delete=delete) + return resp + + def delete_objects_with_quiet(self, bucket_name, obj_list, quiet): + # 批量删除对象 + _obj = [] + for obj in obj_list: + _obj.append({"Key": obj}) + delete = {"Object": _obj, "Quiet": quiet} + resp = self.client.delete_objects( + Bucket=bucket_name + '-' + self.appid, Delete=delete) + return resp + + def copy_object_in_same_bucket(self): + # 桶内copy对象 + bucket_name = "mainbkt" + object_key = "obj_copy" + copy_source = { + "Appid": self.appid, + "Bucket": "mainbkt", + "Key": "obj", + "Region": self.region + } + resp = self.client.copy_object( + Bucket=bucket_name + '-' + self.appid, + Key=object_key, + CopySource=copy_source, + CopyStatus="Copy") + return resp + + def copy_object_in_different_bucket(self): + # 桶内copy对象 + bucket_name = "wook-gao" + object_key = "obj_copy" + copy_source = { + "Appid": self.appid, + "Bucket": "burning", + "Key": "test", + "Region": self.region + } + resp = self.client.copy_object( + Bucket=bucket_name + '-' + self.appid, + Key=object_key, + CopySource=copy_source) + return resp + + def put_object_acl(self, bucket_name, obj_name, acl): + # 设置对象访问控制权限 ACL + self.client.put_object_acl( + Bucket=bucket_name + '-' + self.appid, Key=obj_name, ACL=acl) + return 0 + + def put_object_acl_with_GrantFullControl(self, bucket_name, obj_name, + owner_uin, sub_uin): + # 设置对象访问控制权限 GrantFullControl + grant_full_control = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, + sub_uin) + self.client.put_object_acl( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + GrantFullControl=grant_full_control) + + def put_object_acl_with_GrantRead(self, bucket_name, obj_name, owner_uin, + sub_uin): + # 设置对象访问控制权限 GrantRead + grant_read = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, sub_uin) + self.client.put_object_acl( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + GrantRead=grant_read) + return 0 + + def put_object_acl_with_GrantWrite(self, bucket_name, obj_name, owner_uin, + sub_uin): + # 设置对象访问控制权限 GrantWrite + grant_write = 'id="qcs::cam::uin/%s:uin/%s"' % (owner_uin, sub_uin) + self.client.put_object_acl( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + GrantWrite=grant_write) + + def get_object_acl(self, bucket_name, obj_name): + # 获取对象访问控制权限 + resp = self.client.get_object_acl( + Bucket=bucket_name + '-' + self.appid, Key=obj_name) + return resp + + def create_multipart_upload(self, bucket_name, obj_name): + # 创建分块上传 + resp = self.client.create_multipart_upload( + Bucket=bucket_name + '-' + self.appid, Key=obj_name) + return resp + + def abort_multipart_upload(self, bucket_name, obj_name, upload_id): + # 放弃分块上传 + self.client.abort_multipart_upload( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + UploadId=upload_id) + return 0 + + def upload_part(self, bucket_name, obj_name, part_number, upload_id, + str_len): + # 上传分块 + resp = self.client.upload_part( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + Body="".join( + random.choice(string.ascii_letters + string.digits) + for i in range(str_len)), + PartNumber=part_number, + UploadId=upload_id) + return resp + + def list_parts(self, bucket_name, obj_name, upload_id): + # 列出上传分块 + resp = self.client.list_parts( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + UploadId=upload_id) + return resp + + def complete_multipart_upload(self, bucket_name, obj_name, upload_id, + multipart_upload): + # 完成分块上传 + resp = self.client.complete_multipart_upload( + Bucket=bucket_name + '-' + self.appid, + Key=obj_name, + UploadId=upload_id, + MultipartUpload=multipart_upload) + return resp + + def list_multipart_uploads(self, bucket_name): + resp = self.client.list_multipart_uploads(Bucket=bucket_name + '-' + + self.appid) + return resp + + def upload_file(self, bucket_name, obj_name, file_path): + # 文件上传(断点续传) + resp = self.client.upload_file( + Bucket=bucket_name + "-" + self.appid, + Key=obj_name, + LocalFilePath=file_path) + return resp + + +if __name__ == "__main__": + + main = CosV5PythonSDKTest( + region="", + appid="", + secret_id="", + secret_key="", + end_point="", + scheme="http") + + try: + + # ---------------------- 存储桶相关操作 ---------------------- + + # 创建存储桶 + main.create_bucket(bucket_name="mainbkt") + + # 创建存储桶（带ACL参数） + main.create_bucket_with_ACL( + bucket_name="mainbkt", acl="public-read-write") + + # 创建存储桶（带GrantFullControl参数） + main.create_bucket_with_GrantFullControl( + bucket_name="mainbkt", owner_uin="1", sub_uin="1") + + # 创建存储桶（GrantRead） + main.create_bucket_with_GrantRead( + bucket_name="mainbkt", owner_uin="1", sub_uin="1") + + # 创建存储桶（GrantWrite） + main.create_bucket_with_GrantWrite( + bucket_name="mainbkt", owner_uin="1", sub_uin="1") + + # 获取存储桶ACL + resp = main.get_bucket_acl(bucket_name="mainbkt") + + # head存储桶（查询存储桶是否存在） + main.head_bucket(bucket_name="mainbkt") + main.head_bucket(bucket_name="notexist") + + # 获取bucket地域信息 + resp = main.get_bucket_location(bucket_name="mainbkt") + + # 删除存储桶 + main.delete_bucket(bucket_name="mainbkt") + + # 列出存储桶中对象 + resp = main.list_objects(bucket_name="mainbkt") + + # 列出所有带前缀的对象 + resp = main.list_objects_with_prefix( + bucket_name="mainbkt", prefix="obj") + + # 列出所有对象（设置“/”来模拟目录层级） + resp = main.list_objects_with_delimiter( + bucket_name="mainbkt", delimiter="/") + + # 列出所有对象（带Marker参数） + resp = main.list_objects_with_Marker(bucket_name="mainbkt", marker="a") + + # 列出所有对象（最多不超过Maxkeys个） + resp = main.list_objects_with_Maxkeys(bucket_name="mainbkt", maxkeys=2) + + # 设置存储桶跨域资源配置（CORSRule写在put_bucket_cors方法里） + main.put_bucket_cors(bucket_name="mainbkt") + + # 设置多条存储桶跨域资源配置（CORSRule写在put_bucket_cors方法里） + main.put_bucket_multi_cors(bucket_name="mainbkt") + + # 获取存储桶跨域资源配置 + resp = main.get_bucket_cors(bucket_name="mainbkt") + + # 删除存储桶跨域资源配置 + main.delete_bucket_cors(bucket_name="mainbkt") + + # ---------------------- 对象相关操作 ---------------------- + + # 字节（字串）对象上传 1MB + main.put_object_str( + bucket_name="mainbkt", obj_name="str1m.txt", str_len=1024 * 1024) + + # 字节（字串）对象上传 1GB + main.put_object_str( + bucket_name="mainbkt", + obj_name="str1g.txt", + str_len=1024 * 1024 * 1024) + + # 文件对象上传 5MB + main.put_object_file( + bucket_name="mainbkt", + obj_name="file5m.txt", + file_name="f:\\test5m.txt") + + # 文件对象上传 1GB + main.put_object_file( + bucket_name="mainbkt", + obj_name="file1g.txt", + file_name="f:\\test1g.txt") + + # 下载对象 5MB + main.get_object( + bucket_name="mainbkt", + obj_name="str5m.txt", + file_name="f:\\download5m.txt") + + # 下载对象 1GB + main.get_object( + bucket_name="mainbkt", + obj_name="str1g.txt", + file_name="f:\\download1g.txt") + + # 下载不存在对象 + main.get_object( + bucket_name="mainbkt", + obj_name="notexist.txt", + file_name="f:\\x.txt") + + # 删除对象 + main.delete_object(bucket_name="mainbkt", obj_name="str1m.txt") + + # 删除对象 1GB + main.delete_object(bucket_name="mainbkt", obj_name="str1g.txt") + + # 删除多个对象，传入对象名list + resp = main.delete_objects( + bucket_name="mainbkt", obj_list=["obj", "obja", "objb"]) + + # 删除多个对象，传入对象名list和quiet参数 + resp = main.delete_objects_with_quiet( + bucket_name="mainbkt", + obj_list=["obj", "obja", "objb"], + quiet="true") + + # 获取文件属性 + resp = main.head_object(bucket_name="mainbkt", obj_name="obja") + + # 同存储桶内copy对象（相关参数在copy_object_in_same_bucket方法内修改） + resp = main.copy_object_in_same_bucket() + + # 不同存储桶内copy对象（相关参数在copy_object_in_different_bucket方法内修改） + resp = main.copy_object_in_different_bucket() + + # 设置文件ACL + main.put_object_acl( + bucket_name="mainbkt", obj_name="obj", acl="public-read") + + # 设置文件ACL（带GrantFullControl的用户ID） + main.put_object_acl_with_GrantFullControl( + bucket_name="mainbkt", obj_name="obj", owner_uin="1", sub_uin="1") + + # 设置文件ACL（带GrantRead的用户ID） + main.put_object_acl_with_GrantRead( + bucket_name="mainbkt", obj_name="obj", owner_uin="1", sub_uin="1") + + # 设置文件ACL（带GrantWrite的用户ID） + main.put_object_acl_with_GrantWrite( + bucket_name="mainbkt", obj_name="obj", owner_uin="1", sub_uin="1") + + # 获取对象ACL + resp = main.get_object_acl(bucket_name="mainbkt", obj_name="obj") + + # 创建分块上传 + resp_create = main.create_multipart_upload( + bucket_name="mainbkt", obj_name="part100m.txt") + + upload_id = resp_create["UploadId"] + + # 上传分块，每个分块2m，上传50个分块 + resp_upload_list = [] + for i in range(50): + resp_upload = main.upload_part( + bucket_name="mainbkt", + obj_name="part100m.txt", + part_number=i + 1, + upload_id=upload_id, + str_len=2 * 1024 * 1024) + resp_upload_list.append(resp_upload) + + # 列出所有上传分块 + resp_list = main.list_parts( + bucket_name="mainbkt", + obj_name="part100m.txt", + upload_id=upload_id) + part_list = resp_list["Part"] + + # 完成分块上传 + main.complete_multipart_upload( + bucket_name="mainbkt", + obj_name="part100m.txt", + upload_id=upload_id, + multipart_upload={"Part": part_list}) + + # 放弃分块上传 + main.abort_multipart_upload( + bucket_name="mainbkt", + obj_name="part100m.txt", + upload_id=upload_id) + + # upload_file接口上传测试 + + # 上传对象 5MB + main.upload_file( + bucket_name="mainbkt", + obj_name="upload5m.txt", + file_path="f:\\test5m.txt") + + # 上传对象 20MB + main.upload_file( + bucket_name="mainbkt", + obj_name="upload20m.txt", + file_path="f:\\test20m.txt") + + # 上传对象 21MB + main.upload_file( + bucket_name="mainbkt", + obj_name="upload21m.txt", + file_path="f:\\test21m.txt") + + # 上传对象 1GB + main.upload_file( + bucket_name="mainbkt", + obj_name="upload1g.txt", + file_path="f:\\test1g.txt") + + print("Succeeded") + + except Exception as e: + + # 可细化CosServiceError/CosClientError + print("Failed") diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 7028e73b..d9f6ebf4 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -220,7 +220,7 @@ def get_auth(self, Method, Bucket, Key, Expired=300, Headers={}, Params={}): auth = CosS3Auth(self._conf, Key, Params, Expired) return auth(r).headers['Authorization'] - def send_request(self, method, url, bucket, timeout=30, **kwargs): + def send_request(self, method, url, bucket, timeout=30, cos_request=True, **kwargs): """封装request库发起http请求""" if self._conf._timeout is not None: # 用户自定义超时时间 timeout = self._conf._timeout @@ -264,6 +264,8 @@ def send_request(self, method, url, bucket, timeout=30, **kwargs): continue raise CosClientError(str(e)) + if not cos_request: + return res if res.status_code >= 400: # 所有的4XX,5XX都认为是COSServiceError if method == 'HEAD' and res.status_code == 404: # Head 需要处理 info = dict() @@ -3400,6 +3402,81 @@ def update_object_meta(self, Bucket, Key, **kwargs): ) return response + def put_async_fetch_task(self, Bucket, FetchTaskConfiguration={}, **kwargs): + """发起异步拉取对象到COS的任务 + + :param Bucket(string): 存储桶名称. + :param FetchTaskConfiguration(dict): 异步拉取任务的配置. + :kwargs(dict): 扩展参数. + :return(dict): 异步任务成功返回的结果，包含Taskid等信息. + + .. code-block:: python + + config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 + client = CosS3Client(config) + # 发起异步拉取任务 + response = client.put_async_fetch_task( + Bucket='bucket', + FetchTaskConfiguration={ + 'Url': + 'Key': + 'MD5': + 'SuccessCallbackUrl': + 'FailureCallbackUrl': + } + ) + """ + url = '{scheme}://{region}.migration.myqcloud.com/{bucket}/'.format(scheme=self._conf._scheme, region=self._conf._region, bucket=Bucket) + if self._conf._domain is not None: + url = '{scheme}://{domain}/{bucket}/'.format(scheme=self._conf._scheme, domain=self._conf._domain, bucket=Bucket) + headers = {'Content-Type': 'application/json'} + signed_key = Bucket + '/' + rt = self.send_request( + method='POST', + url=url, + bucket=None, + data=json.dumps(FetchTaskConfiguration), + headers=headers, + auth=CosS3Auth(self._conf, signed_key), + cos_request=False + ) + data = rt.json() + return data + + def get_async_fetch_task(self, Bucket, TaskId, **kwargs): + """获取异步拉取对象到COS的任务状态 + + :param Bucket(string): 存储桶名称. + :param TaskId(string): 异步拉取任务查询的唯一标识. + :kwargs(dict): 扩展参数. + :return(dict): 异步任务的状态 + + .. code-block:: python + + config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 + client = CosS3Client(config) + # 获取异步拉取任务 + response = client.get_async_fetch_task( + Bucket='bucket', + TaskId='string' + ) + """ + url = '{scheme}://{region}.migration.myqcloud.com/{bucket}/{task_id}'.format(scheme=self._conf._scheme, region=self._conf._region, bucket=Bucket, task_id=TaskId) + if self._conf._domain is not None: + url = '{scheme}://{domain}/{bucket}/{task_id}'.format(scheme=self._conf._scheme, domain=self._conf._domain, bucket=Bucket, task_id=TaskId) + headers = {'Content-Type': 'application/json'} + signed_key = '{bucket}/{task_id}'.format(bucket=Bucket, task_id=TaskId) + rt = self.send_request( + method='GET', + url=url, + bucket=None, + headers=headers, + auth=CosS3Auth(self._conf, signed_key), + cos_request=False + ) + data = rt.json() + return data + if __name__ == "__main__": pass From 8fe150bfe8acc056e9ac540b5e2f84714c161255 Mon Sep 17 00:00:00 2001 From: tiedu Date: Fri, 24 Jul 2020 17:01:48 +0800 Subject: [PATCH 2/2] modify demo --- demo/demo.py | 2 +- demo/fetch_demo.py | 37 +++++++++++++++++++++++++++++++++++++ qcloud_cos/version.py | 2 +- setup.py | 2 +- 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 demo/fetch_demo.py diff --git a/demo/demo.py b/demo/demo.py index 31195748..88fe4964 100644 --- a/demo/demo.py +++ b/demo/demo.py @@ -19,7 +19,7 @@ # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 secret_id = 'AKID15IsskiBQACGbAo6WhgcQbVls7HmuG00' # 替换为用户的secret_id secret_key = 'csivKvxxrMvSvQpMWHuIz12pThQQlWRW' # 替换为用户的secret_key -region = 'ap-beijing-1' # 替换为用户的region +region = 'ap-beijing' # 替换为用户的region token = None # 使用临时密钥需要传入Token，默认为空,可不填 config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) # 获取配置对象 client = CosS3Client(config) diff --git a/demo/fetch_demo.py b/demo/fetch_demo.py new file mode 100644 index 00000000..025ccf6d --- /dev/null +++ b/demo/fetch_demo.py @@ -0,0 +1,37 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos import CosServiceError +from qcloud_cos import CosClientError + +import sys +import logging + + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 设置用户属性, 包括secret_id, secret_key, region +# appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 +secret_id = '' # 替换为用户的secret_id +secret_key = '' # 替换为用户的secret_key +region = 'ap-beijing' # 替换为用户的region +token = None # 使用临时密钥需要传入Token，默认为空,可不填 +scheme = 'http' +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) # 获取配置对象 +client = CosS3Client(config) + +test_bucket = 'examplebucket-1250000000' +# 发起拉取任务 +response = client.put_async_fetch_task( + Bucket=test_bucket, + FetchTaskConfiguration={ + 'Url': 'http://examplebucket-1250000000.cos.ap-beijing.myqcloud.com/exampleobject', + 'Key': 'exampleobject' + } +) + +# 查询拉取任务 +response = client.get_async_fetch_task( + Bucket=test_bucket, + TaskId=response['data']['taskid'] +) diff --git a/qcloud_cos/version.py b/qcloud_cos/version.py index be46c417..c267e462 100644 --- a/qcloud_cos/version.py +++ b/qcloud_cos/version.py @@ -1,2 +1,2 @@ -__version__ = '5.1.8.1' +__version__ = '5.1.8.2' diff --git a/setup.py b/setup.py index 930d2fae..cb98d87a 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def long_description(): setup( name='cos-python-sdk-v5', - version='1.8.1', + version='1.8.2', url='https://www.qcloud.com/', license='MIT', author='tiedu, lewzylu, channingliu',