支持同名文件上传时跳过

.gitignore

@@ -16,4 +16,6 @@ coscmd/*.swn
 coscmd/*.swo
 *.project
 *.pydevproject
-.settings/
+.settings/
+.vscode/
+xcoscmd.egg-info

README.rst

@@ -1,4 +1,4 @@
-COSCMD
+XCOSCMD
 #######################
 
 .. image:: https://img.shields.io/pypi/v/coscmd.svg
@@ -16,16 +16,43 @@ _______
 安装指南
 __________
 
-使用pip安装 ::
+::
 
-    pip install -U coscmd
+    python setup.py install
+    cd dist
+    sudo easy_install coscmd-1.8.6.3-py3.6.egg # 提前安装easy_install
+    ln -s {{PYTHON_DIR}}/bin/coscmd xcoscmd # [可选]为避免和官方coscmd冲突，通过软链使用此版本
+或
+::
+    pip install xcoscmd -i https://pypi.tuna.tsinghua.edu.cn/simple
+或
+::
+   pip install xcoscmd -i https://pypi.doubanio.com/simple
+或
+::
+   pip install xcoscmd -i http://mirrors.tencent.com/pypi/simple
+
+改造
+__________
+上传命令，支持上传同名文件不覆盖，直接跳过
 
-手动安装::
+命令::
 
-    python setup.py install
+    xcoscmd -rm ./app /app
+
+上传结果输出优化：上传失败的log红色显示，上传成功的log绿色显示，更加直观的显示上传结果
 
 使用方法
 __________
 
 使用coscmd，参照 https://cloud.tencent.com/document/product/436/10976
 
+附录：CentOS Python环境安装
+__________
+
+::
+
+   yum install python3
+   wget https://bootstrap.pypa.io/3.2/get-pip.py
+   python3 get-pip.py
+

RELEASE.rst

@@ -0,0 +1,12 @@
+XCOSCMD
+
+发布指南
+__________
+
+::
+
+    python setup.py sdist bdist_wheel
+    pip install twine
+    twine check dist/*
+    twine upload dist/*
+

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 from platform import python_version_tuple
-from coscmd import cos_global
+from xcoscmd import cos_global
 
 
 def requirements():
@@ -22,19 +22,19 @@ def long_description():
 
 
 setup(
-    name='coscmd',
+    name='xcoscmd',
     version=cos_global.Version,
     url='https://www.qcloud.com/',
     license='MIT',
-    author='lewzylu',
-    author_email='327874225@qq.com',
-    description='simple command for cos',
+    author='aslinwang',
+    author_email='dream_jet@qq.com',
+    description='simple command for cos, forked from https://github.com/tencentyun/coscmd',
     long_description=long_description(),
     packages=find_packages(),
     install_requires=requirements(),
     entry_points={
         'console_scripts': [
-            'coscmd=coscmd.cos_cmd:_main',
+            'xcoscmd=xcoscmd.cos_cmd:_main',
         ],
     }
 )

coscmd/cos_auth.py → xcoscmd/cos_auth.py

@@ -10,13 +10,13 @@
 from requests.auth import AuthBase
 
 if sys.version > '3':
-    from coscmd.cos_global import Version
-    from coscmd.cos_comm import to_bytes
+    from xcoscmd.cos_global import Version
+    from xcoscmd.cos_comm import to_bytes
 else:
     from cos_global import Version
     from cos_comm import to_bytes
 
-logger = logging.getLogger("coscmd")
+logger = logging.getLogger("xcoscmd")
 
 
 class CosS3Auth(AuthBase):
@@ -80,7 +80,7 @@ def __call__(self, r):
             r.headers['Authorization'] = ""
         if self._token is not None:
             r.headers['x-cos-security-token'] = self._token
-        r.headers['User-agent'] = 'coscmd-v' + Version
+        r.headers['User-agent'] = 'xcoscmd-v' + Version
         logger.debug("sign_key" + str(sign_key))
         logger.debug(r.headers['Authorization'])
 

coscmd/cos_client.py → xcoscmd/cos_client.py

@@ -24,15 +24,17 @@
 import qcloud_cos
 
 if sys.version > '3':
-    from coscmd.cos_auth import CosS3Auth
-    from coscmd.cos_threadpool import SimpleThreadPool
-    from coscmd.cos_comm import *
+    from xcoscmd.cos_auth import CosS3Auth
+    from xcoscmd.cos_threadpool import SimpleThreadPool
+    from xcoscmd.cos_comm import *
+    from xcoscmd.cos_color import *
 else:
     from cos_auth import CosS3Auth
     from cos_threadpool import SimpleThreadPool
     from cos_comm import *
+    from cos_color import *
 
-logger = logging.getLogger("coscmd")
+logger = logging.getLogger("xcoscmd")
 
 
 class CoscmdConfig(object):
@@ -154,6 +156,15 @@ def check_file_md5(self, _local_path, _cos_path, _md5):
                 return False
             else:
                 return True
+
+    def check_file_name(self, _local_path, _cos_path):
+        url = self._conf.uri(path=quote(to_printable_str(_cos_path)))
+        rt = self._session.head(
+            url=url, auth=CosS3Auth(self._conf), stream=True)
+        if rt.status_code != 200: #资源不存在
+            return False
+        else:
+            return True
 
     def sign_url(self, cos_path, timeout=10000):
         cos_path = to_printable_str(cos_path)
@@ -298,10 +309,6 @@ def upload_file_list(upload_filelist):
             return -1
 
     def single_upload(self, local_path, cos_path, _http_headers='{}', **kwargs):
-        logger.info(u"Upload {local_path}   =>   cos://{bucket}/{cos_path}".format(
-            bucket=self._conf._bucket,
-            local_path=local_path,
-            cos_path=cos_path))
         _md5 = ""
         try:
             _http_header = yaml.safe_load(_http_headers)
@@ -327,6 +334,13 @@ def single_upload(self, local_path, cos_path, _http_headers='{}', **kwargs):
                 logger.info(
                     u"The file on cos is the same as the local file, skip upload")
                 return -2
+        if kwargs['mutex'] is True:
+            if self.check_file_name(local_path, cos_path):
+                logger.error(
+                    u"The file {cos_path} on cos has same name as the local file, skip upload".format(
+                        cos_path=cos_path
+                    ))
+                return -2
         try:
             if len(local_path) == 0:
                 data = ""
@@ -340,7 +354,7 @@ def single_upload(self, local_path, cos_path, _http_headers='{}', **kwargs):
         for j in range(self._retry):
             try:
                 if j > 0:
-                    logger.info(u"Retry to upload {local_path}   =>   cos://{bucket}/{cos_path}".format(
+                    logger.warn(u"Retry to upload {local_path}   =>   cos://{bucket}/{cos_path}".format(
                         bucket=self._conf._bucket,
                         local_path=local_path,
                         cos_path=cos_path))
@@ -349,6 +363,10 @@ def single_upload(self, local_path, cos_path, _http_headers='{}', **kwargs):
                 rt = self._session.put(url=url,
                                        auth=CosS3Auth(self._conf), data=data, headers=http_header)
                 if rt.status_code == 200:
+                    logger.info(u"Upload {local_path}   =>   cos://{bucket}/{cos_path} success".format(
+                        bucket=self._conf._bucket,
+                        local_path=local_path,
+                        cos_path=cos_path))
                     return 0
                 else:
                     time.sleep(2**j)
@@ -502,10 +520,6 @@ def complete_multiupload():
                 logger.warn(e)
                 return -1
 
-        logger.info(u"Upload {local_path}   =>   cos://{bucket}/{cos_path}".format(
-            bucket=self._conf._bucket,
-            local_path=local_path,
-            cos_path=cos_path))
         _md5 = ""
         try:
             _http_header = yaml.safe_load(_http_headers)
@@ -531,6 +545,13 @@ def complete_multiupload():
                 logger.info(
                     u"The file on cos is the same as the local file, skip upload")
                 return -2
+        if kwargs['mutex'] is True:
+            if self.check_file_name(local_path, cos_path):
+                logger.error(
+                    u"The file {cos_path} on cos has same name as the local file, skip upload".format(
+                        cos_path=cos_path
+                    ))
+                return -2
         rt = init_multiupload()
         if 0 == rt:
             logger.debug(u"Init multipart upload ok")
@@ -546,7 +567,10 @@ def complete_multiupload():
             return -1
         rt = complete_multiupload()
         if 0 == rt:
-            logger.debug(u"Complete multipart upload ok")
+            logger.info(u"Upload {local_path}(multipart)   =>   cos://{bucket}/{cos_path} success".format(
+                bucket=self._conf._bucket,
+                local_path=local_path,
+                cos_path=cos_path))
         else:
             logger.warn(u"Complete multipart upload failed")
             return -1
@@ -2040,6 +2064,7 @@ def probe(self, **kwargs):
             kw = {
                 "skipmd5": True,
                 "sync": False,
+                "mutex": False,
                 "force": True,
                 "ignore": ""}
             time_start = time.time()

coscmd/cos_cmd.py → xcoscmd/cos_cmd.py

@@ -9,16 +9,16 @@
 import json
 import requests
 from threading import Thread
-from coscmd import cos_global
+from xcoscmd import cos_global
 
 if sys.version > '3':
-    from coscmd.cos_client import CoscmdConfig, CosS3Client
-    from coscmd.cos_global import Version
+    from xcoscmd.cos_client import CoscmdConfig, CosS3Client
+    from xcoscmd.cos_global import Version
 else:
     from cos_client import CoscmdConfig, CosS3Client
     from cos_global import Version
 
-logger = logging.getLogger("coscmd")
+logger = logging.getLogger("xcoscmd")
 
 fs_coding = sys.getfilesystemencoding()
 
@@ -95,7 +95,7 @@ def load_conf():
 
     conf_path = os.path.expanduser(config_path)
     if not os.path.exists(conf_path):
-        logger.warn("{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!".format(conf=to_printable_str(conf_path)))
+        logger.warn("{conf} couldn't be found, please use \'xcoscmd config -h\' to learn how to config xcoscmd!".format(conf=to_printable_str(conf_path)))
         raise IOError
     else:
         logger.debug('{conf} is found'.format(conf=to_printable_str(conf_path)))
@@ -218,6 +218,7 @@ def upload(args):
                 args.cos_path = args.cos_path[1:]
             kwargs = {}
             kwargs['sync'] = args.sync
+            kwargs['mutex'] = args.mutex
             kwargs['skipmd5'] = args.skipmd5
             kwargs['ignore'] = args.ignore.split(',')
             kwargs['force'] = args.force
@@ -639,15 +640,15 @@ def get_version():
 
 def version_check():
     try:
-        ret = requests.get("https://pypi.org/pypi/coscmd/json").content
+        ret = requests.get("https://pypi.org/pypi/xcoscmd/json").content
         res_json = json.loads(ret)
         latest_version = res_json["info"]["version"]
         lat_spl = latest_version.split('.')
         cur_spl = cos_global.Version.split('.')
         if cur_spl[0] < lat_spl[0] or cur_spl[1] < lat_spl[1] or cur_spl[2] < lat_spl[2]:
-            logger.info("The current version of coscmd is {v1} \
+            logger.info("The current version of xcoscmd is {v1} \
 and the latest version is {v2}. It is recommended \
-to upgrade coscmd with the command'pip install coscmd -U'.".format(v1=cos_global.Version, v2=latest_version))
+to upgrade xcoscmd with the command'pip install xcoscmd -U'.".format(v1=cos_global.Version, v2=latest_version))
     except Exception as e:
         logger.debug(e)
 
@@ -656,8 +657,8 @@ def command_thread():
     global res
     res = -1
     desc = """an easy-to-use but powerful command-line tool.
-              try \'coscmd -h\' to get more informations.
-              try \'coscmd sub-command -h\' to learn all command usage, likes \'coscmd upload -h\'"""
+              try \'xcoscmd -h\' to get more informations.
+              try \'xcoscmd sub-command -h\' to learn all command usage, likes \'xcoscmd upload -h\'"""
     parser = ArgumentParser(description=desc)
     parser.add_argument('-d', '--debug', help="Debug mode", action="store_true", default=False)
     parser.add_argument('-b', '--bucket', help="Specify bucket", type=str, default="")
@@ -690,6 +691,7 @@ def command_thread():
     parser_upload.add_argument('-r', '--recursive', help="Upload recursively when upload directory", action="store_true", default=False)
     parser_upload.add_argument('-H', '--headers', help="Specify HTTP headers", type=str, default='{}')
     parser_upload.add_argument('-s', '--sync', help="Upload and skip the same file", action="store_true", default=False)
+    parser_upload.add_argument('-m', '--mutex', help="Upload and skip the file has same name", action="store_true", default=False)
     parser_upload.add_argument('-f', '--force', help="upload without history breakpoint", action="store_true", default=False)
     parser_upload.add_argument('--ignore', help='Specify ignored rules, separated by commas; Example: *.txt,*.docx,*.ppt', type=str, default="")
     parser_upload.add_argument('--skipmd5', help='Upload without x-cos-meta-md5', action="store_true", default=False)
@@ -801,7 +803,7 @@ def command_thread():
 
     args = parser.parse_args()
 
-    logger = logging.getLogger('coscmd')
+    logger = logging.getLogger('xcoscmd')
     logger.setLevel(logging.INFO)
     console = logging.StreamHandler()
     console.setLevel(logging.INFO)
@@ -811,7 +813,7 @@ def command_thread():
     handler = RotatingFileHandler(os.path.expanduser(args.log_path), maxBytes=128*1024*1024, backupCount=1)
     handler.setFormatter(logging.Formatter('%(asctime)s - [%(levelname)s]:  %(message)s'))
     logger.addHandler(handler)
-    logging.getLogger('coscmd').addHandler(console)
+    logging.getLogger('xcoscmd').addHandler(console)
     global pre_appid, pre_bucket, pre_region, config_path
     config_path = args.config_path
     pre_bucket = args.bucket