Merge pull request jjjake#482 from JustAnotherArchivist/ia-upload-kee…

…p-directories

Add --keep-directories option to `ia upload` which preserves the supplied local path

HISTORY.rst

@@ -14,6 +14,8 @@ Unreleased
 
 - ``page`` parameter is no longer required if ``rows`` parameter is specified in search requests.
 - advancedsearch.php endpoint now supports IAS3 authorization.
+- ``ia upload`` now has a ``--keep-directories`` option to use the full local file paths as the
+  remote name.
 
 **Bugfixes**
 

internetarchive/cli/ia_upload.py

@@ -59,6 +59,8 @@
     --no-backup                          Turn off archive.org backups. Clobbered files
                                          will not be saved to history/files/$key.~N~
                                          [default: True].
+    --keep-directories                   Keep directories in the supplied file paths for
+                                         the remote filename. [default: False]
 """
 import os
 import sys
@@ -235,6 +237,8 @@ def read():
             local_file = local_file[0]
         if args['--remote-name']:
             files = {args['--remote-name']: local_file}
+        elif args['--keep-directories']:
+            files = {f: f for f in local_file}
         else:
             files = local_file
 
@@ -264,6 +268,8 @@ def read():
                 if row.get('REMOTE_NAME'):
                     local_file = {row['REMOTE_NAME']: row['file']}
                     del row['REMOTE_NAME']
+                elif args['--keep-directories']:
+                    local_file = {row['file']: row['file']}
                 else:
                     local_file = row['file']
                 identifier = row.get('item', row.get('identifier'))

tests/cli/test_ia_upload.py

@@ -2,6 +2,7 @@
 import responses
 from contextlib import contextmanager
 from io import StringIO
+import os
 import sys
 
 from internetarchive.utils import json
@@ -332,3 +333,49 @@ def insert_test_txt(body):
                 expected_exit_code=1)
 
     assert f'test.txt already exists: {PROTOCOL}//s3.us.archive.org/nasa/test.txt' in caplog.text
+
+
+def test_ia_upload_keep_directories(tmpdir_ch, caplog):
+    os.mkdir('foo')
+    with open('foo/test.txt', 'w') as fh:
+        fh.write('foo')
+    with open('test.csv', 'w') as fh:
+        fh.write('identifier,file\n')
+        fh.write('nasa,foo/test.txt\n')
+
+    # Default behaviour
+    with IaRequestsMock() as rsps:
+        rsps.add_metadata_mock('nasa')
+        rsps.add(responses.PUT, f'{PROTOCOL}//s3.us.archive.org/nasa/test.txt',
+                 body='',
+                 content_type='text/plain')
+        ia_call(['ia', '--log', 'upload', 'nasa', 'foo/test.txt'])
+    assert f'uploaded test.txt to {PROTOCOL}//s3.us.archive.org/nasa/test.txt' in caplog.text
+    caplog.clear()
+
+    with IaRequestsMock() as rsps:
+        rsps.add_metadata_mock('nasa')
+        rsps.add(responses.PUT, f'{PROTOCOL}//s3.us.archive.org/nasa/test.txt',
+                 body='',
+                 content_type='text/plain')
+        ia_call(['ia', '--log', 'upload', '--spreadsheet', 'test.csv'])
+    assert f'uploaded test.txt to {PROTOCOL}//s3.us.archive.org/nasa/test.txt' in caplog.text
+    caplog.clear()
+
+    # With the option
+    with IaRequestsMock() as rsps:
+        rsps.add_metadata_mock('nasa')
+        rsps.add(responses.PUT, f'{PROTOCOL}//s3.us.archive.org/nasa/foo/test.txt',
+                 body='',
+                 content_type='text/plain')
+        ia_call(['ia', '--log', 'upload', 'nasa', 'foo/test.txt', '--keep-directories'])
+    assert f'uploaded foo/test.txt to {PROTOCOL}//s3.us.archive.org/nasa/foo/test.txt' in caplog.text
+    caplog.clear()
+
+    with IaRequestsMock() as rsps:
+        rsps.add_metadata_mock('nasa')
+        rsps.add(responses.PUT, f'{PROTOCOL}//s3.us.archive.org/nasa/foo/test.txt',
+                 body='',
+                 content_type='text/plain')
+        ia_call(['ia', '--log', 'upload', '--spreadsheet', 'test.csv', '--keep-directories'])
+    assert f'uploaded foo/test.txt to {PROTOCOL}//s3.us.archive.org/nasa/foo/test.txt' in caplog.text