Merge pull request #674 from jjjake/s3-check-limit-errors

Fixes #673, IA-S3 check_limit error message bug.

HISTORY.rst

@@ -3,6 +3,13 @@
 Release History
 ---------------
 
+5.2.0 (?)
++++++++++
+
+**Bugfixes**
+
+- Fixed bug where failed requests to IA-S3 check_limit API would be treated as a 503 slowdown error.
+
 5.1.0 (2025-01-07)
 ++++++++++++++++++
 

internetarchive/__version__.py

@@ -1 +1 @@
-__version__ = '5.1.0'
+__version__ = '5.2.0.dev1'

internetarchive/item.py

@@ -889,7 +889,7 @@ def remove_from_simplelist(self, parent, list) -> Response:
         r = self.session.post(self.urls.metadata, data=data)  # type: ignore
         return r
 
-    def upload_file(self, body,
+    def upload_file(self, body,  # noqa: PLR0915; TODO: Refactor this method to reduce complexity
                     key: str | None = None,
                     metadata: Mapping | None = None,
                     file_metadata: Mapping | None = None,
@@ -1079,12 +1079,23 @@ def _build_request():
             return prepared_request
         else:
             try:
+                first_try = True
                 while True:
                     error_msg = ('s3 is overloaded, sleeping for '
                                  f'{retries_sleep} seconds and retrying. '
                                  f'{retries} retries left.')
-                    if retries > 0:
-                        if self.session.s3_is_overloaded(access_key=access_key):
+                    if retries > 0 and not first_try:
+                        try:
+                            overloaded = self.session.s3_is_overloaded(
+                                    access_key=access_key)
+                        except Exception as e:
+                            error_msg = ('error checking if s3 is overloaded via '
+                                         's3.us.archive.org?check_limit=1, '
+                                         f'exception raised: "{e}". '
+                                         f'sleeping for {retries_sleep} seconds and '
+                                         f'retrying. {retries} retries left.')
+                            overloaded = True
+                        if overloaded:
                             sleep(retries_sleep)
                             log.info(error_msg)
                             if verbose:
@@ -1112,6 +1123,7 @@ def _build_request():
                             print(f' warning: {error_msg}', file=sys.stderr)
                         sleep(retries_sleep)
                         retries -= 1
+                        first_try = False
                         continue
                     else:
                         if response.status_code == 503:

internetarchive/session.py

@@ -333,10 +333,7 @@ def s3_is_overloaded(self, identifier=None, access_key=None, request_kwargs=None
             'accesskey': access_key,
             'bucket': identifier,
         }
-        try:
-            r = self.get(u, params=p, **request_kwargs)
-        except Exception:
-            return True
+        r = self.get(u, params=p, **request_kwargs)
         try:
             j = r.json()
         except ValueError: