Fix botocore tests for botocore v1.28.1+ (#675)

* Fix botocore tests for botocore v1.28.1+ Co-authored-by: Timothy Pansino <[email protected]> * Fix boto3 tests for botocore v1.28.1+ Co-authored-by: Timothy Pansino <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> Co-authored-by: Uma Annamalai <[email protected]> * Fix boto3 tests for python 2.7 Co-authored-by: Timothy Pansino <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> Co-authored-by: Uma Annamalai <[email protected]> Co-authored-by: Timothy Pansino <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> Co-authored-by: Uma Annamalai <[email protected]> Co-authored-by: Timothy Pansino <[email protected]>
newrelic · Oct 26, 2022 · 7b82cb9 · 7b82cb9
1 parent a66a33a
commit 7b82cb9
Show file tree

Hide file tree

Showing 3 changed files with 123 additions and 118 deletions.
diff --git a/tests/external_boto3/test_boto3_s3.py b/tests/external_boto3/test_boto3_s3.py
@@ -18,106 +18,106 @@
 import boto3
 import botocore
 import moto
+from testing_support.fixtures import (
+    override_application_settings,
+    validate_transaction_metrics,
+)
+from testing_support.validators.validate_span_events import validate_span_events
 
 from newrelic.api.background_task import background_task
-from testing_support.fixtures import (validate_transaction_metrics,
-        override_application_settings)
-from testing_support.validators.validate_span_events import (
-        validate_span_events)
 
-MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3])
+MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3])
 
 # patch earlier versions of moto to support py37
 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1):
     import re
+
     moto.packages.responses.responses.re._pattern_type = re.Pattern
 
-AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY'
-AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY'
-AWS_REGION_NAME = 'us-west-2'
+AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY"
+AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY"  # nosec
+AWS_REGION_NAME = "us-west-2"
+
+TEST_BUCKET = "python-agent-test-%s" % uuid.uuid4()
+
+BOTOCORE_VERSION = tuple(map(int, botocore.__version__.split(".")))
 
-TEST_BUCKET = 'python-agent-test-%s' % uuid.uuid4()
 
-BOTOCORE_VERSION = tuple(map(int, botocore.__version__.split('.')))
 if BOTOCORE_VERSION < (1, 7, 41):
-    S3_URL = 's3-us-west-2.amazonaws.com'
+    S3_URL = "s3-us-west-2.amazonaws.com"
+    EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET)
+    EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world"
+elif BOTOCORE_VERSION < (1, 28):
+    S3_URL = "s3.us-west-2.amazonaws.com"
+    EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET)
+    EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world"
 else:
-    S3_URL = 's3.us-west-2.amazonaws.com'
+    S3_URL = "%s.s3.us-west-2.amazonaws.com" % TEST_BUCKET
+    EXPECTED_BUCKET_URL = "https://%s/" % S3_URL
+    EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "hello_world"
 
-expected_http_url = 'https://%s/%s' % (S3_URL, TEST_BUCKET)
 
 _s3_scoped_metrics = [
-    ('External/%s/botocore/GET' % S3_URL, 2),
-    ('External/%s/botocore/PUT' % S3_URL, 2),
-    ('External/%s/botocore/DELETE' % S3_URL, 2),
+    ("External/%s/botocore/GET" % S3_URL, 2),
+    ("External/%s/botocore/PUT" % S3_URL, 2),
+    ("External/%s/botocore/DELETE" % S3_URL, 2),
 ]
 
 _s3_rollup_metrics = [
-    ('External/all', 6),
-    ('External/allOther', 6),
-    ('External/%s/all' % S3_URL, 6),
-    ('External/%s/botocore/GET' % S3_URL, 2),
-    ('External/%s/botocore/PUT' % S3_URL, 2),
-    ('External/%s/botocore/DELETE' % S3_URL, 2),
+    ("External/all", 6),
+    ("External/allOther", 6),
+    ("External/%s/all" % S3_URL, 6),
+    ("External/%s/botocore/GET" % S3_URL, 2),
+    ("External/%s/botocore/PUT" % S3_URL, 2),
+    ("External/%s/botocore/DELETE" % S3_URL, 2),
 ]
 
 
-@override_application_settings({'distributed_tracing.enabled': True})
-@validate_span_events(exact_agents={'aws.operation': 'CreateBucket'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'PutObject'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'ListObjects'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'GetObject'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'DeleteObject'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'DeleteBucket'}, count=1)
-@validate_span_events(
-        exact_agents={'http.url': expected_http_url}, count=3)
-@validate_span_events(
-        exact_agents={'http.url': expected_http_url + '/hello_world'}, count=3)
+@override_application_settings({"distributed_tracing.enabled": True})
+@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "GetObject"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "DeleteObject"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "DeleteBucket"}, count=1)
+@validate_span_events(exact_agents={"http.url": EXPECTED_BUCKET_URL}, count=3)
+@validate_span_events(exact_agents={"http.url": EXPECTED_KEY_URL}, count=3)
 @validate_transaction_metrics(
-        'test_boto3_s3:test_s3',
-        scoped_metrics=_s3_scoped_metrics,
-        rollup_metrics=_s3_rollup_metrics,
-        background_task=True)
+    "test_boto3_s3:test_s3", scoped_metrics=_s3_scoped_metrics, rollup_metrics=_s3_rollup_metrics, background_task=True
+)
 @background_task()
 @moto.mock_s3
 def test_s3():
     client = boto3.client(
-            's3',
-            aws_access_key_id=AWS_ACCESS_KEY_ID,
-            aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
-            region_name=AWS_REGION_NAME,
+        "s3",
+        aws_access_key_id=AWS_ACCESS_KEY_ID,
+        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+        region_name=AWS_REGION_NAME,
     )
 
     # Create bucket
-    resp = client.create_bucket(
-            Bucket=TEST_BUCKET,
-            CreateBucketConfiguration={'LocationConstraint': AWS_REGION_NAME}
-    )
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
+    resp = client.create_bucket(Bucket=TEST_BUCKET, CreateBucketConfiguration={"LocationConstraint": AWS_REGION_NAME})
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
 
     # Put object
-    resp = client.put_object(
-            Bucket=TEST_BUCKET,
-            Key='hello_world',
-            Body=b'hello_world_content'
-    )
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
+    resp = client.put_object(Bucket=TEST_BUCKET, Key="hello_world", Body=b"hello_world_content")
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
 
     # List bucket
     resp = client.list_objects(Bucket=TEST_BUCKET)
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
-    assert len(resp['Contents']) == 1
-    assert resp['Contents'][0]['Key'] == 'hello_world'
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
+    assert len(resp["Contents"]) == 1
+    assert resp["Contents"][0]["Key"] == "hello_world"
 
     # Get object
-    resp = client.get_object(Bucket=TEST_BUCKET, Key='hello_world')
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
-    assert resp['Body'].read() == b'hello_world_content'
+    resp = client.get_object(Bucket=TEST_BUCKET, Key="hello_world")
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
+    assert resp["Body"].read() == b"hello_world_content"
 
     # Delete object
-    resp = client.delete_object(Bucket=TEST_BUCKET, Key='hello_world')
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
+    resp = client.delete_object(Bucket=TEST_BUCKET, Key="hello_world")
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204
 
     # Delete bucket
     resp = client.delete_bucket(Bucket=TEST_BUCKET)
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204
diff --git a/tests/external_botocore/test_botocore_s3.py b/tests/external_botocore/test_botocore_s3.py
@@ -15,101 +15,104 @@
 import sys
 import uuid
 
+import botocore
 import botocore.session
 import moto
+from testing_support.fixtures import (
+    override_application_settings,
+    validate_transaction_metrics,
+)
+from testing_support.validators.validate_span_events import validate_span_events
 
 from newrelic.api.background_task import background_task
-from testing_support.fixtures import (validate_transaction_metrics,
-        override_application_settings)
-from testing_support.validators.validate_span_events import (
-        validate_span_events)
 
-MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3])
+MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3])
+BOTOCORE_VERSION = tuple(int(v) for v in botocore.__version__.split(".")[:3])
+
 
 # patch earlier versions of moto to support py37
 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1):
     import re
+
     moto.packages.responses.responses.re._pattern_type = re.Pattern
 
-AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY'
-AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY'
-AWS_REGION = 'us-east-1'
+AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY"
+AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY"  # nosec
+AWS_REGION = "us-east-1"
 
-TEST_BUCKET = 'python-agent-test-%s' % uuid.uuid4()
-S3_URL = 's3.amazonaws.com'
-expected_http_url = 'https://%s/%s' % (S3_URL, TEST_BUCKET)
+TEST_BUCKET = "python-agent-test-%s" % uuid.uuid4()
+if BOTOCORE_VERSION >= (1, 28):
+    S3_URL = "%s.s3.amazonaws.com" % TEST_BUCKET
+    EXPECTED_BUCKET_URL = "https://%s/" % S3_URL
+    EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "hello_world"
+else:
+    S3_URL = "s3.amazonaws.com"
+    EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET)
+    EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world"
 
 
 _s3_scoped_metrics = [
-    ('External/s3.amazonaws.com/botocore/GET', 2),
-    ('External/s3.amazonaws.com/botocore/PUT', 2),
-    ('External/s3.amazonaws.com/botocore/DELETE', 2),
+    ("External/%s/botocore/GET" % S3_URL, 2),
+    ("External/%s/botocore/PUT" % S3_URL, 2),
+    ("External/%s/botocore/DELETE" % S3_URL, 2),
 ]
 
 _s3_rollup_metrics = [
-    ('External/all', 6),
-    ('External/allOther', 6),
-    ('External/s3.amazonaws.com/all', 6),
-    ('External/s3.amazonaws.com/botocore/GET', 2),
-    ('External/s3.amazonaws.com/botocore/PUT', 2),
-    ('External/s3.amazonaws.com/botocore/DELETE', 2),
+    ("External/all", 6),
+    ("External/allOther", 6),
+    ("External/%s/all" % S3_URL, 6),
+    ("External/%s/botocore/GET" % S3_URL, 2),
+    ("External/%s/botocore/PUT" % S3_URL, 2),
+    ("External/%s/botocore/DELETE" % S3_URL, 2),
 ]
 
 
-@override_application_settings({'distributed_tracing.enabled': True})
-@validate_span_events(exact_agents={'aws.operation': 'CreateBucket'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'PutObject'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'ListObjects'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'GetObject'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'DeleteObject'}, count=1)
-@validate_span_events(exact_agents={'aws.operation': 'DeleteBucket'}, count=1)
-@validate_span_events(
-        exact_agents={'http.url': expected_http_url}, count=3)
-@validate_span_events(
-        exact_agents={'http.url': expected_http_url + '/hello_world'}, count=3)
+@override_application_settings({"distributed_tracing.enabled": True})
+@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "GetObject"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "DeleteObject"}, count=1)
+@validate_span_events(exact_agents={"aws.operation": "DeleteBucket"}, count=1)
+@validate_span_events(exact_agents={"http.url": EXPECTED_BUCKET_URL}, count=3)
+@validate_span_events(exact_agents={"http.url": EXPECTED_KEY_URL}, count=3)
 @validate_transaction_metrics(
-        'test_botocore_s3:test_s3',
-        scoped_metrics=_s3_scoped_metrics,
-        rollup_metrics=_s3_rollup_metrics,
-        background_task=True)
+    "test_botocore_s3:test_s3",
+    scoped_metrics=_s3_scoped_metrics,
+    rollup_metrics=_s3_rollup_metrics,
+    background_task=True,
+)
 @background_task()
 @moto.mock_s3
 def test_s3():
     session = botocore.session.get_session()
     client = session.create_client(
-            's3',
-            region_name=AWS_REGION,
-            aws_access_key_id=AWS_ACCESS_KEY_ID,
-            aws_secret_access_key=AWS_SECRET_ACCESS_KEY
+        "s3", region_name=AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY
     )
 
     # Create bucket
     resp = client.create_bucket(Bucket=TEST_BUCKET)
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
 
     # Put object
-    resp = client.put_object(
-            Bucket=TEST_BUCKET,
-            Key='hello_world',
-            Body=b'hello_world_content'
-    )
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
+    resp = client.put_object(Bucket=TEST_BUCKET, Key="hello_world", Body=b"hello_world_content")
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
 
     # List bucket
     resp = client.list_objects(Bucket=TEST_BUCKET)
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
-    assert len(resp['Contents']) == 1
-    assert resp['Contents'][0]['Key'] == 'hello_world'
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
+    assert len(resp["Contents"]) == 1
+    assert resp["Contents"][0]["Key"] == "hello_world"
 
     # Get object
-    resp = client.get_object(Bucket=TEST_BUCKET, Key='hello_world')
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 200
-    assert resp['Body'].read() == b'hello_world_content'
+    resp = client.get_object(Bucket=TEST_BUCKET, Key="hello_world")
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
+    assert resp["Body"].read() == b"hello_world_content"
 
     # Delete object
-    resp = client.delete_object(Bucket=TEST_BUCKET, Key='hello_world')
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
+    resp = client.delete_object(Bucket=TEST_BUCKET, Key="hello_world")
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204
 
     # Delete bucket
     resp = client.delete_bucket(Bucket=TEST_BUCKET)
-    assert resp['ResponseMetadata']['HTTPStatusCode'] == 204
+    assert resp["ResponseMetadata"]["HTTPStatusCode"] == 204
diff --git a/tox.ini b/tox.ini
@@ -95,7 +95,8 @@ envlist =
     python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37},
     memcached-datastore_umemcache-{py27,pypy},
     python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01,
-    python-external_botocore-{py27,py37,py38,py39,py310,py311},
+    python-external_botocore-{py27,py37,py38,py39,py310,py311}-botocorelatest,
+    python-external_botocore-py310-botocore0125,
     python-external_feedparser-py27-feedparser{05,06},
     python-external_http-{py27,py37,py38,py39,py310,py311,pypy},
     python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy,pypy37},
@@ -259,7 +260,8 @@ deps =
     external_boto3-boto01: boto3<2.0
     external_boto3-boto01: moto<2.0
     external_boto3-py27: rsa<4.7.1
-    external_botocore: botocore
+    external_botocore-botocorelatest: botocore
+    external_botocore-botocore0125: botocore<1.26
     external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0
     external_botocore-py27: rsa<4.7.1
     external_botocore-py27: moto[awslambda,ec2,iam]<2.0