diff --git a/tests/framework_starlette/test_application.py b/tests/framework_starlette/test_application.py
index edac4e9f6c..61b0003114 100644
--- a/tests/framework_starlette/test_application.py
+++ b/tests/framework_starlette/test_application.py
@@ -12,8 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
+
 import pytest
 import starlette
+
+from newrelic.common.object_names import callable_name
 from testing_support.fixtures import (
     override_ignore_status_codes,
     validate_transaction_errors,
@@ -88,17 +92,32 @@ def _test():
 
 
 @pytest.mark.parametrize("app_name", ("no_error_handler",))
-@validate_transaction_metrics(
-    "_test_application:middleware_factory.<locals>.middleware",
-    scoped_metrics=[
-        ("Function/_test_application:middleware_factory.<locals>.middleware", 1)
-    ],
-    rollup_metrics=[FRAMEWORK_METRIC],
-)
 def test_exception_in_middleware(target_application, app_name):
     app = target_application[app_name]
-    with pytest.raises(ValueError):
-        app.get("/crash_me_now")
+
+    from starlette import __version__ as version
+    starlette_version = tuple(int(v) for v in version.split("."))
+
+    # Starlette >=0.15 raises an exception group instead of reraising the ValueError
+    # This only occurs on Python versions >=3.8
+    if sys.version_info[0:2] > (3, 7) and starlette_version >= (0, 15, 0):
+        from anyio._backends._asyncio import ExceptionGroup
+        exc_type = ExceptionGroup
+    else:
+        exc_type = ValueError
+
+    @validate_transaction_metrics(
+        "_test_application:middleware_factory.<locals>.middleware",
+        scoped_metrics=[
+            ("Function/_test_application:middleware_factory.<locals>.middleware", 1)
+        ],
+        rollup_metrics=[FRAMEWORK_METRIC],
+    )
+    @validate_transaction_errors(errors=[callable_name(exc_type)])
+    def _test():
+        with pytest.raises(exc_type):  # Later versions of starlette
+            app.get("/crash_me_now")
+    _test()
 
 
 @pytest.mark.parametrize(
diff --git a/tox.ini b/tox.ini
index 9d522078e7..87d211ed81 100644
--- a/tox.ini
+++ b/tox.ini
@@ -120,7 +120,7 @@ envlist =
     python-framework_pyramid-{pypy3,py36,py37,py38,py39}-Pyramidmaster,
     python-framework_sanic-{py38,pypy3}-sanic{190301,1906,1812,1912,200904,210300}
     python-framework_sanic-{py36,py37,py38,pypy3}-saniclatest
-    python-framework_starlette-{py36,py37,py38,py39,pypy3},
+    python-framework_starlette-{py36,py37,py38,py39,pypy3}-starlette{0014,latest},
     libcurl-framework_tornado-{py36,py37,py38,py39,pypy3}-tornado0600,
     libcurl-framework_tornado-{py36,py37,py38,py39,pypy3}-tornadomaster,
     rabbitmq-messagebroker_pika-{py27,py36,py37,py38,py39,pypy,pypy3}-pika{0.13,latest},
@@ -254,7 +254,8 @@ deps =
     framework_sanic-sanic210300: sanic<21.3.1
     framework_sanic-saniclatest: sanic
     framework_sanic-sanic{1812,190301,1906}: aiohttp
-    framework_starlette: starlette
+    framework_starlette-starlette0014: starlette<0.15
+    framework_starlette-starlettelatest: starlette
     framework_tornado: pycurl
     framework_tornado-tornado0600: tornado<6.1
     framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip