From 367e57b85faa126e5160eecd781fc6344a344014 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 31 Mar 2022 13:28:41 -0700 Subject: [PATCH] GraphQL Server Testing (#506) * Finalize Sanic testing * Fix flask framework details with callable * Parametrized testing for graphql-server * Add middleware tests to graphqlserver * Reenable code coverage * [Mega-Linter] Apply linters fixes * Bump Tests * Fix nonlocal binding issues in python 2 Co-authored-by: TimPansino --- newrelic/api/wsgi_application.py | 164 +++--- newrelic/hooks/framework_flask.py | 55 +- .../component_graphqlserver/_test_graphql.py | 89 +++- tests/component_graphqlserver/test_graphql.py | 472 +++++++++++++++++- .../framework_graphql/_target_application.py | 3 + tox.ini | 5 +- 6 files changed, 624 insertions(+), 164 deletions(-) diff --git a/newrelic/api/wsgi_application.py b/newrelic/api/wsgi_application.py index 840c094a41..bc996b59d9 100644 --- a/newrelic/api/wsgi_application.py +++ b/newrelic/api/wsgi_application.py @@ -12,28 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools +import logging import sys import time -import logging -import functools from newrelic.api.application import application_instance -from newrelic.api.transaction import current_transaction -from newrelic.api.time_trace import notice_error -from newrelic.api.web_transaction import WSGIWebTransaction from newrelic.api.function_trace import FunctionTrace from newrelic.api.html_insertion import insert_html_snippet, verify_body_exists - +from newrelic.api.time_trace import notice_error +from newrelic.api.transaction import current_transaction +from newrelic.api.web_transaction import WSGIWebTransaction from newrelic.common.object_names import callable_name -from newrelic.common.object_wrapper import wrap_object, FunctionWrapper - +from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.packages import six _logger = logging.getLogger(__name__) class _WSGIApplicationIterable(object): - def __init__(self, transaction, generator): self.transaction = transaction self.generator = generator @@ -68,8 +65,7 @@ def start_trace(self): self.transaction._sent_start = time.time() if not self.response_trace: - self.response_trace = FunctionTrace( - name='Response', group='Python/WSGI') + self.response_trace = FunctionTrace(name="Response", group="Python/WSGI") self.response_trace.__enter__() def close(self): @@ -81,13 +77,12 @@ def close(self): self.response_trace = None try: - with FunctionTrace( - name='Finalize', group='Python/WSGI'): + with FunctionTrace(name="Finalize", group="Python/WSGI"): if isinstance(self.generator, _WSGIApplicationMiddleware): self.generator.close() - elif hasattr(self.generator, 'close'): + elif hasattr(self.generator, "close"): name = callable_name(self.generator.close) with FunctionTrace(name): self.generator.close() @@ -105,7 +100,6 @@ def close(self): class _WSGIInputWrapper(object): - def __init__(self, transaction, input): self.__transaction = transaction self.__input = input @@ -114,7 +108,7 @@ def __getattr__(self, name): return getattr(self.__input, name) def close(self): - if hasattr(self.__input, 'close'): + if hasattr(self.__input, "close"): self.__input.close() def read(self, *args, **kwargs): @@ -204,8 +198,7 @@ def __init__(self, application, environ, start_response, transaction): # Grab the iterable returned by the wrapped WSGI # application. - self.iterable = self.application(self.request_environ, - self.start_response) + self.iterable = self.application(self.request_environ, self.start_response) def process_data(self, data): # If this is the first data block, then immediately try @@ -217,7 +210,7 @@ def html_to_be_inserted(): header = self.transaction.browser_timing_header() if not header: - return b'' + return b"" footer = self.transaction.browser_timing_footer() @@ -228,10 +221,12 @@ def html_to_be_inserted(): if modified is not None: if self.debug: - _logger.debug('RUM insertion from WSGI middleware ' - 'triggered on first yielded string from ' - 'response. Bytes added was %r.', - len(modified) - len(data)) + _logger.debug( + "RUM insertion from WSGI middleware " + "triggered on first yielded string from " + "response. Bytes added was %r.", + len(modified) - len(data), + ) if self.content_length is not None: length = len(modified) - len(data) @@ -264,7 +259,7 @@ def html_to_be_inserted(): if self.response_data: self.response_data.append(data) - data = b''.join(self.response_data) + data = b"".join(self.response_data) self.response_data = [] # Perform the insertion of the HTML. This should always @@ -276,10 +271,12 @@ def html_to_be_inserted(): if modified is not None: if self.debug: - _logger.debug('RUM insertion from WSGI middleware ' - 'triggered on subsequent string yielded from ' - 'response. Bytes added was %r.', - len(modified) - len(data)) + _logger.debug( + "RUM insertion from WSGI middleware " + "triggered on subsequent string yielded from " + "response. Bytes added was %r.", + len(modified) - len(data), + ) if self.content_length is not None: length = len(modified) - len(data) @@ -297,11 +294,10 @@ def flush_headers(self): # additional data was inserted into the response. if self.content_length is not None: - header = (('Content-Length', str(self.content_length))) + header = ("Content-Length", str(self.content_length)) self.response_headers.append(header) - self.outer_write = self.outer_start_response(self.response_status, - self.response_headers, *self.response_args) + self.outer_write = self.outer_start_response(self.response_status, self.response_headers, *self.response_args) def inner_write(self, data): # If the write() callable is used, we do not attempt to @@ -345,8 +341,7 @@ def start_response(self, status, response_headers, *args): # This is because it can be disabled using an API call. # Also check whether RUM insertion has already occurred. - if (self.transaction.autorum_disabled or - self.transaction.rum_header_generated): + if self.transaction.autorum_disabled or self.transaction.rum_header_generated: self.flush_headers() self.pass_through = True @@ -370,7 +365,7 @@ def start_response(self, status, response_headers, *args): for (name, value) in response_headers: _name = name.lower() - if _name == 'content-length': + if _name == "content-length": try: content_length = int(value) continue @@ -378,13 +373,13 @@ def start_response(self, status, response_headers, *args): except ValueError: pass_through = True - elif _name == 'content-type': + elif _name == "content-type": content_type = value - elif _name == 'content-encoding': + elif _name == "content-encoding": content_encoding = value - elif _name == 'content-disposition': + elif _name == "content-disposition": content_disposition = value headers.append((name, value)) @@ -408,9 +403,7 @@ def should_insert_html(): return False - if (content_disposition is not None and - content_disposition.split(';')[0].strip().lower() == - 'attachment'): + if content_disposition is not None and content_disposition.split(";")[0].strip().lower() == "attachment": return False if content_type is None: @@ -419,7 +412,7 @@ def should_insert_html(): settings = self.transaction.settings allowed_content_type = settings.browser_monitoring.content_type - if content_type.split(';')[0] not in allowed_content_type: + if content_type.split(";")[0] not in allowed_content_type: return False return True @@ -443,7 +436,7 @@ def close(self): # Call close() on the iterable as required by the # WSGI specification. - if hasattr(self.iterable, 'close'): + if hasattr(self.iterable, "close"): name = callable_name(self.iterable.close) with FunctionTrace(name): self.iterable.close() @@ -518,11 +511,27 @@ def __iter__(self): yield data -def WSGIApplicationWrapper(wrapped, application=None, name=None, - group=None, framework=None): +def WSGIApplicationWrapper(wrapped, application=None, name=None, group=None, framework=None): + + # Python 2 does not allow rebinding nonlocal variables, so to fix this + # framework must be stored in list so it can be edited by closure. + _framework = [framework] + + def get_framework(): + """Used to delay imports by passing framework as a callable.""" + framework = _framework[0] + if isinstance(framework, tuple) or framework is None: + return framework + + if callable(framework): + framework = framework() + _framework[0] = framework + + if framework is not None and not isinstance(framework, tuple): + framework = (framework, None) + _framework[0] = framework - if framework is not None and not isinstance(framework, tuple): - framework = (framework, None) + return framework def _nr_wsgi_application_wrapper_(wrapped, instance, args, kwargs): # Check to see if any transaction is present, even an inactive @@ -530,6 +539,7 @@ def _nr_wsgi_application_wrapper_(wrapped, instance, args, kwargs): # stopped already. transaction = current_transaction(active_only=False) + framework = get_framework() if transaction: # If there is any active transaction we will return without @@ -545,8 +555,7 @@ def _nr_wsgi_application_wrapper_(wrapped, instance, args, kwargs): # supportability metrics. if framework: - transaction.add_framework_info( - name=framework[0], version=framework[1]) + transaction.add_framework_info(name=framework[0], version=framework[1]) # Also override the web transaction name to be the name of # the wrapped callable if not explicitly named, and we want @@ -560,9 +569,8 @@ def _nr_wsgi_application_wrapper_(wrapped, instance, args, kwargs): if name is None and settings: if framework is not None: naming_scheme = settings.transaction_name.naming_scheme - if naming_scheme in (None, 'framework'): - transaction.set_transaction_name( - callable_name(wrapped), priority=1) + if naming_scheme in (None, "framework"): + transaction.set_transaction_name(callable_name(wrapped), priority=1) elif name: transaction.set_transaction_name(name, group, priority=1) @@ -580,11 +588,11 @@ def _args(environ, start_response, *args, **kwargs): target_application = application - if 'newrelic.app_name' in environ: - app_name = environ['newrelic.app_name'] + if "newrelic.app_name" in environ: + app_name = environ["newrelic.app_name"] - if ';' in app_name: - app_names = [n.strip() for n in app_name.split(';')] + if ";" in app_name: + app_names = [n.strip() for n in app_name.split(";")] app_name = app_names[0] target_application = application_instance(app_name) for altname in app_names[1:]: @@ -598,7 +606,7 @@ def _args(environ, start_response, *args, **kwargs): # FIXME Should this allow for multiple apps if a string. - if not hasattr(application, 'activate'): + if not hasattr(application, "activate"): target_application = application_instance(application) # Now start recording the actual web transaction. @@ -609,8 +617,7 @@ def _args(environ, start_response, *args, **kwargs): # reporting as supportability metrics. if framework: - transaction.add_framework_info( - name=framework[0], version=framework[1]) + transaction.add_framework_info(name=framework[0], version=framework[1]) # Override the initial web transaction name to be the supplied # name, or the name of the wrapped callable if wanting to use @@ -630,24 +637,20 @@ def _args(environ, start_response, *args, **kwargs): naming_scheme = settings.transaction_name.naming_scheme if framework is not None: - if naming_scheme in (None, 'framework'): - transaction.set_transaction_name( - callable_name(wrapped), priority=1) + if naming_scheme in (None, "framework"): + transaction.set_transaction_name(callable_name(wrapped), priority=1) - elif naming_scheme in ('component', 'framework'): - transaction.set_transaction_name( - callable_name(wrapped), priority=1) + elif naming_scheme in ("component", "framework"): + transaction.set_transaction_name(callable_name(wrapped), priority=1) elif name: transaction.set_transaction_name(name, group, priority=1) def _start_response(status, response_headers, *args): - additional_headers = transaction.process_response( - status, response_headers, *args) + additional_headers = transaction.process_response(status, response_headers, *args) - _write = start_response(status, - response_headers + additional_headers, *args) + _write = start_response(status, response_headers + additional_headers, *args) def write(data): if not transaction._sent_start: @@ -667,17 +670,13 @@ def write(data): # Should always exist, but check as test harnesses may not # have it. - if 'wsgi.input' in environ: - environ['wsgi.input'] = _WSGIInputWrapper(transaction, - environ['wsgi.input']) + if "wsgi.input" in environ: + environ["wsgi.input"] = _WSGIInputWrapper(transaction, environ["wsgi.input"]) - with FunctionTrace( - name='Application', group='Python/WSGI'): + with FunctionTrace(name="Application", group="Python/WSGI"): with FunctionTrace(name=callable_name(wrapped)): - if (settings and settings.browser_monitoring.enabled and - not transaction.autorum_disabled): - result = _WSGIApplicationMiddleware(wrapped, - environ, _start_response, transaction) + if settings and settings.browser_monitoring.enabled and not transaction.autorum_disabled: + result = _WSGIApplicationMiddleware(wrapped, environ, _start_response, transaction) else: result = wrapped(environ, _start_response) @@ -691,11 +690,10 @@ def write(data): def wsgi_application(application=None, name=None, group=None, framework=None): - return functools.partial(WSGIApplicationWrapper, application=application, - name=name, group=group, framework=framework) + return functools.partial( + WSGIApplicationWrapper, application=application, name=name, group=group, framework=framework + ) -def wrap_wsgi_application(module, object_path, application=None, - name=None, group=None, framework=None): - wrap_object(module, object_path, WSGIApplicationWrapper, - (application, name, group, framework)) +def wrap_wsgi_application(module, object_path, application=None, name=None, group=None, framework=None): + wrap_object(module, object_path, WSGIApplicationWrapper, (application, name, group, framework)) diff --git a/newrelic/hooks/framework_flask.py b/newrelic/hooks/framework_flask.py index 4535b3289a..16cdc4eec6 100644 --- a/newrelic/hooks/framework_flask.py +++ b/newrelic/hooks/framework_flask.py @@ -16,17 +16,16 @@ """ -from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.api.function_trace import ( FunctionTrace, - wrap_function_trace, FunctionTraceWrapper, + wrap_function_trace, ) -from newrelic.api.transaction import current_transaction from newrelic.api.time_trace import notice_error - -from newrelic.common.object_wrapper import wrap_function_wrapper, function_wrapper +from newrelic.api.transaction import current_transaction +from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper def framework_details(): @@ -173,9 +172,7 @@ def _bind_params(code_or_exception, f): return wrapped(code_or_exception, f) -def _nr_wrapper_Flask_try_trigger_before_first_request_functions_( - wrapped, instance, args, kwargs -): +def _nr_wrapper_Flask_try_trigger_before_first_request_functions_(wrapped, instance, args, kwargs): transaction = current_transaction() @@ -266,30 +263,22 @@ def instrument_flask_views(module): def instrument_flask_app(module): - wrap_wsgi_application(module, "Flask.wsgi_app", framework=framework_details()) + wrap_wsgi_application(module, "Flask.wsgi_app", framework=framework_details) - wrap_function_wrapper( - module, "Flask.add_url_rule", _nr_wrapper_Flask_add_url_rule_input_ - ) + wrap_function_wrapper(module, "Flask.add_url_rule", _nr_wrapper_Flask_add_url_rule_input_) if hasattr(module.Flask, "endpoint"): wrap_function_wrapper(module, "Flask.endpoint", _nr_wrapper_Flask_endpoint_) - wrap_function_wrapper( - module, "Flask.handle_http_exception", _nr_wrapper_Flask_handle_http_exception_ - ) + wrap_function_wrapper(module, "Flask.handle_http_exception", _nr_wrapper_Flask_handle_http_exception_) # Use the same wrapper for initial user exception processing and # fallback for unhandled exceptions. if hasattr(module.Flask, "handle_user_exception"): - wrap_function_wrapper( - module, "Flask.handle_user_exception", _nr_wrapper_Flask_handle_exception_ - ) + wrap_function_wrapper(module, "Flask.handle_user_exception", _nr_wrapper_Flask_handle_exception_) - wrap_function_wrapper( - module, "Flask.handle_exception", _nr_wrapper_Flask_handle_exception_ - ) + wrap_function_wrapper(module, "Flask.handle_exception", _nr_wrapper_Flask_handle_exception_) # The _register_error_handler() method was only introduced in # Flask version 0.7.0. @@ -326,27 +315,19 @@ def instrument_flask_app(module): if hasattr(module.Flask, "preprocess_request"): wrap_function_trace(module, "Flask.preprocess_request") - wrap_function_wrapper( - module, "Flask.before_request", _nr_wrapper_Flask_before_request_ - ) + wrap_function_wrapper(module, "Flask.before_request", _nr_wrapper_Flask_before_request_) if hasattr(module.Flask, "process_response"): wrap_function_trace(module, "Flask.process_response") - wrap_function_wrapper( - module, "Flask.after_request", _nr_wrapper_Flask_after_request_ - ) + wrap_function_wrapper(module, "Flask.after_request", _nr_wrapper_Flask_after_request_) if hasattr(module.Flask, "do_teardown_request"): wrap_function_trace(module, "Flask.do_teardown_request") - wrap_function_wrapper( - module, "Flask.teardown_request", _nr_wrapper_Flask_teardown_request_ - ) + wrap_function_wrapper(module, "Flask.teardown_request", _nr_wrapper_Flask_teardown_request_) if hasattr(module.Flask, "do_teardown_appcontext"): wrap_function_trace(module, "Flask.do_teardown_appcontext") - wrap_function_wrapper( - module, "Flask.teardown_appcontext", _nr_wrapper_Flask_teardown_appcontext_ - ) + wrap_function_wrapper(module, "Flask.teardown_appcontext", _nr_wrapper_Flask_teardown_appcontext_) def instrument_flask_templating(module): @@ -448,9 +429,7 @@ def instrument_flask_blueprints(module): wrap_function_wrapper(module, "Blueprint.endpoint", _nr_wrapper_Blueprint_endpoint_) if hasattr(module.Blueprint, "before_request"): - wrap_function_wrapper( - module, "Blueprint.before_request", _nr_wrapper_Blueprint_before_request_ - ) + wrap_function_wrapper(module, "Blueprint.before_request", _nr_wrapper_Blueprint_before_request_) if hasattr(module.Blueprint, "before_app_request"): wrap_function_wrapper( module, @@ -465,9 +444,7 @@ def instrument_flask_blueprints(module): ) if hasattr(module.Blueprint, "after_request"): - wrap_function_wrapper( - module, "Blueprint.after_request", _nr_wrapper_Blueprint_after_request_ - ) + wrap_function_wrapper(module, "Blueprint.after_request", _nr_wrapper_Blueprint_after_request_) if hasattr(module.Blueprint, "after_app_request"): wrap_function_wrapper( module, diff --git a/tests/component_graphqlserver/_test_graphql.py b/tests/component_graphqlserver/_test_graphql.py index b478cf83e1..50b5621f9a 100644 --- a/tests/component_graphqlserver/_test_graphql.py +++ b/tests/component_graphqlserver/_test_graphql.py @@ -12,27 +12,88 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json + +import webtest +from flask import Flask +from framework_graphql._target_application import _target_application as schema +from graphql_server.flask import GraphQLView as FlaskView +from graphql_server.sanic import GraphQLView as SanicView from sanic import Sanic -from graphql_server.sanic import GraphQLView from testing_support.asgi_testing import AsgiTest -from graphql import GraphQLObjectType, GraphQLString, GraphQLSchema, GraphQLField +def set_middlware(middleware, view_middleware): + view_middleware.clear() + if middleware: + try: + view_middleware.extend(middleware) + except TypeError: + view_middleware.append(middleware) -def resolve_hello(root, info): - return "Hello!" -hello_field = GraphQLField(GraphQLString, resolve=resolve_hello) -query = GraphQLObjectType( - name="Query", - fields={ - "hello": hello_field, - }, -) +# Sanic +target_application = dict() -app = Sanic(name="SanicGraphQL") +sanic_app = Sanic(name="SanicGraphQL") +sanic_middleware = [] +sanic_view = SanicView.as_view(schema=schema, middleware=sanic_middleware) routes = [ - app.add_route(GraphQLView.as_view(schema=GraphQLSchema(query=query)), "/graphql"), + sanic_app.add_route(sanic_view, "/graphql"), ] +sanic_app = AsgiTest(sanic_app) + + +def sanic_execute(query, middleware=None): + set_middlware(middleware, sanic_middleware) + response = sanic_app.make_request( + "POST", "/graphql", body=json.dumps({"query": query}), headers={"Content-Type": "application/json"} + ) + body = json.loads(response.body.decode("utf-8")) + + if not isinstance(query, str) or "error" in query: + try: + assert response.status != 200 + except AssertionError: + assert body["errors"] + else: + assert response.status == 200 + assert "errors" not in body or not body["errors"] + + return response + + +target_application["Sanic"] = sanic_execute + +# Flask + +flask_app = Flask("FlaskGraphQL") +flask_middleware = [] +flask_app.add_url_rule("/graphql", view_func=FlaskView.as_view("graphql", schema=schema, middleware=flask_middleware)) +flask_app = webtest.TestApp(flask_app) + + +def flask_execute(query, middleware=None): + if not isinstance(query, str) or "error" in query: + expect_errors = True + else: + expect_errors = False + + set_middlware(middleware, flask_middleware) + response = flask_app.post( + "/graphql", + json.dumps({"query": query}), + headers={"Content-Type": "application/json"}, + expect_errors=expect_errors, + ) + + body = json.loads(response.body.decode("utf-8")) + if expect_errors: + assert body["errors"] + else: + assert "errors" not in body or not body["errors"] + + return response + -target_application = AsgiTest(app) +target_application["Flask"] = flask_execute diff --git a/tests/component_graphqlserver/test_graphql.py b/tests/component_graphqlserver/test_graphql.py index ee3c5445ba..6064bc698f 100644 --- a/tests/component_graphqlserver/test_graphql.py +++ b/tests/component_graphqlserver/test_graphql.py @@ -12,79 +12,499 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json +import importlib import pytest -from testing_support.fixtures import dt_enabled, validate_transaction_metrics +from testing_support.fixtures import ( + dt_enabled, + validate_transaction_errors, + validate_transaction_metrics, +) from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_count import ( validate_transaction_count, ) +from newrelic.common.object_names import callable_name + @pytest.fixture(scope="session") -def target_application(): +def is_graphql_2(): + from graphql import __version__ as version + + major_version = int(version.split(".")[0]) + return major_version == 2 + + +@pytest.fixture(scope="session", params=("Sanic", "Flask")) +def target_application(request): import _test_graphql - return _test_graphql.target_application + framework = request.param + version = importlib.import_module(framework.lower()).__version__ + + return framework, version, _test_graphql.target_application[framework] + + +def example_middleware(next, root, info, **args): # pylint: disable=W0622 + return_value = next(root, info, **args) + return return_value + + +def error_middleware(next, root, info, **args): # pylint: disable=W0622 + raise RuntimeError("Runtime Error!") + + +_runtime_error_name = callable_name(RuntimeError) +_test_runtime_error = [(_runtime_error_name, "Runtime Error!")] +_graphql_base_rollup_metrics = [ + ("GraphQL/all", 1), + ("GraphQL/allWeb", 1), + ("GraphQL/GraphQLServer/all", 1), + ("GraphQL/GraphQLServer/allWeb", 1), +] +_view_metrics = { + "Sanic": "Function/graphql_server.sanic.graphqlview:GraphQLView.post", + "Flask": "Function/graphql_server.flask.graphqlview:graphql", +} + + +def test_basic(target_application): + framework, version, target_application = target_application + from graphql import __version__ as graphql_version + from graphql_server import __version__ as graphql_server_version + + FRAMEWORK_METRICS = [ + ("Python/Framework/GraphQL/%s" % graphql_version, 1), + ("Python/Framework/GraphQLServer/%s" % graphql_server_version, 1), + ("Python/Framework/%s/%s" % (framework, version), 1), + ] + + @validate_transaction_metrics( + "query//hello", + "GraphQL", + rollup_metrics=_graphql_base_rollup_metrics + FRAMEWORK_METRICS, + ) + def _test(): + response = target_application("{ hello }") + + _test() @dt_enabled -def test_graphql_metrics_and_attrs(target_application): +def test_query_and_mutation(target_application): + framework, version, target_application = target_application from graphql import __version__ as graphql_version from graphql_server import __version__ as graphql_server_version - from sanic import __version__ as sanic_version FRAMEWORK_METRICS = [ ("Python/Framework/GraphQL/%s" % graphql_version, 1), ("Python/Framework/GraphQLServer/%s" % graphql_server_version, 1), - ("Python/Framework/Sanic/%s" % sanic_version, 1), + ("Python/Framework/%s/%s" % (framework, version), 1), ] - _test_scoped_metrics = [ - ("GraphQL/resolve/GraphQLServer/hello", 1), - ("GraphQL/operation/GraphQLServer/query//hello", 1), - ("Function/graphql_server.sanic.graphqlview:GraphQLView.post", 1), + _test_query_scoped_metrics = [ + ("GraphQL/resolve/GraphQLServer/storage", 1), + ("GraphQL/operation/GraphQLServer/query//storage", 1), + (_view_metrics[framework], 1), ] - _test_unscoped_metrics = [ + _test_query_unscoped_metrics = [ ("GraphQL/all", 1), ("GraphQL/GraphQLServer/all", 1), ("GraphQL/allWeb", 1), ("GraphQL/GraphQLServer/allWeb", 1), - ] + _test_scoped_metrics + ] + _test_query_scoped_metrics + _test_mutation_scoped_metrics = [ + ("GraphQL/resolve/GraphQLServer/storage_add", 1), + ("GraphQL/operation/GraphQLServer/mutation//storage_add", 1), + (_view_metrics[framework], 1), + ] + _test_mutation_unscoped_metrics = [ + ("GraphQL/all", 1), + ("GraphQL/GraphQLServer/all", 1), + ("GraphQL/allWeb", 1), + ("GraphQL/GraphQLServer/allWeb", 1), + ] + _test_mutation_scoped_metrics + + _expected_mutation_operation_attributes = { + "graphql.operation.type": "mutation", + "graphql.operation.name": "", + "graphql.operation.query": "mutation { storage_add(string: ?) }", + } + _expected_mutation_resolver_attributes = { + "graphql.field.name": "storage_add", + "graphql.field.parentType": "Mutation", + "graphql.field.path": "storage_add", + "graphql.field.returnType": "String", + } _expected_query_operation_attributes = { "graphql.operation.type": "query", "graphql.operation.name": "", - "graphql.operation.query": "{ hello }", + "graphql.operation.query": "query { storage }", } _expected_query_resolver_attributes = { + "graphql.field.name": "storage", + "graphql.field.parentType": "Query", + "graphql.field.path": "storage", + "graphql.field.returnType": "[String]", + } + + def _test(): + @validate_transaction_metrics( + "mutation//storage_add", + "GraphQL", + scoped_metrics=_test_mutation_scoped_metrics, + rollup_metrics=_test_mutation_unscoped_metrics + FRAMEWORK_METRICS, + ) + @validate_span_events(exact_agents=_expected_mutation_operation_attributes) + @validate_span_events(exact_agents=_expected_mutation_resolver_attributes) + def _mutation(): + return target_application('mutation { storage_add(string: "abc") }') + + @validate_transaction_metrics( + "query//storage", + "GraphQL", + scoped_metrics=_test_query_scoped_metrics, + rollup_metrics=_test_query_unscoped_metrics + FRAMEWORK_METRICS, + ) + @validate_span_events(exact_agents=_expected_query_operation_attributes) + @validate_span_events(exact_agents=_expected_query_resolver_attributes) + def _query(): + return target_application("query { storage }") + + response = _mutation() + response = _query() + + # These are separate assertions because pypy stores 'abc' as a unicode string while other Python versions do not + assert "storage" in str(response.body.decode("utf-8")) + assert "abc" in str(response.body.decode("utf-8")) + + _test() + + +@dt_enabled +def test_middleware(target_application): + framework, version, target_application = target_application + _test_middleware_metrics = [ + ("GraphQL/operation/GraphQLServer/query//hello", 1), + ("GraphQL/resolve/GraphQLServer/hello", 1), + ("Function/test_graphql:example_middleware", 1), + ] + + # Base span count 5: Transaction, View, Operation, Middleware, and 1 Resolver + # For Flask, add 9 more for WSGI and framework related spans + span_count = {"Flask": 14, "Sanic": 5} + + @validate_transaction_metrics( + "query//hello", + "GraphQL", + scoped_metrics=_test_middleware_metrics, + rollup_metrics=_test_middleware_metrics + _graphql_base_rollup_metrics, + ) + @validate_span_events(count=span_count[framework]) + def _test(): + response = target_application("{ hello }", middleware=[example_middleware]) + + _test() + + +@dt_enabled +def test_exception_in_middleware(target_application): + framework, version, target_application = target_application + query = "query MyQuery { error_middleware }" + field = "error_middleware" + + # Metrics + _test_exception_scoped_metrics = [ + ("GraphQL/operation/GraphQLServer/query/MyQuery/%s" % field, 1), + ("GraphQL/resolve/GraphQLServer/%s" % field, 1), + ] + _test_exception_rollup_metrics = [ + ("Errors/all", 1), + ("Errors/allWeb", 1), + ("Errors/WebTransaction/GraphQL/test_graphql:error_middleware", 1), + ] + _test_exception_scoped_metrics + + # Attributes + _expected_exception_resolver_attributes = { + "graphql.field.name": field, + "graphql.field.parentType": "Query", + "graphql.field.path": field, + "graphql.field.returnType": "String", + } + _expected_exception_operation_attributes = { + "graphql.operation.type": "query", + "graphql.operation.name": "MyQuery", + "graphql.operation.query": query, + } + + @validate_transaction_metrics( + "test_graphql:error_middleware", + "GraphQL", + scoped_metrics=_test_exception_scoped_metrics, + rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, + ) + @validate_span_events(exact_agents=_expected_exception_operation_attributes) + @validate_span_events(exact_agents=_expected_exception_resolver_attributes) + @validate_transaction_errors(errors=_test_runtime_error) + def _test(): + response = target_application(query, middleware=[error_middleware]) + + _test() + + +@pytest.mark.parametrize("field", ("error", "error_non_null")) +@dt_enabled +def test_exception_in_resolver(target_application, field): + framework, version, target_application = target_application + query = "query MyQuery { %s }" % field + + txn_name = "framework_graphql._target_application:resolve_error" + + # Metrics + _test_exception_scoped_metrics = [ + ("GraphQL/operation/GraphQLServer/query/MyQuery/%s" % field, 1), + ("GraphQL/resolve/GraphQLServer/%s" % field, 1), + ] + _test_exception_rollup_metrics = [ + ("Errors/all", 1), + ("Errors/allWeb", 1), + ("Errors/WebTransaction/GraphQL/%s" % txn_name, 1), + ] + _test_exception_scoped_metrics + + # Attributes + _expected_exception_resolver_attributes = { + "graphql.field.name": field, + "graphql.field.parentType": "Query", + "graphql.field.path": field, + "graphql.field.returnType": "String!" if "non_null" in field else "String", + } + _expected_exception_operation_attributes = { + "graphql.operation.type": "query", + "graphql.operation.name": "MyQuery", + "graphql.operation.query": query, + } + + @validate_transaction_metrics( + txn_name, + "GraphQL", + scoped_metrics=_test_exception_scoped_metrics, + rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, + ) + @validate_span_events(exact_agents=_expected_exception_operation_attributes) + @validate_span_events(exact_agents=_expected_exception_resolver_attributes) + @validate_transaction_errors(errors=_test_runtime_error) + def _test(): + response = target_application(query) + + _test() + + +@dt_enabled +@pytest.mark.parametrize( + "query,exc_class", + [ + ("query MyQuery { error_missing_field }", "GraphQLError"), + ("{ syntax_error ", "graphql.error.syntax_error:GraphQLSyntaxError"), + ], +) +def test_exception_in_validation(target_application, is_graphql_2, query, exc_class): + framework, version, target_application = target_application + if "syntax" in query: + txn_name = "graphql.language.parser:parse" + else: + if is_graphql_2: + txn_name = "graphql.validation.validation:validate" + else: + txn_name = "graphql.validation.validate:validate" + + # Import path differs between versions + if exc_class == "GraphQLError": + from graphql.error import GraphQLError + + exc_class = callable_name(GraphQLError) + + _test_exception_scoped_metrics = [ + ("GraphQL/operation/GraphQLServer///", 1), + ] + _test_exception_rollup_metrics = [ + ("Errors/all", 1), + ("Errors/allWeb", 1), + ("Errors/WebTransaction/GraphQL/%s" % txn_name, 1), + ] + _test_exception_scoped_metrics + + # Attributes + _expected_exception_operation_attributes = { + "graphql.operation.type": "", + "graphql.operation.name": "", + "graphql.operation.query": query, + } + + @validate_transaction_metrics( + txn_name, + "GraphQL", + scoped_metrics=_test_exception_scoped_metrics, + rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, + ) + @validate_span_events(exact_agents=_expected_exception_operation_attributes) + @validate_transaction_errors(errors=[exc_class]) + def _test(): + response = target_application(query) + + _test() + + +@dt_enabled +def test_operation_metrics_and_attrs(target_application): + framework, version, target_application = target_application + operation_metrics = [("GraphQL/operation/GraphQLServer/query/MyQuery/library", 1)] + operation_attrs = { + "graphql.operation.type": "query", + "graphql.operation.name": "MyQuery", + } + + # Base span count 10: Transaction, View, Operation, and 7 Resolvers + # library, library.name, library.book + # library.book.name and library.book.id for each book resolved (in this case 2) + # For Flask, add 9 more for WSGI and framework related spans + span_count = {"Flask": 19, "Sanic": 10} + + @validate_transaction_metrics( + "query/MyQuery/library", + "GraphQL", + scoped_metrics=operation_metrics, + rollup_metrics=operation_metrics + _graphql_base_rollup_metrics, + ) + @validate_span_events(count=span_count[framework]) + @validate_span_events(exact_agents=operation_attrs) + def _test(): + response = target_application("query MyQuery { library(index: 0) { branch, book { id, name } } }") + + _test() + + +@dt_enabled +def test_field_resolver_metrics_and_attrs(target_application): + framework, version, target_application = target_application + field_resolver_metrics = [("GraphQL/resolve/GraphQLServer/hello", 1)] + graphql_attrs = { "graphql.field.name": "hello", "graphql.field.parentType": "Query", "graphql.field.path": "hello", "graphql.field.returnType": "String", } - @validate_span_events(exact_agents=_expected_query_operation_attributes) - @validate_span_events(exact_agents=_expected_query_resolver_attributes) + # Base span count 4: Transaction, View, Operation, and 1 Resolver + # For Flask, add 9 more for WSGI and framework related spans + span_count = {"Flask": 13, "Sanic": 4} + @validate_transaction_metrics( "query//hello", "GraphQL", - scoped_metrics=_test_scoped_metrics, - rollup_metrics=_test_unscoped_metrics + FRAMEWORK_METRICS, + scoped_metrics=field_resolver_metrics, + rollup_metrics=field_resolver_metrics + _graphql_base_rollup_metrics, ) + @validate_span_events(count=span_count[framework]) + @validate_span_events(exact_agents=graphql_attrs) def _test(): - response = target_application.make_request( - "POST", "/graphql", body=json.dumps({"query": "{ hello }"}), headers={"Content-Type": "application/json"} - ) - assert response.status == 200 + response = target_application("{ hello }") assert "Hello!" in response.body.decode("utf-8") _test() +_test_queries = [ + ("{ hello }", "{ hello }"), # Basic query extraction + ("{ error }", "{ error }"), # Extract query on field error + ( + "{ library(index: 0) { branch } }", + "{ library(index: ?) { branch } }", + ), # Integers + ('{ echo(echo: "123") }', "{ echo(echo: ?) }"), # Strings with numerics + ('{ echo(echo: "test") }', "{ echo(echo: ?) }"), # Strings + ('{ TestEcho: echo(echo: "test") }', "{ TestEcho: echo(echo: ?) }"), # Aliases + ('{ TestEcho: echo(echo: "test") }', "{ TestEcho: echo(echo: ?) }"), # Variables + ( # Fragments + '{ ...MyFragment } fragment MyFragment on Query { echo(echo: "test") }', + "{ ...MyFragment } fragment MyFragment on Query { echo(echo: ?) }", + ), +] + + +@dt_enabled +@pytest.mark.parametrize("query,obfuscated", _test_queries) +def test_query_obfuscation(target_application, query, obfuscated): + framework, version, target_application = target_application + graphql_attrs = {"graphql.operation.query": obfuscated} + + if callable(query): + query = query() + + @validate_span_events(exact_agents=graphql_attrs) + def _test(): + response = target_application(query) + + _test() + + +_test_queries = [ + ("{ hello }", "/hello"), # Basic query + ("{ error }", "/error"), # Extract deepest path on field error + ('{ echo(echo: "test") }', "/echo"), # Fields with arguments + ( + "{ library(index: 0) { branch, book { isbn branch } } }", + "/library", + ), # Complex Example, 1 level + ( + "{ library(index: 0) { book { author { first_name }} } }", + "/library.book.author.first_name", + ), # Complex Example, 2 levels + ("{ library(index: 0) { id, book { name } } }", "/library.book.name"), # Filtering + ('{ TestEcho: echo(echo: "test") }', "/echo"), # Aliases + ( + '{ search(contains: "A") { __typename ... on Book { name } } }', + "/search.name", + ), # InlineFragment + ( + '{ hello echo(echo: "test") }', + "", + ), # Multiple root selections. (need to decide on final behavior) + # FragmentSpread + ( + "{ library(index: 0) { book { ...MyFragment } } } fragment MyFragment on Book { name id }", # Fragment filtering + "/library.book.name", + ), + ( + "{ library(index: 0) { book { ...MyFragment } } } fragment MyFragment on Book { author { first_name } }", + "/library.book.author.first_name", + ), + ( + "{ library(index: 0) { book { ...MyFragment } magazine { ...MagFragment } } } fragment MyFragment on Book { author { first_name } } fragment MagFragment on Magazine { name }", + "/library", + ), +] + + +@dt_enabled +@pytest.mark.parametrize("query,expected_path", _test_queries) +def test_deepest_unique_path(target_application, query, expected_path): + framework, version, target_application = target_application + if expected_path == "/error": + txn_name = "framework_graphql._target_application:resolve_error" + else: + txn_name = "query/%s" % expected_path + + @validate_transaction_metrics( + txn_name, + "GraphQL", + ) + def _test(): + response = target_application(query) + + _test() + + @validate_transaction_count(0) def test_ignored_introspection_transactions(target_application): - response = target_application.make_request( - "POST", "/graphql", body=json.dumps({"query": "{ __schema { types { name } } }"}), headers={"Content-Type": "application/json"} - ) - assert response.status == 200 + framework, version, target_application = target_application + response = target_application("{ __schema { types { name } } }") diff --git a/tests/framework_graphql/_target_application.py b/tests/framework_graphql/_target_application.py index ae920ce2e7..7bef5e9754 100644 --- a/tests/framework_graphql/_target_application.py +++ b/tests/framework_graphql/_target_application.py @@ -185,6 +185,7 @@ def resolve_error(root, info): ) error_field = GraphQLField(GraphQLString, resolver=resolve_error) error_non_null_field = GraphQLField(GraphQLNonNull(GraphQLString), resolver=resolve_error) + error_middleware_field = GraphQLField(GraphQLString, resolver=resolve_hello) except TypeError: hello_field = GraphQLField(GraphQLString, resolve=resolve_hello) library_field = GraphQLField( @@ -212,6 +213,7 @@ def resolve_error(root, info): ) error_field = GraphQLField(GraphQLString, resolve=resolve_error) error_non_null_field = GraphQLField(GraphQLNonNull(GraphQLString), resolve=resolve_error) + error_middleware_field = GraphQLField(GraphQLString, resolve=resolve_hello) query = GraphQLObjectType( name="Query", @@ -223,6 +225,7 @@ def resolve_error(root, info): "storage": storage_field, "error": error_field, "error_non_null": error_non_null_field, + "error_middleware": error_middleware_field, }, ) diff --git a/tox.ini b/tox.ini index 81d38e1730..94ddfff494 100644 --- a/tox.ini +++ b/tox.ini @@ -148,7 +148,7 @@ envlist = usefixtures = collector_available_fixture collector_agent_registration - ; code_coverage + code_coverage [testenv] deps = @@ -183,8 +183,9 @@ deps = component_flask_rest: flask-restful component_flask_rest: flask-restplus component_flask_rest: flask-restx - component_graphqlserver: graphql-server[sanic]==3.0.0b5 + component_graphqlserver: graphql-server[sanic,flask]==3.0.0b5 component_graphqlserver: sanic>20 + component_graphqlserver: Flask component_graphqlserver: jinja2 component_tastypie-tastypie0143: django-tastypie<0.14.4 component_tastypie-{py27,pypy}-tastypie0143: django<1.12