From 7b2ad05d09f77d8fbb18d7cbd1abf37b3b5e2686 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 22 Jan 2024 14:05:22 -0800 Subject: [PATCH 1/2] [experiments] Move forward some of my expired experiments (#35620) Closes #35620 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35620 from ctiller:expired 8bd5f24d10ba72608e49a543ac722f83ba3e5c09 PiperOrigin-RevId: 600564608 --- src/core/lib/experiments/experiments.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 266c0b540b644..1d5016a8a247b 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -115,7 +115,7 @@ - name: multiping description: Allow more than one ping to be in flight at a time by default. - expiry: 2024/01/15 + expiry: 2024/06/15 owner: ctiller@google.com test_tags: [flow_control_test] - name: peer_state_based_framing @@ -231,7 +231,7 @@ test_tags: [] - name: unconstrained_max_quota_buffer_size description: Discard the cap on the max free pool size for one memory allocator - expiry: 2024/02/01 + expiry: 2024/09/01 owner: ctiller@google.com test_tags: [resource_quota_test] - name: v3_backend_metric_filter @@ -261,7 +261,7 @@ - name: work_serializer_clears_time_cache description: Have the work serializer clear the time cache when it dispatches work. - expiry: 2024/01/01 + expiry: 2024/04/01 owner: ctiller@google.com test_tags: [] - name: work_serializer_dispatch From 1afcc3bd13ad8d56b9dd6c3a3fe591613d3b2c07 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Mon, 22 Jan 2024 14:12:49 -0800 Subject: [PATCH 2/2] [Python O11y] Build and distrib O11y package (#35578) Actually build O11y artifacts. ### Testing * Manually installed the `.whl` and verified it's working locally (For Python 3.8 + Linux). Closes #35578 PiperOrigin-RevId: 600566829 --- doc/python/sphinx/grpc_observability.rst | 7 ++ src/python/grpcio_observability/README.rst | 93 ++++++++++++++++++- .../_open_telemetry_observability.py | 2 + .../_open_telemetry_plugin.py | 21 ++++- .../tests/observability/BUILD.bazel | 19 ++++ .../_from_observability_import_star.py | 25 +++++ .../observability/_observability_api_test.py | 37 ++++++++ src/python/grpcio_tests/tests/tests.json | 1 + tools/run_tests/artifacts/artifact_targets.py | 2 +- .../artifacts/build_artifact_python.sh | 13 ++- 10 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 doc/python/sphinx/grpc_observability.rst create mode 100644 src/python/grpcio_tests/tests/observability/_from_observability_import_star.py create mode 100644 src/python/grpcio_tests/tests/observability/_observability_api_test.py diff --git a/doc/python/sphinx/grpc_observability.rst b/doc/python/sphinx/grpc_observability.rst new file mode 100644 index 0000000000000..c0316f5d4f7fc --- /dev/null +++ b/doc/python/sphinx/grpc_observability.rst @@ -0,0 +1,7 @@ +gRPC Python Observability +==================== + +Module Contents +--------------- + +.. automodule:: grpc_observability diff --git a/src/python/grpcio_observability/README.rst b/src/python/grpcio_observability/README.rst index b46aa89147f57..a597057198f52 100644 --- a/src/python/grpcio_observability/README.rst +++ b/src/python/grpcio_observability/README.rst @@ -1,2 +1,91 @@ -### TODO(xuanwn) -* Fill in content. \ No newline at end of file +gRPC Python Observability +=========== +Package for gRPC Python Observability. + +More details can be found in `OpenTelemetry Metrics gRFC `_. + +How gRPC Python Observability Works +------------------------- + +gRPC Python is a wrapper layer built upon the gRPC Core (written in C/C++). Most of telemetry data +is collected at core layer and then exported to Python layer. To optimize performance and reduce +the overhead of acquiring the GIL too frequently, telemetry data is initially cached at the Core layer +and then exported to the Python layer in batches. + +Note that while this approach enhances efficiency, it will introduce a slight delay between the +time the data is collected and the time it becomes available through Python exporters. + + +Supported Python Versions +------------------------- +Python >= 3.7 + +Installation +------------ + +Currently gRPC Python Observability is **only available for Linux**. + +Installing From PyPI +~~~~~~~~~~~~~~~~~~~~ + +:: + + $ pip install grpcio-observability + + +Installing From Source +~~~~~~~~~~~~~~~~~~~~~~ + +Building from source requires that you have the Python headers (usually a +package named :code:`python-dev`) and Cython installed. It further requires a +GCC-like compiler to go smoothly; you can probably get it to work without +GCC-like stuff, but you may end up having a bad time. + +:: + + $ export REPO_ROOT=grpc # REPO_ROOT can be any directory of your choice + $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc $REPO_ROOT + $ cd $REPO_ROOT + $ git submodule update --init + + $ cd src/python/grpcio_observability + $ python -m make_grpcio_observability + + # For the next command do `sudo pip install` if you get permission-denied errors + $ GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . + + +Dependencies +------------------------- +gRPC Python Observability Depends on the following packages: + +:: + + grpcio + opentelemetry-sdk==1.21.0 + opentelemetry-api==1.21.0 + + +Usage +----- + +You can find example usage in `Python example folder `_. + +We also provide several environment variables to help you optimize gRPC python observability for your particular use. + +1. GRPC_PYTHON_CENSUS_EXPORT_BATCH_INTERVAL + * This controls how frequently telemetry data collected within gRPC Core is sent to Python layer. + * Default value is 0.5 (Seconds). + +2. GRPC_PYTHON_CENSUS_MAX_EXPORT_BUFFER_SIZE + * This controls the maximum number of telemetry data items that can be held in the buffer within gRPC Core before they are sent to Python. + * Default value is 10,000. + +3. GRPC_PYTHON_CENSUS_EXPORT_THRESHOLD + * This setting acts as a trigger: When the buffer in gRPC Core reaches a certain percentage of its capacity, the telemetry data is sent to Python. + * Default value is 0.7 (Which means buffer will start export when it's 70% full). + +4. GRPC_PYTHON_CENSUS_EXPORT_THREAD_TIMEOUT + * This controls the maximum time allowed for the exporting thread (responsible for sending data to Python) to complete. + * Main thread will terminate the exporting thread after this timeout. + * Default value is 10 (Seconds). diff --git a/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py b/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py index 8f3a96475df81..e6f94e77211e6 100644 --- a/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py +++ b/src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py @@ -59,6 +59,8 @@ class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin): """OpenTelemetry based plugin implementation. + This is class is part of an EXPERIMENTAL API. + Args: exporter: Exporter used to export data. plugin: OpenTelemetryPlugin to enable. diff --git a/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py b/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py index 823d30fa6b43d..212a9ec1b82b0 100644 --- a/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py +++ b/src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py @@ -90,14 +90,33 @@ def get_label_injector(self) -> Optional[OpenTelemetryLabelInjector]: # pylint: disable=no-self-use class OpenTelemetryPlugin: - """Describes a Plugin for OpenTelemetry observability.""" + """Describes a Plugin for OpenTelemetry observability. + + This is class is part of an EXPERIMENTAL API. + """ def get_plugin_options( self, ) -> Iterable[OpenTelemetryPluginOption]: + """ + This function will be used to get plugin options which are enabled for + this OpenTelemetryPlugin instance. + + Returns: + An Iterable of class OpenTelemetryPluginOption which will be enabled for + this OpenTelemetryPlugin. + """ return [] def get_meter_provider(self) -> Optional[MeterProvider]: + """ + This function will be used to get the MeterProvider for this OpenTelemetryPlugin + instance. + + Returns: + A MeterProvider which will be used to collect telemetry data, or None which + means no metrics will be collected. + """ return None def target_attribute_filter( diff --git a/src/python/grpcio_tests/tests/observability/BUILD.bazel b/src/python/grpcio_tests/tests/observability/BUILD.bazel index 29bd6a90e6090..bd4e10b450a18 100644 --- a/src/python/grpcio_tests/tests/observability/BUILD.bazel +++ b/src/python/grpcio_tests/tests/observability/BUILD.bazel @@ -18,6 +18,11 @@ py_library( srcs = ["_test_server.py"], ) +py_library( + name = "_from_observability_import_star", + srcs = ["_from_observability_import_star.py"], +) + py_test( name = "_observability_test", size = "small", @@ -45,3 +50,17 @@ py_test( "//src/python/grpcio_tests/tests/testing", ], ) + +py_test( + name = "_observability_api_test", + size = "small", + srcs = ["_observability_api_test.py"], + imports = ["../../"], + main = "_observability_api_test.py", + deps = [ + ":_from_observability_import_star", + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_observability/grpc_observability:pyobservability", + "//src/python/grpcio_tests/tests/testing", + ], +) diff --git a/src/python/grpcio_tests/tests/observability/_from_observability_import_star.py b/src/python/grpcio_tests/tests/observability/_from_observability_import_star.py new file mode 100644 index 0000000000000..e218c49cdb231 --- /dev/null +++ b/src/python/grpcio_tests/tests/observability/_from_observability_import_star.py @@ -0,0 +1,25 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +_BEFORE_IMPORT = tuple(globals()) + +from grpc_observability import * # pylint: disable=wildcard-import,unused-wildcard-import + +_AFTER_IMPORT = tuple(globals()) + +GRPC_OBSERVABILITY_ELEMENTS = tuple( + element + for element in _AFTER_IMPORT + if element not in _BEFORE_IMPORT and element != "_BEFORE_IMPORT" +) diff --git a/src/python/grpcio_tests/tests/observability/_observability_api_test.py b/src/python/grpcio_tests/tests/observability/_observability_api_test.py new file mode 100644 index 0000000000000..efdd63dca0a7f --- /dev/null +++ b/src/python/grpcio_tests/tests/observability/_observability_api_test.py @@ -0,0 +1,37 @@ +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test of gRPC Python Observability's application-layer API.""" + +import logging +import unittest + +from tests.observability import _from_observability_import_star + + +class AllTest(unittest.TestCase): + def testAll(self): + expected_observability_code_elements = ( + "OpenTelemetryObservability", + "OpenTelemetryPlugin", + ) + + self.assertCountEqual( + expected_observability_code_elements, + _from_observability_import_star.GRPC_OBSERVABILITY_ELEMENTS, + ) + + +if __name__ == "__main__": + logging.basicConfig() + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 7ba494ac5b611..657cb94c02862 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -9,6 +9,7 @@ "tests.health_check._health_servicer_test.HealthServicerTest", "tests.interop._insecure_intraop_test.InsecureIntraopTest", "tests.interop._secure_intraop_test.SecureIntraopTest", + "tests.observability._observability_api_test.AllTest", "tests.observability._observability_test.ObservabilityTest", "tests.observability._open_telemetry_observability_test.OpenTelemetryObservabilityTest", "tests.protoc_plugin._python_plugin_test.ModuleMainTest", diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index e32acb2a700a6..5368a6e3ca010 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -142,7 +142,7 @@ def build_jobspec(self, inner_jobs=None): if self.platform == "macos": environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64" - environ["GRPC_UNIVERSAL2_REPAIR"] = "true" + environ["GRPC_BUILD_MAC"] = "true" if self.platform == "linux_extra": # Crosscompilation build for armv7 (e.g. Raspberry Pi) diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 396aae5e3bb96..fb0c1920e48f1 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -89,6 +89,7 @@ ancillary_package_dir=( "src/python/grpcio_reflection/" "src/python/grpcio_status/" "src/python/grpcio_testing/" + "src/python/grpcio_observability/" ) # Copy license to ancillary package directories so it will be distributed. @@ -184,7 +185,7 @@ fix_faulty_universal2_wheel() { # This is necessary due to https://github.com/pypa/wheel/issues/406. # wheel incorrectly generates a universal2 artifact that only contains # x86_64 libraries. -if [ "$GRPC_UNIVERSAL2_REPAIR" != "" ]; then +if [ "$GRPC_BUILD_MAC" != "" ]; then for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do fix_faulty_universal2_wheel "$WHEEL" done @@ -280,4 +281,14 @@ then ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_admin/setup.py \ sdist bdist_wheel cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR" + + # Build grpcio_observability source distribution + # Skips MacOS since grpcio_observability does not support MacOS. + if [ "$GRPC_BUILD_MAC" == "" ]; then + "${PYTHON}" src/python/grpcio_observability/make_grpcio_observability.py + ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py \ + sdist bdist_wheel + cp -r src/python/grpcio_observability/dist/* "$ARTIFACT_DIR" + fi + fi