Merge branch 'shush' into shush-connect

doc/python/sphinx/grpc_observability.rst

@@ -0,0 +1,7 @@
+gRPC Python Observability
+====================
+
+Module Contents
+---------------
+
+.. automodule:: grpc_observability

src/core/lib/experiments/experiments.yaml

@@ -115,7 +115,7 @@
 - name: multiping
   description:
     Allow more than one ping to be in flight at a time by default.
-  expiry: 2024/01/15
+  expiry: 2024/06/15
   owner: [email protected]
   test_tags: [flow_control_test]
 - name: peer_state_based_framing
@@ -231,7 +231,7 @@
   test_tags: []
 - name: unconstrained_max_quota_buffer_size
   description: Discard the cap on the max free pool size for one memory allocator
-  expiry: 2024/02/01
+  expiry: 2024/09/01
   owner: [email protected]
   test_tags: [resource_quota_test]
 - name: v3_backend_metric_filter
@@ -261,7 +261,7 @@
 - name: work_serializer_clears_time_cache
   description:
     Have the work serializer clear the time cache when it dispatches work.
-  expiry: 2024/01/01
+  expiry: 2024/04/01
   owner: [email protected]
   test_tags: []
 - name: work_serializer_dispatch

src/python/grpcio_observability/README.rst

@@ -1,2 +1,91 @@
-### TODO(xuanwn)
-* Fill in content.
+gRPC Python Observability
+===========
+Package for gRPC Python Observability.
+
+More details can be found in `OpenTelemetry Metrics gRFC <https://github.com/grpc/proposal/blob/master/A66-otel-stats.md#opentelemetry-metrics>`_.
+
+How gRPC Python Observability Works
+-------------------------
+
+gRPC Python is a wrapper layer built upon the gRPC Core (written in C/C++). Most of telemetry data
+is collected at core layer and then exported to Python layer. To optimize performance and reduce
+the overhead of acquiring the GIL too frequently, telemetry data is initially cached at the Core layer
+and then exported to the Python layer in batches.
+
+Note that while this approach enhances efficiency, it will introduce a slight delay between the
+time the data is collected and the time it becomes available through Python exporters.
+
+
+Supported Python Versions
+-------------------------
+Python >= 3.7
+
+Installation
+------------
+
+Currently gRPC Python Observability is **only available for Linux**.
+
+Installing From PyPI
+~~~~~~~~~~~~~~~~~~~~
+
+::
+
+  $ pip install grpcio-observability
+
+
+Installing From Source
+~~~~~~~~~~~~~~~~~~~~~~
+
+Building from source requires that you have the Python headers (usually a
+package named :code:`python-dev`) and Cython installed. It further requires a
+GCC-like compiler to go smoothly; you can probably get it to work without
+GCC-like stuff, but you may end up having a bad time.
+
+::
+
+  $ export REPO_ROOT=grpc  # REPO_ROOT can be any directory of your choice
+  $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc $REPO_ROOT
+  $ cd $REPO_ROOT
+  $ git submodule update --init
+
+  $ cd src/python/grpcio_observability
+  $ python -m make_grpcio_observability
+
+  # For the next command do `sudo pip install` if you get permission-denied errors
+  $ GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+
+
+Dependencies
+-------------------------
+gRPC Python Observability Depends on the following packages:
+
+::
+
+  grpcio
+  opentelemetry-sdk==1.21.0
+  opentelemetry-api==1.21.0
+
+
+Usage
+-----
+
+You can find example usage in `Python example folder <https://github.com/grpc/grpc/tree/master/examples/python/observability>`_.
+
+We also provide several environment variables to help you optimize gRPC python observability for your particular use.
+
+1. GRPC_PYTHON_CENSUS_EXPORT_BATCH_INTERVAL
+    * This controls how frequently telemetry data collected within gRPC Core is sent to Python layer.
+    * Default value is 0.5 (Seconds).
+
+2. GRPC_PYTHON_CENSUS_MAX_EXPORT_BUFFER_SIZE
+    * This controls the maximum number of telemetry data items that can be held in the buffer within gRPC Core before they are sent to Python.
+    * Default value is 10,000.
+
+3. GRPC_PYTHON_CENSUS_EXPORT_THRESHOLD
+    * This setting acts as a trigger: When the buffer in gRPC Core reaches a certain percentage of its capacity, the telemetry data is sent to Python.
+    * Default value is 0.7 (Which means buffer will start export when it's 70% full).
+
+4. GRPC_PYTHON_CENSUS_EXPORT_THREAD_TIMEOUT
+    * This controls the maximum time allowed for the exporting thread (responsible for sending data to Python) to complete.
+    * Main thread will terminate the exporting thread after this timeout.
+    * Default value is 10 (Seconds).

src/python/grpcio_observability/grpc_observability/_open_telemetry_observability.py

@@ -59,6 +59,8 @@
 class OpenTelemetryObservability(grpc._observability.ObservabilityPlugin):
     """OpenTelemetry based plugin implementation.
 
+    This is class is part of an EXPERIMENTAL API.
+
     Args:
       exporter: Exporter used to export data.
       plugin: OpenTelemetryPlugin to enable.

src/python/grpcio_observability/grpc_observability/_open_telemetry_plugin.py

@@ -90,14 +90,33 @@ def get_label_injector(self) -> Optional[OpenTelemetryLabelInjector]:
 
 # pylint: disable=no-self-use
 class OpenTelemetryPlugin:
-    """Describes a Plugin for OpenTelemetry observability."""
+    """Describes a Plugin for OpenTelemetry observability.
+
+    This is class is part of an EXPERIMENTAL API.
+    """
 
     def get_plugin_options(
         self,
     ) -> Iterable[OpenTelemetryPluginOption]:
+        """
+        This function will be used to get plugin options which are enabled for
+        this OpenTelemetryPlugin instance.
+
+        Returns:
+            An Iterable of class OpenTelemetryPluginOption which will be enabled for
+            this OpenTelemetryPlugin.
+        """
         return []
 
     def get_meter_provider(self) -> Optional[MeterProvider]:
+        """
+        This function will be used to get the MeterProvider for this OpenTelemetryPlugin
+        instance.
+
+        Returns:
+            A MeterProvider which will be used to collect telemetry data, or None which
+            means no metrics will be collected.
+        """
         return None
 
     def target_attribute_filter(

src/python/grpcio_tests/tests/observability/BUILD.bazel

@@ -18,6 +18,11 @@ py_library(
     srcs = ["_test_server.py"],
 )
 
+py_library(
+    name = "_from_observability_import_star",
+    srcs = ["_from_observability_import_star.py"],
+)
+
 py_test(
     name = "_observability_test",
     size = "small",
@@ -45,3 +50,17 @@ py_test(
         "//src/python/grpcio_tests/tests/testing",
     ],
 )
+
+py_test(
+    name = "_observability_api_test",
+    size = "small",
+    srcs = ["_observability_api_test.py"],
+    imports = ["../../"],
+    main = "_observability_api_test.py",
+    deps = [
+        ":_from_observability_import_star",
+        "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_observability/grpc_observability:pyobservability",
+        "//src/python/grpcio_tests/tests/testing",
+    ],
+)

src/python/grpcio_tests/tests/observability/_from_observability_import_star.py

@@ -0,0 +1,25 @@
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+_BEFORE_IMPORT = tuple(globals())
+
+from grpc_observability import *  # pylint: disable=wildcard-import,unused-wildcard-import
+
+_AFTER_IMPORT = tuple(globals())
+
+GRPC_OBSERVABILITY_ELEMENTS = tuple(
+    element
+    for element in _AFTER_IMPORT
+    if element not in _BEFORE_IMPORT and element != "_BEFORE_IMPORT"
+)

src/python/grpcio_tests/tests/observability/_observability_api_test.py

@@ -0,0 +1,37 @@
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test of gRPC Python Observability's application-layer API."""
+
+import logging
+import unittest
+
+from tests.observability import _from_observability_import_star
+
+
+class AllTest(unittest.TestCase):
+    def testAll(self):
+        expected_observability_code_elements = (
+            "OpenTelemetryObservability",
+            "OpenTelemetryPlugin",
+        )
+
+        self.assertCountEqual(
+            expected_observability_code_elements,
+            _from_observability_import_star.GRPC_OBSERVABILITY_ELEMENTS,
+        )
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    unittest.main(verbosity=2)

src/python/grpcio_tests/tests/tests.json

@@ -9,6 +9,7 @@
   "tests.health_check._health_servicer_test.HealthServicerTest",
   "tests.interop._insecure_intraop_test.InsecureIntraopTest",
   "tests.interop._secure_intraop_test.SecureIntraopTest",
+  "tests.observability._observability_api_test.AllTest",
   "tests.observability._observability_test.ObservabilityTest",
   "tests.observability._open_telemetry_observability_test.OpenTelemetryObservabilityTest",
   "tests.protoc_plugin._python_plugin_test.ModuleMainTest",

tools/run_tests/artifacts/artifact_targets.py

@@ -142,7 +142,7 @@ def build_jobspec(self, inner_jobs=None):
 
         if self.platform == "macos":
             environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64"
-            environ["GRPC_UNIVERSAL2_REPAIR"] = "true"
+            environ["GRPC_BUILD_MAC"] = "true"
 
         if self.platform == "linux_extra":
             # Crosscompilation build for armv7 (e.g. Raspberry Pi)

tools/run_tests/artifacts/build_artifact_python.sh

@@ -89,6 +89,7 @@ ancillary_package_dir=(
   "src/python/grpcio_reflection/"
   "src/python/grpcio_status/"
   "src/python/grpcio_testing/"
+  "src/python/grpcio_observability/"
 )
 
 # Copy license to ancillary package directories so it will be distributed.
@@ -184,7 +185,7 @@ fix_faulty_universal2_wheel() {
 # This is necessary due to https://github.com/pypa/wheel/issues/406.
 # wheel incorrectly generates a universal2 artifact that only contains
 # x86_64 libraries.
-if [ "$GRPC_UNIVERSAL2_REPAIR" != "" ]; then
+if [ "$GRPC_BUILD_MAC" != "" ]; then
   for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do
     fix_faulty_universal2_wheel "$WHEEL"
   done
@@ -280,4 +281,14 @@ then
   ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_admin/setup.py \
       sdist bdist_wheel
   cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR"
+
+  # Build grpcio_observability source distribution
+  # Skips MacOS since grpcio_observability does not support MacOS.
+  if [ "$GRPC_BUILD_MAC" == "" ]; then
+    "${PYTHON}" src/python/grpcio_observability/make_grpcio_observability.py
+    ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py \
+        sdist bdist_wheel
+    cp -r src/python/grpcio_observability/dist/* "$ARTIFACT_DIR"
+  fi
+
 fi