Automated fix for refs/heads/test-musl-linux-arm64

src/core/lib/iomgr/closure.h

@@ -211,10 +211,7 @@ inline grpc_closure* grpc_closure_create(grpc_iomgr_cb_func cb, void* cb_arg) {
   grpc_closure_create(cb, cb_arg)
 #endif
 
-#define GRPC_CLOSURE_LIST_INIT \
-  {                            \
-    nullptr, nullptr           \
-  }
+#define GRPC_CLOSURE_LIST_INIT {nullptr, nullptr}
 
 inline void grpc_closure_list_init(grpc_closure_list* closure_list) {
   closure_list->head = closure_list->tail = nullptr;

templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template

@@ -0,0 +1,27 @@
+%YAML 1.2
+--- |
+  # Copyright 2024 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!!
+
+  FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546
+
+  <%include file="../python_pip_builds.include"/>
+
+  <%include file="../ccache.include"/>
+
+  <%include file="../git_config.include"/>
+
+  RUN apk add openssl openssl-dev

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -36,6 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd",
     "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0",
     "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0",
+    "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e",
     "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d",
     "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1",
     "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb",
@@ -67,6 +68,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:263ea79d940c905233624b02751194474408338b0f5ae2fb822966b2f20f47ce",
     "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:0e102df67f31aeb6afe68250603288c2f1c98ccf360d1c42d751b8451da94b48",
     "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:527e2e9ec4db0c52a53b50abfd59907a1b7e221168dc401686f6a48d33bddc5c",
+    "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27",
     "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:94b57e5ea31ebc29af734474bcaff3074770778e5d27557cdc06d755ee8bc7ed",
     "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:edf4a0c8333c9309e52f323aa7315bbc0e5643216613cab4ecd2bce3d1ec26c0",
     "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878",

tools/dockerfile/distribtest/python_alpine_aarch64.current_version

@@ -0,0 +1 @@
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:086c6b7ba4177b5c086a4e1e36019fdd1d86a061@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e

tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.10-alpine3.14
+
+# Our test infrastructure demands bash
+RUN apk update && apk add bash
+
+RUN pip3 install virtualenv

tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version

@@ -0,0 +1 @@
+us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:b171ae144fc76a473fdfad8e25c36a02b886f6a9@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27

tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile

@@ -0,0 +1,66 @@
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!!
+
+FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546
+
+#===================================
+# Install Python build requirements
+RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp313-cp313/bin/pip install --upgrade 'cython<4.0.0rc1'
+
+#=================
+# Install ccache
+
+# Install ccache from source since ccache 3.x packaged with most linux distributions
+# does not support Redis backend for caching.
+RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.7.5/ccache-4.7.5.tar.gz \
+    && tar -zxf ccache.tar.gz \
+    && cd ccache-4.7.5 \
+    && mkdir build && cd build \
+    && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \
+    && make -j4 && make install \
+    && cd ../.. \
+    && rm -rf ccache-4.7.5 ccache.tar.gz
+
+
+# TODO: simplify the list of third_party modules list
+# NOTE: git>=2.46 allows leading paths like third_party/* to include all subdirectories
+# current docker base images use git versions lower than 2.46 and hence require separate configs for each submodule  
+RUN git config --global --add safe.directory /var/local/jenkins/grpc
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/bloaty
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/xds
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googleapis
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googletest
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opencensus-proto
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/boringssl-with-bazel
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/envoy-api
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protobuf
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/zlib
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/benchmark
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/re2
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/abseil-cpp
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry-cpp
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protoc-gen-validate
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/cares/cares
+RUN git config --global protocol.file.allow always
+
+RUN apk add openssl openssl-dev

tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg

@@ -0,0 +1,31 @@
+# Copyright 2025 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 musllinux_1_1"
+}

tools/internal_ci/linux/grpc_distribtests_python.cfg

@@ -16,11 +16,16 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
-timeout_mins: 240
+timeout_mins: 840
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
     regex: "github/grpc/reports/**"
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64 musllinux_1_1"
+}

tools/internal_ci/linux/grpc_distribtests_python.sh

@@ -23,10 +23,14 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-# some distribtests use a pre-registered binfmt_misc hook
-# to automatically execute foreign binaries (such as aarch64)
-# under qemu emulator.
-source tools/internal_ci/helper_scripts/prepare_qemu_rc
+if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then
+  echo "Skipping prepare_qemu_rc'"
+else
+  # some distribtests use a pre-registered binfmt_misc hook
+  # to automatically execute foreign binaries (such as aarch64)
+  # under qemu emulator.
+  source tools/internal_ci/helper_scripts/prepare_qemu_rc
+fi
 
 # configure ccache
 source tools/internal_ci/helper_scripts/prepare_ccache_rc
@@ -40,7 +44,11 @@ mkdir -p input_artifacts
 cp -r artifacts/* input_artifacts/ || true
 
 # This step simply collects python artifacts from subdirectories of input_artifacts/ and copies them to artifacts/
-tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+if [[ "${TASK_RUNNER_EXTRA_FILTERS}" == "aarch64 musllinux_1_1" ]]; then
+  tools/run_tests/task_runner.py -f package linux python ${TASK_RUNNER_EXTRA_FILTERS} -x build_packages/sponge_log.xml || FAILED="true"
+else
+  tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true"
+fi
 
 # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder.
 # in addition to that, preserve the contents of "artifacts" directory since we want kokoro

tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg

@@ -16,7 +16,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
-timeout_mins: 240
+timeout_mins: 840
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"

tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg

@@ -23,3 +23,13 @@ action {
     regex: "github/grpc/reports/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 -e x64 x86 armv7"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "x64 x86 armv7"
+# }

tools/internal_ci/linux/release/grpc_distribtests_python.cfg

@@ -16,11 +16,16 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
-timeout_mins: 240
+timeout_mins: 840
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
     regex: "github/grpc/reports/**"
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64 musllinux_1_1"
+}

tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg

@@ -26,5 +26,5 @@ action {
 
 env_vars {
   key: "RUN_TESTS_FLAGS"
-  value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=5400"
+  value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=7200"
 }

tools/run_tests/artifacts/artifact_targets.py

@@ -200,19 +200,25 @@ def build_jobspec(self, inner_jobs=None):
             )
             environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version)
             environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE"
-            environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
-            environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE"
 
-            if self.arch == "x86":
+            if self.arch in ("x86", "aarch64"):
                 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE"
 
+            if self.arch == "aarch64":
+                # As we won't strip the binary with auditwheel (see below), strip
+                # it at link time.
+                environ["LDFLAGS"] = "-s"
+            else:
+                environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
+                environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE"
+
             return create_docker_jobspec(
                 self.name,
                 "tools/dockerfile/grpc_artifact_python_%s_%s"
                 % (self.platform, self.arch),
                 "tools/run_tests/artifacts/build_artifact_python.sh",
                 environ=environ,
-                timeout_seconds=60 * 60 * 2,
+                timeout_seconds=60 * 60 * 10,
             )
         elif self.platform == "windows":
             environ["EXT_COMPILER"] = "msvc"
@@ -452,6 +458,16 @@ def targets():
             PythonArtifact(
                 "musllinux_1_1", "x86", "cp313-cp313", presubmit=True
             ),
+            PythonArtifact(
+                "musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True
+            ),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp311-cp311"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp312-cp312"),
+            PythonArtifact(
+                "musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True
+            ),
             PythonArtifact("macos", "x64", "python3.8", presubmit=True),
             PythonArtifact("macos", "x64", "python3.9"),
             PythonArtifact("macos", "x64", "python3.10"),

tools/run_tests/artifacts/distribtest_targets.py

@@ -461,6 +461,7 @@ def targets():
         PythonDistribTest(
             "linux", "aarch64", "python38_buster", presubmit=True
         ),
+        PythonDistribTest("linux", "aarch64", "alpine"),
         PythonDistribTest(
             "linux", "x64", "alpine3.18", source=True, presubmit=True
         ),

tools/run_tests/artifacts/package_targets.py

@@ -142,9 +142,17 @@ def build_jobspec(self, inner_jobs=None):
 class PythonPackage:
     """Collects python eggs and wheels created in the artifact phase"""
 
-    def __init__(self):
+    def __init__(self, platform="", arch=""):
         self.name = "python_package"
         self.labels = ["package", "python", "linux"]
+        self.platform = platform
+        self.arch = arch
+        if self.platform:
+            self.labels.append(platform)
+            self.name += "_" + platform
+        if self.arch:
+            self.labels.append(arch)
+            self.name += "_" + arch
 
     def pre_build_jobspecs(self):
         return []
@@ -154,9 +162,16 @@ def build_jobspec(self, inner_jobs=None):
         # since the python package build does very little, we can use virtually
         # any image that has new-enough python, so reusing one of the images used
         # for artifact building seems natural.
+        dockerfile_dir = (
+            "tools/dockerfile/grpc_artifact_python_manylinux2014_x64"
+        )
+        if "musllinux_1_1" in self.platform and "aarch64" in self.arch:
+            dockerfile_dir = (
+                "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64",
+            )
         return create_docker_jobspec(
             self.name,
-            "tools/dockerfile/grpc_artifact_python_manylinux2014_x64",
+            dockerfile_dir,
             "tools/run_tests/artifacts/build_package_python.sh",
             environ={"PYTHON": "/opt/python/cp39-cp39/bin/python"},
         )
@@ -189,5 +204,6 @@ def targets():
         CSharpPackage("windows"),
         RubyPackage(),
         PythonPackage(),
+        PythonPackage("musllinux_1_1", "aarch64"),
         PHPPackage(),
     ]

tools/run_tests/python_utils/jobset.py

@@ -45,6 +45,8 @@ def strip_non_ascii_chars(s):
 def sanitized_environment(env):
     sanitized = {}
     for key, value in list(env.items()):
+        sys.stderr.write(f"strip_non_ascii_chars key: {key}, value: {value}\n")
+        sys.stderr.flush()
         sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value)
     return sanitized