Automated fix for refs/heads/test-musl-linux-arm64

src/core/lib/iomgr/closure.h

@@ -211,10 +211,7 @@ inline grpc_closure* grpc_closure_create(grpc_iomgr_cb_func cb, void* cb_arg) {
   grpc_closure_create(cb, cb_arg)
 #endif
 
-#define GRPC_CLOSURE_LIST_INIT \
-  {                            \
-    nullptr, nullptr           \
-  }
+#define GRPC_CLOSURE_LIST_INIT {nullptr, nullptr}
 
 inline void grpc_closure_list_init(grpc_closure_list* closure_list) {
   closure_list->head = closure_list->tail = nullptr;

templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template

@@ -0,0 +1,27 @@
+%YAML 1.2
+--- |
+  # Copyright 2024 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+  # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!!
+
+  FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546
+
+  <%include file="../python_pip_builds.include"/>
+
+  <%include file="../ccache.include"/>
+
+  <%include file="../git_config.include"/>
+
+  RUN apk add openssl openssl-dev

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -36,6 +36,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:004e02902825b54c7f0d60eaed0819acf6f10c24853bf8f793001114e9969abd",
     "tools/dockerfile/distribtest/csharp_ubuntu2204_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu2204_x64@sha256:a439f2ccbc666f231e511a8e58eb7f66a3de4820a4d5aded1e62275cf8ac49f0",
     "tools/dockerfile/distribtest/php8_debian12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php8_debian12_x64@sha256:42399d061f234ee0ef79b333555db0e11ca4f106bb1d49276abf99c459c104f0",
+    "tools/dockerfile/distribtest/python_alpine_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e",
     "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:1bbc6fa5b4b650d3037d089e164d364e05a6daf1ed6dd1025ba07cc127f73d7d",
     "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1",
     "tools/dockerfile/distribtest/python_bullseye_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_bullseye_x64@sha256:80553398f0c59c1dc186052f4f2deaf18fea582f6d1d166eec6ea298639031fb",
@@ -67,6 +68,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:263ea79d940c905233624b02751194474408338b0f5ae2fb822966b2f20f47ce",
     "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:0e102df67f31aeb6afe68250603288c2f1c98ccf360d1c42d751b8451da94b48",
     "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:527e2e9ec4db0c52a53b50abfd59907a1b7e221168dc401686f6a48d33bddc5c",
+    "tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27",
     "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:94b57e5ea31ebc29af734474bcaff3074770778e5d27557cdc06d755ee8bc7ed",
     "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:edf4a0c8333c9309e52f323aa7315bbc0e5643216613cab4ecd2bce3d1ec26c0",
     "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878",

tools/dockerfile/distribtest/python_alpine_aarch64.current_version

@@ -0,0 +1 @@
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_aarch64:086c6b7ba4177b5c086a4e1e36019fdd1d86a061@sha256:da5d191e62811fd3470d593aa9a7c2bfc8ee7d3453db92db13d62edb2eb9f39e

tools/dockerfile/distribtest/python_alpine_aarch64/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.10-alpine3.14
+
+# Our test infrastructure demands bash
+RUN apk update && apk add bash
+
+RUN pip3 install virtualenv

tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64.current_version

@@ -0,0 +1 @@
+us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_aarch64:b171ae144fc76a473fdfad8e25c36a02b886f6a9@sha256:29f16b30b10d113aeba7e34c2fa5f8a71ea9dde809cf31902ca572147a1aca27

tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile

@@ -0,0 +1,66 @@
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/dockerfile/grpc_artifact_python_musllinux_1_1_aarch64/Dockerfile.template`!!!
+
+FROM quay.io/pypa/musllinux_1_1_aarch64:2024-09-09-f386546
+
+#===================================
+# Install Python build requirements
+RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<4.0.0rc1'
+RUN /opt/python/cp313-cp313/bin/pip install --upgrade 'cython<4.0.0rc1'
+
+#=================
+# Install ccache
+
+# Install ccache from source since ccache 3.x packaged with most linux distributions
+# does not support Redis backend for caching.
+RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/download/v4.7.5/ccache-4.7.5.tar.gz \
+    && tar -zxf ccache.tar.gz \
+    && cd ccache-4.7.5 \
+    && mkdir build && cd build \
+    && cmake -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON .. \
+    && make -j4 && make install \
+    && cd ../.. \
+    && rm -rf ccache-4.7.5 ccache.tar.gz
+
+
+# TODO: simplify the list of third_party modules list
+# NOTE: git>=2.46 allows leading paths like third_party/* to include all subdirectories
+# current docker base images use git versions lower than 2.46 and hence require separate configs for each submodule  
+RUN git config --global --add safe.directory /var/local/jenkins/grpc
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/bloaty
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/xds
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googleapis
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/googletest
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opencensus-proto
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/boringssl-with-bazel
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/envoy-api
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protobuf
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/zlib
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/benchmark
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/re2
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/abseil-cpp
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/opentelemetry-cpp
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/protoc-gen-validate
+RUN git config --global --add safe.directory /var/local/jenkins/grpc/.git/modules/third_party/cares/cares
+RUN git config --global protocol.file.allow always
+
+RUN apk add openssl openssl-dev

tools/internal_ci/linux/arm64/grpc_distribtests_python_arm64.cfg

@@ -0,0 +1,36 @@
+# Copyright 2025 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 -e x64 x86 armv7"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "x64 x86 armv7"
+# }

tools/internal_ci/linux/grpc_distribtests_python.cfg

@@ -16,11 +16,21 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
-timeout_mins: 240
+timeout_mins: 840
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
     regex: "github/grpc/reports/**"
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "aarch64"
+# }

tools/internal_ci/linux/grpc_distribtests_python.sh

@@ -26,7 +26,13 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 # some distribtests use a pre-registered binfmt_misc hook
 # to automatically execute foreign binaries (such as aarch64)
 # under qemu emulator.
-source tools/internal_ci/helper_scripts/prepare_qemu_rc
+# TASK_RUNNER_EXTRA_FILTERS="${TASK_RUNNER_EXTRA_FILTERS:-""}" # Handle unset variable
+
+# if [[ "$TASK_RUNNER_EXTRA_FILTERS" == *"aarch64"* && ! ("$TASK_RUNNER_EXTRA_FILTERS" == *"x64"* || "$TASK_RUNNER_EXTRA_FILTERS" == *"x86"* || "$TASK_RUNNER_EXTRA_FILTERS" == *"armv7"* ) ]]; then
+#   echo "Skipping prepare_qemu_rc because TASK_RUNNER_EXTRA_FILTERS matches the condition."
+# else
+#   source tools/internal_ci/helper_scripts/prepare_qemu_rc
+# fi
 
 # configure ccache
 source tools/internal_ci/helper_scripts/prepare_ccache_rc

tools/internal_ci/linux/grpc_distribtests_python_arm64.cfg

@@ -0,0 +1,36 @@
+# Copyright 2025 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 -e x64 x86 armv7"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "x64 x86 armv7"
+# }

tools/internal_ci/linux/pull_request/grpc_distribtests_python.cfg

@@ -16,7 +16,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
-timeout_mins: 240
+timeout_mins: 840
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"

tools/internal_ci/linux/pull_request/grpc_examples_sanity_tests.cfg

@@ -23,3 +23,13 @@ action {
     regex: "github/grpc/reports/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "aarch64 -e x64 x86 armv7"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "x64 x86 armv7"
+# }

tools/internal_ci/linux/release/grpc_distribtests_python.cfg

@@ -16,11 +16,21 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_python.sh"
-timeout_mins: 240
+timeout_mins: 840
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
     regex: "github/grpc/reports/**"
     regex: "github/grpc/artifacts/**"
   }
 }
+
+env_vars {
+  key: "TASK_RUNNER_EXTRA_FILTERS"
+  value: "-e aarch64"
+}
+
+# env_vars {
+#   key: "TASK_RUNNER_EXCLUDE_FILTERS"
+#   value: "aarch64"
+# }

tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg

@@ -26,5 +26,5 @@ action {
 
 env_vars {
   key: "RUN_TESTS_FLAGS"
-  value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=5400"
+  value: "-f basictests windows python -j 1 --inner_jobs 8 --max_time=7200"
 }

tools/run_tests/artifacts/artifact_targets.py

@@ -200,19 +200,25 @@ def build_jobspec(self, inner_jobs=None):
             )
             environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version)
             environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE"
-            environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
-            environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE"
 
-            if self.arch == "x86":
+            if self.arch in ("x86", "aarch64"):
                 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE"
 
+            if self.arch == "aarch64":
+                # As we won't strip the binary with auditwheel (see below), strip
+                # it at link time.
+                environ["LDFLAGS"] = "-s"
+            else:
+                environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE"
+                environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE"
+
             return create_docker_jobspec(
                 self.name,
                 "tools/dockerfile/grpc_artifact_python_%s_%s"
                 % (self.platform, self.arch),
                 "tools/run_tests/artifacts/build_artifact_python.sh",
                 environ=environ,
-                timeout_seconds=60 * 60 * 2,
+                timeout_seconds=60 * 60 * 10,
             )
         elif self.platform == "windows":
             environ["EXT_COMPILER"] = "msvc"
@@ -452,6 +458,16 @@ def targets():
             PythonArtifact(
                 "musllinux_1_1", "x86", "cp313-cp313", presubmit=True
             ),
+            PythonArtifact(
+                "musllinux_1_1", "aarch64", "cp38-cp38", presubmit=True
+            ),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp39-cp39"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp310-cp310"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp311-cp311"),
+            PythonArtifact("musllinux_1_1", "aarch64", "cp312-cp312"),
+            PythonArtifact(
+                "musllinux_1_1", "aarch64", "cp313-cp313", presubmit=True
+            ),
             PythonArtifact("macos", "x64", "python3.8", presubmit=True),
             PythonArtifact("macos", "x64", "python3.9"),
             PythonArtifact("macos", "x64", "python3.10"),

tools/run_tests/artifacts/distribtest_targets.py

@@ -461,6 +461,7 @@ def targets():
         PythonDistribTest(
             "linux", "aarch64", "python38_buster", presubmit=True
         ),
+        PythonDistribTest("linux", "aarch64", "alpine"),
         PythonDistribTest(
             "linux", "x64", "alpine3.18", source=True, presubmit=True
         ),

tools/run_tests/run_tests_matrix.py

@@ -33,6 +33,9 @@
 # C/C++ tests can take long time
 _CPP_RUNTESTS_TIMEOUT = 6 * 60 * 60
 
+# Python tests can take long time
+_PYTHON_RUNTESTS_TIMEOUT = 2 * 60 * 60
+
 # Set timeout high for ObjC for Cocoapods to install pods
 _OBJC_RUNTESTS_TIMEOUT = 4 * 60 * 60
 
@@ -278,6 +281,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
         labels=["basictests", "multilang"],
         extra_args=extra_args + ["--report_multi_target"],
         inner_jobs=inner_jobs,
+        timeout_seconds=_PYTHON_RUNTESTS_TIMEOUT,
     )
 
     # ARM64 Linux Python tests