Use FetchContent for both pybind11 and nanobind. (#18872)

This avoids various pinning problems and ensures that the version that
IREE specifies at the top of a build is used consistently throughout.
The nanobind incantation was taken from shortfin. The pybind11
incantation was adapted from [a
comment](pybind/pybind11#2817 (comment))
and extended to use find_package integration.

The latter is sufficient for MLIR's pybind11 finding to delegate to the
one set at the top level. The MLIR code for finding pybind11 is ancient
and should be modernized to use FetchContent and find_package
integration with a pinned version. This would ensure consistent interop
with the rest of the ecosystem.

---------

Signed-off-by: Stella Laurenzo <[email protected]>
Co-authored-by: Marius Brehler <[email protected]>

.gitmodules

@@ -7,10 +7,6 @@
 [submodule "third_party/vulkan_headers"]
 	path = third_party/vulkan_headers
 	url = https://github.com/KhronosGroup/Vulkan-Headers.git
-[submodule "third_party/pybind11"]
-	path = third_party/pybind11
-	url = https://github.com/pybind/pybind11.git
-	branch = stable
 [submodule "third_party/benchmark"]
 	path = third_party/benchmark
 	url = https://github.com/google/benchmark.git

CMakeLists.txt

@@ -772,6 +772,30 @@ endif()
 # MLIR/LLVM Dependency
 #-------------------------------------------------------------------------------
 
+# Both the IREE and MLIR Python bindings require pybind11. We initialize it here
+# at the top level so that everything uses ours consistently.
+if(IREE_BUILD_PYTHON_BINDINGS AND IREE_BUILD_COMPILER)
+  set(pybind11_VERSION 2.13.6)
+  include(FetchContent)
+  FetchContent_Declare(
+      pybind11
+      GIT_REPOSITORY https://github.com/pybind/pybind11
+      GIT_TAG        v${pybind11_VERSION}
+  )
+  set(PYBIND11_FINDPYTHON ON)
+  FetchContent_MakeAvailable(pybind11)
+  # pybind11 source fetches do not include find_package integration, which is
+  # a shame since sub-projects can require that to work. If we were using
+  # CMake 3.24, we could just add OVERRIDE_FIND_PACKAGE to the
+  # FetchContent_Declare call above and it would take care of doing the
+  # following to let subsequent sub-project find_package calls to resolve
+  # successfully.
+  set(pybind11_DIR "${pybind11_BINARY_DIR}")
+  file(WRITE "${pybind11_BINARY_DIR}/pybind11Config.cmake" "")
+  file(WRITE "${pybind11_BINARY_DIR}/pybind11ConfigVersion.cmake"
+       "set(PACKAGE_VERSION ${pybind11_VERSION})\nset(PACKAGE_VERSION_COMPATIBLE TRUE)")
+endif()
+
 if(NOT IREE_BUILD_COMPILER)
   message(STATUS "Not adding LLVM/MLIR because the configuration does not require it")
 else()
@@ -921,19 +945,6 @@ if(IREE_BUILD_TESTS)
   include(iree_configure_testing)
 endif()
 
-if(IREE_BUILD_PYTHON_BINDINGS)
-  # The compiler uses pybind11
-  if(IREE_BUILD_COMPILER)
-    if(NOT TARGET pybind11::module)
-      message(STATUS "Using bundled pybind11")
-      set(PYBIND11_FINDPYTHON ON)
-      add_subdirectory(third_party/pybind11 EXCLUDE_FROM_ALL)
-    else()
-      message(STATUS "Not including bundled pybind11 (already configured)")
-    endif()
-  endif()
-endif()
-
 if(IREE_TARGET_BACKEND_METAL_SPIRV)
   # SPIRV-Cross is needed to cross compile SPIR-V into MSL source code.
   iree_set_spirv_cross_cmake_options()

compiler/pyproject.toml

@@ -3,15 +3,10 @@ requires = [
     "setuptools>=42",
     "wheel",
     "cmake",
-    # Note that the compiler wheel does not presently need nanobind, but
-    # it's build is enabled by the same flag which enables the runtime
-    # configuration, which does.
-    "nanobind==2.2.0",
     "ninja",
     # MLIR build depends.
     "numpy",
     "packaging",
-    "pybind11==2.13.6",
     "sympy",
 ]
 build-backend = "setuptools.build_meta"

runtime/bindings/python/CMakeLists.txt

@@ -4,21 +4,14 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-if(NOT nanobind_FOUND)
-  find_package(nanobind CONFIG QUIET)
-  if(NOT nanobind_FOUND)
-    execute_process(
-      COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-      OUTPUT_VARIABLE NB_DIR
-      RESULT_VARIABLE RC)
-    if(RC AND NOT RC EQUAL 0)
-      message(WARNING "Probing for nanobind failed. Please install the project's Python dependencies or '${Python_EXECUTABLE} -m pip install nanobind'")
-    endif()
-    list(APPEND CMAKE_PREFIX_PATH "${NB_DIR}")
-  endif()
-  find_package(nanobind CONFIG REQUIRED)
-endif()
+# nanobind
+include(FetchContent)
+FetchContent_Declare(
+  nanobind
+  GIT_REPOSITORY https://github.com/wjakob/nanobind.git
+  GIT_TAG        784efa2a0358a4dc5432c74f5685ee026e20f2b6 # 2.2.0
+)
+FetchContent_MakeAvailable(nanobind)
 
 set(_EXTRA_INSTALL_TOOL_TARGETS)
 set(_TRACY_ENABLED OFF)

runtime/bindings/python/iree/runtime/build_requirements.txt

@@ -5,12 +5,7 @@
 
 pip>=21.3
 setuptools>=62.4.0
-nanobind==2.2.0
 numpy>=2.0.0b1
 requests>=2.28.0
 wheel>=0.36.2
 sympy==1.12.1
-
-# TODO: nanobind is used in the runtime but the compiler uses pybind and
-# removing this breaks CI bots; remove this.
-pybind11==2.13.6

runtime/pyproject.toml

@@ -3,7 +3,6 @@ requires = [
     "setuptools>=42",
     "wheel",
     "cmake",
-    "nanobind==2.2.0",
     "ninja",
     "numpy>=2.0.0b1",
     "packaging",