From 64ebf70cb6dc4e97421076da925bed9df6b9e49e Mon Sep 17 00:00:00 2001 From: Ganden Schaffner Date: Sat, 3 Aug 2024 00:00:00 -0700 Subject: [PATCH] Fix pip-sync --python-executable evaluating markers for the wrong environment Co-authored-by: chrysle <96722107+chrysle@users.noreply.github.com> --- piptools/scripts/sync.py | 49 ++++++++++++++++++++++++++++++++++++++-- piptools/sync.py | 46 +++++++++++++++++++++++++++++++++---- 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index b83e811cb..a6c0d850c 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -1,11 +1,14 @@ from __future__ import annotations import itertools +import json import os +import platform import shlex import shutil import sys from pathlib import Path +from subprocess import run # nosec from typing import cast import click @@ -13,6 +16,7 @@ from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import get_environment +from pip._vendor.packaging.markers import Environment from .. import sync from .._compat import Distribution, parse_requirements @@ -100,6 +104,9 @@ def cli( if python_executable: _validate_python_executable(python_executable) + environment = _get_environment(python_executable) + else: + environment = {} install_command = cast(InstallCommand, create_command("install")) options, _ = install_command.parse_args([]) @@ -113,7 +120,9 @@ def cli( ) try: - merged_requirements = sync.merge(requirements, ignore_conflicts=force) + merged_requirements = sync.merge( + requirements, ignore_conflicts=force, environment=environment + ) except PipToolsError as e: log.error(str(e)) sys.exit(2) @@ -128,7 +137,9 @@ def cli( local_only=python_executable is None, paths=paths, ) - to_install, to_uninstall = sync.diff(merged_requirements, installed_dists) + to_install, to_uninstall = sync.diff( + merged_requirements, installed_dists, environment + ) install_flags = _compose_install_flags( finder, @@ -177,6 +188,40 @@ def _validate_python_executable(python_executable: str) -> None: sys.exit(2) +def _get_environment(python_executable: str) -> Environment: + """ + Return the marker variables of an environment. + """ + # On all platforms, ``json.loads`` supports only UTF-8, UTF-16, and UTF-32. + # Prior to PEP 686, a Python subprocess's ``sys.stdout`` will not always + # default to using one of these, so we must set it. + if platform.system() != "Windows": + # Set it to UTF-8. (This is mostly unnecessary as on most Unix systems + # Python will already default to this; see + # https://peps.python.org/pep-0686/#backward-compatibility.) + subprocess_PYTHONIOENCODING = "utf8" + else: + # Set it to UTF-16 because: + # + # * On Windows, in this situation (i.e. when stdout is a pipe) + # ``sys.stdout``'s encoding will default to CP-1252, which is not + # supported by ``json.loads``. + # * ``pip`` emits mangled output with UTF-8 on Windows; see + # https://github.com/Textualize/rich/issues/2882. + subprocess_PYTHONIOENCODING = "utf16" + + return Environment( + **json.loads( + run( # nosec + [python_executable, "-m", "pip", "inspect"], + env={**os.environ, "PYTHONIOENCODING": subprocess_PYTHONIOENCODING}, + check=True, + capture_output=True, + ).stdout + )["environment"] + ) + + def _compose_install_flags( finder: PackageFinder, no_index: bool, diff --git a/piptools/sync.py b/piptools/sync.py index c1d690aee..d3faea64c 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -4,6 +4,7 @@ import os import sys import tempfile +from functools import wraps from subprocess import run # nosec from typing import Deque, Iterable, Mapping, ValuesView @@ -40,6 +41,38 @@ ] +def patch_match_markers() -> None: + """ + Monkey patches ``pip._internal.req.InstallRequirement.match_markers`` to + allow us to pass environment other than "extra". + """ + + @wraps(InstallRequirement.match_markers) + def match_markers( + self: InstallRequirement, + extras_requested: Iterable[str] | None = None, + environment: dict[str, str] = {}, + ) -> bool: + assert "extra" not in environment + + if not extras_requested: + # Provide an extra to safely evaluate the markers + # without matching any extra + extras_requested = ("",) + if self.markers is not None: + return any( + self.markers.evaluate({"extra": extra, **environment}) + for extra in extras_requested + ) + else: + return True + + InstallRequirement.match_markers = match_markers + + +patch_match_markers() + + def dependency_tree( installed_keys: Mapping[str, Distribution], root_key: str ) -> set[str]: @@ -93,7 +126,9 @@ def get_dists_to_ignore(installed: Iterable[Distribution]) -> list[str]: def merge( - requirements: Iterable[InstallRequirement], ignore_conflicts: bool + requirements: Iterable[InstallRequirement], + ignore_conflicts: bool, + environment: dict[str, str] = {}, ) -> ValuesView[InstallRequirement]: by_key: dict[str, InstallRequirement] = {} @@ -101,7 +136,7 @@ def merge( # Limitation: URL requirements are merged by precise string match, so # "file:///example.zip#egg=example", "file:///example.zip", and # "example==1.0" will not merge with each other - if ireq.match_markers(): + if ireq.match_markers(environment=environment): key = key_from_ireq(ireq) if not ignore_conflicts: @@ -158,6 +193,7 @@ def diff_key_from_req(req: Distribution) -> str: def diff( compiled_requirements: Iterable[InstallRequirement], installed_dists: Iterable[Distribution], + environment: dict[str, str] = {}, ) -> tuple[set[InstallRequirement], set[str]]: """ Calculate which packages should be installed or uninstalled, given a set @@ -172,13 +208,15 @@ def diff( pkgs_to_ignore = get_dists_to_ignore(installed_dists) for dist in installed_dists: key = diff_key_from_req(dist) - if key not in requirements_lut or not requirements_lut[key].match_markers(): + if key not in requirements_lut or not requirements_lut[key].match_markers( + environment=environment + ): to_uninstall.add(key) elif requirements_lut[key].specifier.contains(dist.version): satisfied.add(key) for key, requirement in requirements_lut.items(): - if key not in satisfied and requirement.match_markers(): + if key not in satisfied and requirement.match_markers(environment=environment): to_install.add(requirement) # Make sure to not uninstall any packages that should be ignored