From 695fad25f8e4768b9267b9d89c34a71f772aa69d Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 26 Dec 2024 12:55:26 -0500 Subject: [PATCH] perf: cache requires-python checks & skip debug logging The `requires-python` check is pretty fast, but when performed for 10000 links, the checks consume a nontrivial amount of time. For example, while installing (pre-cached + --dry-run) a pared down list of homeassistant dependencies (n=117), link evaluation took 15% of the total runtime, with check_requires_python() accounting for half (7.5%) of that. The cache can be kept pretty small as requires-python specifiers often repeat, and when they do change, it's often in chunks (or between entirely different packages). For example, setuptools has like 1500 links, but only ~12 different `requires-python` specifiers. In addition, _log_skipped_link() is a hot method and unfortunately expensive as it hashes the link on every call. Fortunately, we can return early when debug logging is not enabled. In the same homeassistant run, this saves 0.7% of the runtime. --- news/13128.feature.rst | 1 + src/pip/_internal/index/package_finder.py | 5 +++++ src/pip/_internal/utils/packaging.py | 1 + 3 files changed, 7 insertions(+) create mode 100644 news/13128.feature.rst diff --git a/news/13128.feature.rst b/news/13128.feature.rst new file mode 100644 index 00000000000..6985d78b87e --- /dev/null +++ b/news/13128.feature.rst @@ -0,0 +1 @@ +Cache ``python-requires`` checks while filtering potential installation candidates. diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index c10103320e3..85628ee5d7a 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -736,6 +736,11 @@ def _sort_links(self, links: Iterable[Link]) -> List[Link]: return no_eggs + eggs def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None: + # This is a hot method so don't waste time hashing links unless we're + # actually going to log 'em. + if not logger.isEnabledFor(logging.DEBUG): + return + entry = (link, result, detail) if entry not in self._logged_links: # Put the link at the end so the reason is more visible and because diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py index 4b8fa0fe397..caad70f7fd1 100644 --- a/src/pip/_internal/utils/packaging.py +++ b/src/pip/_internal/utils/packaging.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) +@functools.lru_cache(maxsize=32) def check_requires_python( requires_python: Optional[str], version_info: Tuple[int, ...] ) -> bool: