From 457ff5e3b4808553dcdd044d95d08e456a7c4508 Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 18 Dec 2024 23:46:18 -0500 Subject: [PATCH] extra_env_vars support fnmatch globs --- docs/notes/2.25.x.md | 3 ++- src/python/pants/engine/env_vars.py | 12 +++++++++++- src/python/pants/engine/environment_test.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index 95c506003ae..433a37fde05 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -21,6 +21,7 @@ Thank you to [Klayvio](https://www.klaviyo.com/) and [Normal Computing](https:// - Fixed a longstanding bug in the processing of [synthetic targets](https://www.pantsbuild.org/2.24/docs/writing-plugins/the-target-api/concepts#synthetic-targets-api). This fix has the side-effect of requiring immutability and hashability of scalar values in BUILD files, which was always assumed but not enforced. This may cause BUILD file parsing errors, if you have custom field types involving custom mutable data structures. See ([#21725](https://github.com/pantsbuild/pants/pull/21725)) for more. - [Fixed](https://github.com/pantsbuild/pants/pull/21665) bug where `pants --export-resolve= --export-py-generated-sources-in-resolve=` fails (see [#21659](https://github.com/pantsbuild/pants/issues/21659) for more info). - [Fixed](https://github.com/pantsbuild/pants/pull/21694) bug where an `archive` target is unable to produce a ZIP file with no extension (see [#21693](https://github.com/pantsbuild/pants/issues/21693) for more info). +- `extra_env_vars` (on many subsystems) now supports a generalised glob syntax using Python [fnmatch](https://docs.python.org/3/library/fnmatch.html) to construct patterns like `AWS_*`, `TF_*`, and `S2TESTS_*`. #### Remote Caching/Execution @@ -56,7 +57,7 @@ Fixed an error which was caused when the same tool appeaed in both the `--docker Strict adherence to the [schema of Helm OCI registry configuration](https://www.pantsbuild.org/2.25/reference/subsystems/helm#registries) is now required. Previously we did ad-hoc coercion of some field values, so that, e.g., you could provide a "true"/"false" string as a boolean value. Now we require actual booleans. -The `helm_infer.external_docker_images` glob syntax has been generalized. In addition to `*`, you can now use Python [fnmatch](https://docs.python.org/3/library/fnmatch.html) to construct matterns like `quay.io/*`. +The `helm_infer.external_docker_images` glob syntax has been generalized. In addition to `*`, you can now use Python [fnmatch](https://docs.python.org/3/library/fnmatch.html) to construct patterns like `quay.io/*`. Fixed a bug where linting with the Helm backend enabled could induce serialization errors with the [workunit-logger](https://www.pantsbuild.org/2.25/reference/subsystems/workunit-logger). diff --git a/src/python/pants/engine/env_vars.py b/src/python/pants/engine/env_vars.py index 58a54e783bb..1290eada5aa 100644 --- a/src/python/pants/engine/env_vars.py +++ b/src/python/pants/engine/env_vars.py @@ -3,6 +3,7 @@ from __future__ import annotations +import fnmatch import re from dataclasses import dataclass from typing import Dict, Optional, Sequence @@ -53,7 +54,8 @@ def check_and_set(name: str, value: Optional[str]): if name_value_match: check_and_set(name_value_match[1], name_value_match[2]) elif shorthand_re.match(env_var): - check_and_set(env_var, self.get(env_var)) + for name, value in self.get_or_match(env_var).items(): + check_and_set(name, value) else: raise ValueError( f"An invalid variable was requested via the --test-extra-env-var " @@ -62,6 +64,14 @@ def check_and_set(name: str, value: Optional[str]): return FrozenDict(env_var_subset) + def get_or_match(self, name_or_pattern: str) -> dict[str, str]: + """ + Get the value of an envvar if it has an exact match, otherwise all fnmatches. + """ + if name_or_pattern in self: + return {name_or_pattern: self.get(name_or_pattern)} + return {k: v for k, v in self.items() if fnmatch.fnmatch(k, name_or_pattern)} + @dataclass(frozen=True) class EnvironmentVarsRequest: diff --git a/src/python/pants/engine/environment_test.py b/src/python/pants/engine/environment_test.py index 8ffd3e67769..b579c24e000 100644 --- a/src/python/pants/engine/environment_test.py +++ b/src/python/pants/engine/environment_test.py @@ -37,3 +37,18 @@ def test_invalid_variable() -> None: "An invalid variable was requested via the --test-extra-env-var mechanism: 3INVALID" in str(exc) ) + + +def test_envvar_fnmatch() -> None: + """Test fnmatch patterns correctly pull in all matching envvars""" + + pants_env = CompleteEnvironmentVars({ + "LETTER_C": "prefix_char_match", + "LETTER_PI": "prefix", + }) + + char_match = pants_env.get_subset(["LETTER_?"]) + assert char_match == {"LETTER_C": "prefix_char_match"} + + multichar_match = pants_env.get_subset(["LETTER_*"]) + assert multichar_match == {"LETTER_C": "prefix_char_match", "LETTER_PI": "prefix"}