From b86c6d4b52c5876a0f8841318ea6b76d3c0349d2 Mon Sep 17 00:00:00 2001 From: cmidgley Date: Wed, 3 Apr 2024 14:39:41 -0400 Subject: [PATCH 1/9] Add --ipn feature to set IPN on new pats - Add --ipn to cli - When part created and --ipn set, set IPN to the SKU (PartImporter.create_manufacturer_part) --- inventree_part_import/cli.py | 7 ++++++- inventree_part_import/part_importer.py | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/inventree_part_import/cli.py b/inventree_part_import/cli.py index 807bc9f..440e1ae 100644 --- a/inventree_part_import/cli.py +++ b/inventree_part_import/cli.py @@ -57,6 +57,7 @@ def wrapper(*args, **kwargs): @click.option("-d", "--dry", is_flag=True, help="Run without modifying InvenTree database.") @click.option("-c", "--config-dir", help="Override path to config directory.") @click.option("-v", "--verbose", is_flag=True, help="Enable verbose output for debugging.") +@click.option("--ipn", is_flag=True, help="If a new part is being created, set the IPN to the search part number.") @click.option("--show-config-dir", is_flag=True, help="Show path to config directory and exit.") @click.option("--configure", type=AvailableSuppliersChoices, help="Configure supplier.") @click.option("--update", metavar="CATEGORY", help="Update all parts from InvenTree CATEGORY.") @@ -72,6 +73,7 @@ def inventree_part_import( only=None, interactive="false", dry=False, + ipn=False, config_dir=False, verbose=False, show_config_dir=False, @@ -147,6 +149,9 @@ def inventree_part_import( if not verbose: error_helper.INFO_END = "\r" + + if ipn: + hint("--ipn will set new parts IPN to the search part number.") if dry: warning(DRY_MODE_WARNING, prefix="") @@ -191,7 +196,7 @@ def inventree_part_import( # make sure suppliers.yaml exists get_suppliers(reload=True) setup_supplier_companies(inventree_api) - importer = PartImporter(inventree_api, interactive=interactive == "true", verbose=verbose) + importer = PartImporter(inventree_api, interactive=interactive == "true", verbose=verbose, ipn=ipn) if update or update_recursive: info(f"updating {len(parts)} parts from '{category_path}'", end="\n") diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index ba36317..91706ee 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -29,11 +29,12 @@ def __or__(self, other): return self if self.value < other.value else other class PartImporter: - def __init__(self, inventree_api, interactive=False, verbose=False): + def __init__(self, inventree_api, interactive=False, verbose=False, ipn=False): self.api = inventree_api self.interactive = interactive self.verbose = verbose self.dry_run = hasattr(inventree_api, "DRY_RUN") + self.ipn = ipn # preload pre_creation_hooks get_pre_creation_hooks() @@ -246,8 +247,8 @@ def create_manufacturer_part( category.add_alias(api_part.category_path[-1]) self.category_map[api_part.category_path[-1].lower()] = category - info(f"creating part {api_part.MPN} in '{category.part_category.pathstring}' ...") - part = Part.create(self.api, {"category": category.part_category.pk, **part_data}) + info(f"creating part {api_part.MPN} in '{category.part_category.pathstring}' ...") + part = Part.create(self.api, {"category": category.part_category.pk, **({"IPN": api_part.SKU} if self.api else {}),**part_data}) manufacturer = create_manufacturer(self.api, api_part.manufacturer) info(f"creating manufacturer part {api_part.MPN} ...") @@ -458,7 +459,7 @@ def update_parameter(parameter, value): parameter.save({"data": value}) except HTTPError as e: msg = e.args[0]["body"] - return f"failed to update parameter '{parameter.name}' to '{value}' with '{msg}'" + return f"failed to update parameter '{getattr(parameter, 'name', '(unknown)')}' to '{value}' with '{msg}'" SANITIZE_PARAMETER = re.compile("±") From da2927e8fe991a98dbd65a790a341e566235cb1e Mon Sep 17 00:00:00 2001 From: cmidgley Date: Wed, 3 Apr 2024 15:23:15 -0400 Subject: [PATCH 2/9] Adjust help text --- inventree_part_import/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventree_part_import/cli.py b/inventree_part_import/cli.py index 440e1ae..7413771 100644 --- a/inventree_part_import/cli.py +++ b/inventree_part_import/cli.py @@ -57,7 +57,7 @@ def wrapper(*args, **kwargs): @click.option("-d", "--dry", is_flag=True, help="Run without modifying InvenTree database.") @click.option("-c", "--config-dir", help="Override path to config directory.") @click.option("-v", "--verbose", is_flag=True, help="Enable verbose output for debugging.") -@click.option("--ipn", is_flag=True, help="If a new part is being created, set the IPN to the search part number.") +@click.option("--ipn", is_flag=True, help="When import creates a new part, sets the IPN to the search part number.") @click.option("--show-config-dir", is_flag=True, help="Show path to config directory and exit.") @click.option("--configure", type=AvailableSuppliersChoices, help="Configure supplier.") @click.option("--update", metavar="CATEGORY", help="Update all parts from InvenTree CATEGORY.") From e40de3d104fbdbd6d0c485cb8982cb14c2e59127 Mon Sep 17 00:00:00 2001 From: cmidgley Date: Thu, 4 Apr 2024 06:38:25 -0400 Subject: [PATCH 3/9] Revert fix to update_parameter - moving to a different branch for separate pull --- inventree_part_import/part_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index 91706ee..805d0ed 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -459,7 +459,7 @@ def update_parameter(parameter, value): parameter.save({"data": value}) except HTTPError as e: msg = e.args[0]["body"] - return f"failed to update parameter '{getattr(parameter, 'name', '(unknown)')}' to '{value}' with '{msg}'" + return f"failed to update parameter '{parameter.name}' to '{value}' with '{msg}'" SANITIZE_PARAMETER = re.compile("±") From 3af1697e7423340fc7b36b1f8ab9fb5938dda246 Mon Sep 17 00:00:00 2001 From: cmidgley Date: Thu, 4 Apr 2024 06:43:15 -0400 Subject: [PATCH 4/9] Clean up endline spaces --- inventree_part_import/part_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index 805d0ed..ff6224e 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -247,7 +247,7 @@ def create_manufacturer_part( category.add_alias(api_part.category_path[-1]) self.category_map[api_part.category_path[-1].lower()] = category - info(f"creating part {api_part.MPN} in '{category.part_category.pathstring}' ...") + info(f"creating part {api_part.MPN} in '{category.part_category.pathstring}' ...") part = Part.create(self.api, {"category": category.part_category.pk, **({"IPN": api_part.SKU} if self.api else {}),**part_data}) manufacturer = create_manufacturer(self.api, api_part.manufacturer) From 569ad697d9525e172974417b302273318ff448a5 Mon Sep 17 00:00:00 2001 From: cmidgley Date: Thu, 4 Apr 2024 07:57:50 -0400 Subject: [PATCH 5/9] Fix crash when importing parts and parameters don't validate - pass 'name' instead of trying to deref 'parameter.name' --- inventree_part_import/part_importer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index ba36317..d4ce954 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -378,7 +378,7 @@ def setup_parameters(self, part, api_part: ApiPart, update_existing=True): if existing_parameter := existing_parameters.get(name): if update_existing and existing_parameter.data != value: async_results.append(thread_pool.apply_async( - update_parameter, (existing_parameter, value) + update_parameter, (existing_parameter, name, value) )) else: if parameter_template := self.parameter_templates.get(name): @@ -453,12 +453,12 @@ def create_parameter(inventree_api, part, parameter_template, value): msg = e.args[0]["body"] return f"failed to create parameter '{parameter_template.name}' with '{msg}'" -def update_parameter(parameter, value): +def update_parameter(parameter, name, value): try: parameter.save({"data": value}) except HTTPError as e: msg = e.args[0]["body"] - return f"failed to update parameter '{parameter.name}' to '{value}' with '{msg}'" + return f"failed to update parameter '{name}' to '{value}' with '{msg}'" SANITIZE_PARAMETER = re.compile("±") From 80586b3210b113a80dece7769ad9806a4bf7c005 Mon Sep 17 00:00:00 2001 From: Bobbe <34186858+30350n@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:09:46 +0200 Subject: [PATCH 6/9] Remove name argument from update_parameter again --- inventree_part_import/part_importer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index d4ce954..036c648 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -378,7 +378,7 @@ def setup_parameters(self, part, api_part: ApiPart, update_existing=True): if existing_parameter := existing_parameters.get(name): if update_existing and existing_parameter.data != value: async_results.append(thread_pool.apply_async( - update_parameter, (existing_parameter, name, value) + update_parameter, (existing_parameter, value) )) else: if parameter_template := self.parameter_templates.get(name): @@ -453,12 +453,13 @@ def create_parameter(inventree_api, part, parameter_template, value): msg = e.args[0]["body"] return f"failed to create parameter '{parameter_template.name}' with '{msg}'" -def update_parameter(parameter, name, value): +def update_parameter(parameter, value): try: parameter.save({"data": value}) except HTTPError as e: msg = e.args[0]["body"] - return f"failed to update parameter '{name}' to '{value}' with '{msg}'" + parameter_name = parameter.template_detail["name"] + return f"failed to update parameter '{parameter_name}' to '{value}' with '{msg}'" SANITIZE_PARAMETER = re.compile("±") From 8b43d495965ed3f79095e8e71a7d200c6f304dc1 Mon Sep 17 00:00:00 2001 From: cmidgley Date: Sun, 5 May 2024 12:20:52 -0400 Subject: [PATCH 7/9] Rework IPN to be based on templates with part-category inheritance --- README.md | 46 +++++++++++++++ inventree_part_import/categories.py | 10 ++-- inventree_part_import/cli.py | 14 ++--- inventree_part_import/config/__init__.py | 1 + inventree_part_import/part_importer.py | 75 +++++++++++++++++++++++- pyproject.toml | 1 + 6 files changed, 134 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f35daf3..c5fcf6d 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ The following parameters have to be set: (set to null to disable) - `interactive_category_matches`: the maximum number of categories to display in interactive mode - `interactive_parameter_matches`: the maximum number of parameters to display in interactive mode +- `ipn_template`: Optional default template for defining IPN part numbers. See [IPN Templates](ipn_templates). - `part_selection_format`: standard python format str used to format each line of the interactive part selection menu (any fields from the `ApiPart` dataclass can be used, defaults to: `"{MPN} | {manufacturer} | {SKU} | {supplier_link}"`) @@ -147,6 +148,7 @@ Additionally you can define the following meta attributes (starting with `_`): - `_aliases` has to be a list of supplier category names which will be mapped to that category - `_description` specifies the categories description (defaults to category name) - `_ignore` makes `inventree-part-import` ignore that category, as well as any subcategories +- `_ipn_template` specifies a template to use for defining IPN part numbers (see [IPN Templates](#ipn-templates)) - `_parameters` has to be a list of parameter names (for parameters defined in [`parameters.yaml`](#parametersyaml)) this category uses
**note: parameters get inherited by sub categories** @@ -195,6 +197,50 @@ Input Voltage: _unit: V # experimental, this can lead to import problems ``` +### IPN Templates + +You can optionally use IPN templates to define a custom IPN name on parts. If you do not configure +any templates, the IPN value is not used. When templates are defined, which are standard Jinja2 +templates, the template result along with the CLI option `--ipn never|new|always`, are used to +define the IPN value. You can have a single default template for all imports, or customize the +template per category in the heirarchy. + +Templates have several context variables available: + +- `category`: the category name of the part +- `manufacturer`: the name of the part manufacturer +- `parameters`: a dictionary of all parameters +- `part_id`: a unique number (the primary ID of the part), useful to create unique numbers +- `MPN`: the manufacturer part number +- `SKU`: the suppliers part SKU +- `supplier`: the name of the supplier + +Some examples: +- `PN-{{ part_id }}`: A unique ID such as `PN-382` +- `{{ supplier }}-{{ SKU }}`: A combination of supplier name and SKU, such as `LCSC-C38221` +- `{{ parameters.Resistance }}-{{parameters.Wattage }}-{{parameters["Package Type"] }}`: A name + built from parameters, such as `18.2K-0.25W-0603`. + + > Some vendors have more consistent parameters than others, so consider using the `--dry` CLI + > option on several parts which will show the template results without updating the database. + + > A missing value for a context variable, such as a parameter that doesn't exist, will result in an empty value. + > Template values are filtered to remove all leading, trailing, and duplicate common + > separator values (`-`, `_`, and ` `), to avoid names like `RES---322` when parameter values are + > not matched. + +The first supplier that finds a matching part will be used to define the context variables for the +template (for example, the parameters from the first successful supplier search). Use the `-s ` option to always search a specific supplier first. + +You optionally specify category-specific tempates in +`(categories.yaml)[categoriesyaml]` using `_ipn_template`. For example, `Resistor` might have +`RES-{{ parameters.Resistance }}` whereas `Capacitor` might use `CAP-{{ parameters.Capacitance }}`. +Templates are searched in hierarchical order, starting with the closest category and working up the +tree to the top level. If no category template is found, the default template +in `(config.yaml)[configyaml]` under `ipn_template` is used. If no template is found, the IPN number will not be +added. Use the `--ipn never|new|always` CLI option for runtime control, where `new` +is the default behavior (only add an IPN if the part does not already have one) + ### Pre Creation Hooks (`hooks.py`) Pre creation hooks are functions that get run after part information has been parsed from a diff --git a/inventree_part_import/categories.py b/inventree_part_import/categories.py index 07bca0b..c7e78d6 100644 --- a/inventree_part_import/categories.py +++ b/inventree_part_import/categories.py @@ -3,7 +3,7 @@ from inventree.part import ParameterTemplate, PartCategory, PartCategoryParameterTemplate from .config import (CATEGORIES_CONFIG, PARAMETERS_CONFIG, get_categories_config, - get_parameters_config, update_config_file) + get_parameters_config, update_config_file, get_config) from .error_helper import * def setup_categories_and_parameters(inventree_api): @@ -161,6 +161,7 @@ class Category: ignore: bool structural: bool aliases: list[str] = field(default_factory=list) + ipn_template: str = "" parameters: list[str] = field(default_factory=list) part_category: PartCategory = None @@ -194,8 +195,8 @@ def add_alias(self, alias): f"'{CATEGORIES_CONFIG}'" ) -CATEGORY_ATTRIBUTES = {"_parameters", "_description", "_ignore", "_structural", "_aliases"} -def parse_category_recursive(categories_dict, parameters=tuple(), path=tuple()): +CATEGORY_ATTRIBUTES = {"_parameters", "_description", "_ignore", "_structural", "_aliases", "_ipn_template"} +def parse_category_recursive(categories_dict, parameters=tuple(), path=tuple(), parent=None): if not categories_dict: return {} @@ -224,10 +225,11 @@ def parse_category_recursive(categories_dict, parameters=tuple(), path=tuple()): ignore=values.get("_ignore", False), structural=values.get("_structural", False), aliases=values.get("_aliases", []), + ipn_template=values.get("_ipn_template", get_config().get("ipn_template", "") if parent is None else parent.ipn_template), parameters=new_parameters, ) - categories.update(parse_category_recursive(values, new_parameters, new_path)) + categories.update(parse_category_recursive(values, new_parameters, new_path, categories[new_path])) return categories diff --git a/inventree_part_import/cli.py b/inventree_part_import/cli.py index 0decde4..b35bd8c 100644 --- a/inventree_part_import/cli.py +++ b/inventree_part_import/cli.py @@ -14,7 +14,7 @@ setup_inventree_api, update_config_file, update_supplier_config) from .error_helper import * from .inventree_helpers import get_category, get_category_parts -from .part_importer import ImportResult, PartImporter +from .part_importer import ImportResult, PartImporter, IPNSetting from .suppliers import get_suppliers, setup_supplier_companies def handle_errors(func): @@ -44,6 +44,7 @@ def wrapper(*args, **kwargs): AvailableSuppliersChoices = click.Choice(_available_suppliers.keys(), case_sensitive=False) InteractiveChoices = click.Choice(("default", "false", "true", "twice"), case_sensitive=False) +IPNChoices = click.Choice(("new", "never", "always"), case_sensitive=False) @click.command @click.pass_context @@ -57,7 +58,9 @@ def wrapper(*args, **kwargs): @click.option("-d", "--dry", is_flag=True, help="Run without modifying InvenTree database.") @click.option("-c", "--config-dir", help="Override path to config directory.") @click.option("-v", "--verbose", is_flag=True, help="Enable verbose output for debugging.") -@click.option("--ipn", is_flag=True, help="When import creates a new part, sets the IPN to the search part number.") +@click.option("--ipn", type=IPNChoices, default="new", help="Update IPN mode. 'new' (default) will add IPN only when " + "part has none, 'never' will never update IPN and 'always' will update it for all parts. Requires IPN templates " + "to have been configured.") @click.option("--show-config-dir", is_flag=True, help="Show path to config directory and exit.") @click.option("--configure", type=AvailableSuppliersChoices, help="Configure supplier.") @click.option("--update", metavar="CATEGORY", help="Update all parts from InvenTree CATEGORY.") @@ -73,7 +76,7 @@ def inventree_part_import( only=None, interactive="false", dry=False, - ipn=False, + ipn="new", config_dir=False, verbose=False, show_config_dir=False, @@ -149,9 +152,6 @@ def inventree_part_import( if not verbose: error_helper.INFO_END = "\r" - - if ipn: - hint("--ipn will set new parts IPN to the search part number.") if dry: warning(DRY_MODE_WARNING, prefix="") @@ -196,7 +196,7 @@ def inventree_part_import( # make sure suppliers.yaml exists get_suppliers(reload=True) setup_supplier_companies(inventree_api) - importer = PartImporter(inventree_api, interactive=interactive == "true", verbose=verbose, ipn=ipn) + importer = PartImporter(inventree_api, interactive=interactive == "true", verbose=verbose, ipn = IPNSetting[ipn.upper()]) if update or update_recursive: info(f"updating {len(parts)} parts from '{category_path}'", end="\n") diff --git a/inventree_part_import/config/__init__.py b/inventree_part_import/config/__init__.py index a961c12..ca1a3d4 100644 --- a/inventree_part_import/config/__init__.py +++ b/inventree_part_import/config/__init__.py @@ -115,6 +115,7 @@ def setup_inventree_api(): "datasheets", "interactive_category_matches", "interactive_parameter_matches", + "ipn_template", "part_selection_format", "auto_detect_columns", *DEFAULT_CONFIG_VARS, diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index 2f05f4f..ccb7bf1 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -9,6 +9,7 @@ from requests.compat import quote from requests.exceptions import HTTPError from thefuzz import fuzz +from jinja2 import Template from .categories import setup_categories_and_parameters from .config import CATEGORIES_CONFIG, CONFIG, get_config, get_pre_creation_hooks @@ -28,8 +29,13 @@ class ImportResult(Enum): def __or__(self, other): return self if self.value < other.value else other +class IPNSetting(Enum): + NEVER = 0 + NEW = 1 + ALWAYS = 2 + class PartImporter: - def __init__(self, inventree_api, interactive=False, verbose=False, ipn=False): + def __init__(self, inventree_api, interactive=False, verbose=False, ipn=IPNSetting.NEW): self.api = inventree_api self.interactive = interactive self.verbose = verbose @@ -59,6 +65,7 @@ def import_part( import_result = ImportResult.SUCCESS self.existing_manufacturer_part = None + self.existing_part = None search_results = search(search_term, supplier_id, only_supplier) for supplier, async_results in search_results: info(f"searching at {supplier.name} ...") @@ -107,6 +114,9 @@ def import_part( other_results.wait() return ImportResult.ERROR + if not self.import_part_ipn(api_part, supplier): + import_result |= ImportResult + if not self.existing_manufacturer_part: import_result |= ImportResult.FAILURE @@ -143,6 +153,66 @@ def select_api_part(api_parts: list[ApiPart]): index = select(choices, deselected_prefix=" ", selected_prefix="> ") return [*api_parts, None][index] + def import_part_ipn(self, api_part, supplier): + # only add IPN if --ipn ALWAYS, or --ipn NEW and part doesn't yet have an IPN + if self.ipn == IPNSetting.NEVER: + return True + if self.ipn == IPNSetting.NEW and hasattr(self.existing_part, "IPN") and self.existing_part.IPN: + return True + + # find the outer most mapped category (the leaf category) + category = None + for subcategory in reversed(api_part.category_path): + mapped_subcategory = self.category_map.get(subcategory.lower()) + if mapped_subcategory: + category = mapped_subcategory + break + else: + return True + + # locate the closest template in category heirarchy + ipn_template = None + for subcategory in reversed(category.path): + mapped_subcategory = self.category_map.get(subcategory.lower()) + if mapped_subcategory: # and hasattr(mapped_subcategory, "ipn_template") and mapped_subcategory.ipn_template and mapped_subcategory.ipn_template.strip(): + ipn_template = mapped_subcategory.ipn_template + break + else: + return True + + # set up jinja2 context + context = { + "part_id": self.existing_part.pk, + "supplier": supplier.name, + "category": category.name if category else "", + "parameters": api_part.parameters, + **{attr: getattr(api_part, attr) for attr in dir(api_part) if not attr.startswith('_') and not callable(getattr(api_part, attr))}, + } + + # render the IPN template + try: + template = Template(ipn_template) + ipn = template.render(context) + except Exception as e: + error(f"failed to render IPN template '{ipn_template}' with: {e}") + return False + + # if we got a resulting ipn, other than just separators, update the part + ipn = ipn.strip(" -_") + if ipn: + # remove any duplicate separators + ipn = re.sub(r'\s+', ' ', ipn) + ipn = re.sub(r'-+', '-', ipn) + ipn = re.sub(r'_+', '_', ipn) + + if not self.dry_run: + update_object_data(self.existing_part, {"IPN": ipn}) + + if self.verbose or self.dry_run: + info(f"set part IPN to '{ipn}' using template '{ipn_template}'") + + return True + def import_supplier_part(self, supplier: Company, api_part: ApiPart, part: Part = None): import_result = ImportResult.SUCCESS @@ -201,6 +271,7 @@ def import_supplier_part(self, supplier: Company, api_part: ApiPart, part: Part import_result |= result self.existing_manufacturer_part = manufacturer_part + self.existing_part = part supplier_part_data = { "part": 0 if self.dry_run else part.pk, @@ -248,7 +319,7 @@ def create_manufacturer_part( self.category_map[api_part.category_path[-1].lower()] = category info(f"creating part {api_part.MPN} in '{category.part_category.pathstring}' ...") - part = Part.create(self.api, {"category": category.part_category.pk, **({"IPN": api_part.SKU} if self.api else {}),**part_data}) + part = Part.create(self.api, {"category": category.part_category.pk, **part_data}) manufacturer = create_manufacturer(self.api, api_part.manufacturer) info(f"creating manufacturer part {api_part.MPN} ...") diff --git a/pyproject.toml b/pyproject.toml index 4d51c69..ad0ddae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "fake-useragent", "inventree>=0.13.2", "isocodes", + "jinja2", "mouser>=0.1.5", "platformdirs>=3.2.0", "pyyaml", From 68f40ac546664850ee13d15bdabb80e76fc438c7 Mon Sep 17 00:00:00 2001 From: Bobbe Date: Wed, 8 May 2024 03:03:16 +0200 Subject: [PATCH 8/9] Format code, Rename ipn_template to ipn_format, Refactor IPNSetting --- README.md | 25 ++++++++--------- inventree_part_import/categories.py | 18 ++++++++----- inventree_part_import/cli.py | 14 +++++----- inventree_part_import/config/__init__.py | 2 +- inventree_part_import/part_importer.py | 34 ++++++++++++------------ 5 files changed, 50 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index c5fcf6d..7ad3d6f 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ The following parameters have to be set: (set to null to disable) - `interactive_category_matches`: the maximum number of categories to display in interactive mode - `interactive_parameter_matches`: the maximum number of parameters to display in interactive mode -- `ipn_template`: Optional default template for defining IPN part numbers. See [IPN Templates](ipn_templates). +- `ipn_format`: Optional default template for defining IPN part numbers. See [IPN Templates](ipn_formats). - `part_selection_format`: standard python format str used to format each line of the interactive part selection menu (any fields from the `ApiPart` dataclass can be used, defaults to: `"{MPN} | {manufacturer} | {SKU} | {supplier_link}"`) @@ -148,7 +148,7 @@ Additionally you can define the following meta attributes (starting with `_`): - `_aliases` has to be a list of supplier category names which will be mapped to that category - `_description` specifies the categories description (defaults to category name) - `_ignore` makes `inventree-part-import` ignore that category, as well as any subcategories -- `_ipn_template` specifies a template to use for defining IPN part numbers (see [IPN Templates](#ipn-templates)) +- `_ipn_format` specifies a template to use for defining IPN part numbers (see [IPN Templates](#ipn-templates)) - `_parameters` has to be a list of parameter names (for parameters defined in [`parameters.yaml`](#parametersyaml)) this category uses
**note: parameters get inherited by sub categories** @@ -203,7 +203,7 @@ You can optionally use IPN templates to define a custom IPN name on parts. If y any templates, the IPN value is not used. When templates are defined, which are standard Jinja2 templates, the template result along with the CLI option `--ipn never|new|always`, are used to define the IPN value. You can have a single default template for all imports, or customize the -template per category in the heirarchy. +template per category in the hierarchy. Templates have several context variables available: @@ -216,28 +216,29 @@ Templates have several context variables available: - `supplier`: the name of the supplier Some examples: + - `PN-{{ part_id }}`: A unique ID such as `PN-382` - `{{ supplier }}-{{ SKU }}`: A combination of supplier name and SKU, such as `LCSC-C38221` - `{{ parameters.Resistance }}-{{parameters.Wattage }}-{{parameters["Package Type"] }}`: A name - built from parameters, such as `18.2K-0.25W-0603`. - + built from parameters, such as `18.2K-0.25W-0603`. + > Some vendors have more consistent parameters than others, so consider using the `--dry` CLI - > option on several parts which will show the template results without updating the database. - + > option on several parts which will show the template results without updating the database. + > A missing value for a context variable, such as a parameter that doesn't exist, will result in an empty value. > Template values are filtered to remove all leading, trailing, and duplicate common - > separator values (`-`, `_`, and ` `), to avoid names like `RES---322` when parameter values are + > separator values (`-`, `_`, and spaces), to avoid names like `RES---322` when parameter values are > not matched. The first supplier that finds a matching part will be used to define the context variables for the template (for example, the parameters from the first successful supplier search). Use the `-s ` option to always search a specific supplier first. -You optionally specify category-specific tempates in -`(categories.yaml)[categoriesyaml]` using `_ipn_template`. For example, `Resistor` might have +You optionally specify category-specific templates in +`(categories.yaml)[categoriesyaml]` using `_ipn_format`. For example, `Resistor` might have `RES-{{ parameters.Resistance }}` whereas `Capacitor` might use `CAP-{{ parameters.Capacitance }}`. Templates are searched in hierarchical order, starting with the closest category and working up the -tree to the top level. If no category template is found, the default template -in `(config.yaml)[configyaml]` under `ipn_template` is used. If no template is found, the IPN number will not be +tree to the top level. If no category template is found, the default template +in `(config.yaml)[configyaml]` under `ipn_format` is used. If no template is found, the IPN number will not be added. Use the `--ipn never|new|always` CLI option for runtime control, where `new` is the default behavior (only add an IPN if the part does not already have one) diff --git a/inventree_part_import/categories.py b/inventree_part_import/categories.py index c7e78d6..1a34b0d 100644 --- a/inventree_part_import/categories.py +++ b/inventree_part_import/categories.py @@ -2,8 +2,8 @@ from inventree.part import ParameterTemplate, PartCategory, PartCategoryParameterTemplate -from .config import (CATEGORIES_CONFIG, PARAMETERS_CONFIG, get_categories_config, - get_parameters_config, update_config_file, get_config) +from .config import (CATEGORIES_CONFIG, PARAMETERS_CONFIG, get_categories_config, get_config, + get_parameters_config, update_config_file) from .error_helper import * def setup_categories_and_parameters(inventree_api): @@ -161,7 +161,7 @@ class Category: ignore: bool structural: bool aliases: list[str] = field(default_factory=list) - ipn_template: str = "" + ipn_format: str = "" parameters: list[str] = field(default_factory=list) part_category: PartCategory = None @@ -195,7 +195,9 @@ def add_alias(self, alias): f"'{CATEGORIES_CONFIG}'" ) -CATEGORY_ATTRIBUTES = {"_parameters", "_description", "_ignore", "_structural", "_aliases", "_ipn_template"} +CATEGORY_ATTRIBUTES = { + "_parameters", "_description", "_ignore", "_structural", "_aliases", "_ipn_format", +} def parse_category_recursive(categories_dict, parameters=tuple(), path=tuple(), parent=None): if not categories_dict: return {} @@ -215,21 +217,23 @@ def parse_category_recursive(categories_dict, parameters=tuple(), path=tuple(), if child.startswith("_") and child not in CATEGORY_ATTRIBUTES: warning(f"ignoring unknown special attribute '{child}' in category '{name}'") + default_ipn_format = parent.ipn_format if parent else get_config().get("ipn_format") + new_parameters = parameters + tuple(values.get("_parameters", [])) new_path = path + (name,) - categories[new_path] = Category( + categories[new_path] = category = Category( name=name, path=list(new_path), description=values.get("_description", name), ignore=values.get("_ignore", False), structural=values.get("_structural", False), aliases=values.get("_aliases", []), - ipn_template=values.get("_ipn_template", get_config().get("ipn_template", "") if parent is None else parent.ipn_template), + ipn_format=values.get("_ipn_format", default_ipn_format), parameters=new_parameters, ) - categories.update(parse_category_recursive(values, new_parameters, new_path, categories[new_path])) + categories.update(parse_category_recursive(values, new_parameters, new_path, category)) return categories diff --git a/inventree_part_import/cli.py b/inventree_part_import/cli.py index b35bd8c..79c9c84 100644 --- a/inventree_part_import/cli.py +++ b/inventree_part_import/cli.py @@ -14,7 +14,7 @@ setup_inventree_api, update_config_file, update_supplier_config) from .error_helper import * from .inventree_helpers import get_category, get_category_parts -from .part_importer import ImportResult, PartImporter, IPNSetting +from .part_importer import ImportResult, IPNSetting, PartImporter from .suppliers import get_suppliers, setup_supplier_companies def handle_errors(func): @@ -44,7 +44,7 @@ def wrapper(*args, **kwargs): AvailableSuppliersChoices = click.Choice(_available_suppliers.keys(), case_sensitive=False) InteractiveChoices = click.Choice(("default", "false", "true", "twice"), case_sensitive=False) -IPNChoices = click.Choice(("new", "never", "always"), case_sensitive=False) +IPNChoices = click.Choice(("false", "true", "overwrite"), case_sensitive=False) @click.command @click.pass_context @@ -55,12 +55,12 @@ def wrapper(*args, **kwargs): "Enable interactive mode. 'twice' will run once normally, then rerun in interactive " "mode for any parts that failed to import correctly." )) +@click.option("-I", "--ipn", type=IPNChoices, default="true", + help="Set internal part number according to ipn_format." +) @click.option("-d", "--dry", is_flag=True, help="Run without modifying InvenTree database.") @click.option("-c", "--config-dir", help="Override path to config directory.") @click.option("-v", "--verbose", is_flag=True, help="Enable verbose output for debugging.") -@click.option("--ipn", type=IPNChoices, default="new", help="Update IPN mode. 'new' (default) will add IPN only when " - "part has none, 'never' will never update IPN and 'always' will update it for all parts. Requires IPN templates " - "to have been configured.") @click.option("--show-config-dir", is_flag=True, help="Show path to config directory and exit.") @click.option("--configure", type=AvailableSuppliersChoices, help="Configure supplier.") @click.option("--update", metavar="CATEGORY", help="Update all parts from InvenTree CATEGORY.") @@ -196,7 +196,9 @@ def inventree_part_import( # make sure suppliers.yaml exists get_suppliers(reload=True) setup_supplier_companies(inventree_api) - importer = PartImporter(inventree_api, interactive=interactive == "true", verbose=verbose, ipn = IPNSetting[ipn.upper()]) + importer = PartImporter( + inventree_api, interactive=interactive == "true", verbose=verbose, ipn=IPNSetting[ipn] + ) if update or update_recursive: info(f"updating {len(parts)} parts from '{category_path}'", end="\n") diff --git a/inventree_part_import/config/__init__.py b/inventree_part_import/config/__init__.py index ca1a3d4..8f22b0d 100644 --- a/inventree_part_import/config/__init__.py +++ b/inventree_part_import/config/__init__.py @@ -115,7 +115,7 @@ def setup_inventree_api(): "datasheets", "interactive_category_matches", "interactive_parameter_matches", - "ipn_template", + "ipn_format", "part_selection_format", "auto_detect_columns", *DEFAULT_CONFIG_VARS, diff --git a/inventree_part_import/part_importer.py b/inventree_part_import/part_importer.py index ccb7bf1..54296d3 100644 --- a/inventree_part_import/part_importer.py +++ b/inventree_part_import/part_importer.py @@ -6,10 +6,10 @@ from cutie import select from inventree.company import Company, ManufacturerPart, SupplierPart, SupplierPriceBreak from inventree.part import Parameter, Part +from jinja2 import Template from requests.compat import quote from requests.exceptions import HTTPError from thefuzz import fuzz -from jinja2 import Template from .categories import setup_categories_and_parameters from .config import CATEGORIES_CONFIG, CONFIG, get_config, get_pre_creation_hooks @@ -29,13 +29,10 @@ class ImportResult(Enum): def __or__(self, other): return self if self.value < other.value else other -class IPNSetting(Enum): - NEVER = 0 - NEW = 1 - ALWAYS = 2 +IPNSetting = Enum("false", "true", "overwrite") class PartImporter: - def __init__(self, inventree_api, interactive=False, verbose=False, ipn=IPNSetting.NEW): + def __init__(self, inventree_api, interactive=False, verbose=False, ipn=IPNSetting.true): self.api = inventree_api self.interactive = interactive self.verbose = verbose @@ -155,9 +152,9 @@ def select_api_part(api_parts: list[ApiPart]): def import_part_ipn(self, api_part, supplier): # only add IPN if --ipn ALWAYS, or --ipn NEW and part doesn't yet have an IPN - if self.ipn == IPNSetting.NEVER: + if self.ipn == IPNSetting.false: return True - if self.ipn == IPNSetting.NEW and hasattr(self.existing_part, "IPN") and self.existing_part.IPN: + if self.ipn == IPNSetting.true and getattr(self.existing_part, "IPN", None): return True # find the outer most mapped category (the leaf category) @@ -170,12 +167,11 @@ def import_part_ipn(self, api_part, supplier): else: return True - # locate the closest template in category heirarchy - ipn_template = None + # locate the closest template in category hierarchy + ipn_format = None for subcategory in reversed(category.path): - mapped_subcategory = self.category_map.get(subcategory.lower()) - if mapped_subcategory: # and hasattr(mapped_subcategory, "ipn_template") and mapped_subcategory.ipn_template and mapped_subcategory.ipn_template.strip(): - ipn_template = mapped_subcategory.ipn_template + if mapped_subcategory := self.category_map.get(subcategory.lower()): + ipn_format = mapped_subcategory.ipn_format break else: return True @@ -186,15 +182,19 @@ def import_part_ipn(self, api_part, supplier): "supplier": supplier.name, "category": category.name if category else "", "parameters": api_part.parameters, - **{attr: getattr(api_part, attr) for attr in dir(api_part) if not attr.startswith('_') and not callable(getattr(api_part, attr))}, + **{ + attr: getattr(api_part, attr) + for attr in dir(api_part) + if not attr.startswith('_') and not callable(getattr(api_part, attr)) + }, } # render the IPN template try: - template = Template(ipn_template) + template = Template(ipn_format) ipn = template.render(context) except Exception as e: - error(f"failed to render IPN template '{ipn_template}' with: {e}") + error(f"failed to render IPN template '{ipn_format}' with: {e}") return False # if we got a resulting ipn, other than just separators, update the part @@ -209,7 +209,7 @@ def import_part_ipn(self, api_part, supplier): update_object_data(self.existing_part, {"IPN": ipn}) if self.verbose or self.dry_run: - info(f"set part IPN to '{ipn}' using template '{ipn_template}'") + info(f"set part IPN to '{ipn}' using template '{ipn_format}'") return True From 66589ac4c204bc4e1efcab63ac8f594f0592f363 Mon Sep 17 00:00:00 2001 From: Bobbe Date: Wed, 8 May 2024 13:17:11 +0200 Subject: [PATCH 9/9] Clarify --ipn help string --- inventree_part_import/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventree_part_import/cli.py b/inventree_part_import/cli.py index 79c9c84..4c32bf3 100644 --- a/inventree_part_import/cli.py +++ b/inventree_part_import/cli.py @@ -56,7 +56,7 @@ def wrapper(*args, **kwargs): "mode for any parts that failed to import correctly." )) @click.option("-I", "--ipn", type=IPNChoices, default="true", - help="Set internal part number according to ipn_format." + help="Set internal part number according to the ipn_format config variable." ) @click.option("-d", "--dry", is_flag=True, help="Run without modifying InvenTree database.") @click.option("-c", "--config-dir", help="Override path to config directory.")