diff --git a/src/sirocco/core/_tasks/icon_task.py b/src/sirocco/core/_tasks/icon_task.py index cc5a2949..712c514d 100644 --- a/src/sirocco/core/_tasks/icon_task.py +++ b/src/sirocco/core/_tasks/icon_task.py @@ -7,7 +7,7 @@ import f90nml from sirocco.core.graph_items import Task -from sirocco.parsing._yaml_data_models import ConfigIconTaskSpecs +from sirocco.parsing.yaml_data_models import ConfigIconTaskSpecs @dataclass(kw_only=True) @@ -51,7 +51,7 @@ def update_core_namelists_from_workflow(self): self.core_namelists["icon_master.namelist"]["master_time_control_nml"].update( { "experimentStartDate": self.start_date.isoformat() + "Z", - "experimentStopDate": self.end_date.isoformat() + "Z", + "experimentStopDate": self.stop_date.isoformat() + "Z", } ) self.core_namelists["icon_master.namelist"]["master_nml"]["lrestart"] = any( diff --git a/src/sirocco/core/_tasks/shell_task.py b/src/sirocco/core/_tasks/shell_task.py index cc7dbdf8..d637b4d1 100644 --- a/src/sirocco/core/_tasks/shell_task.py +++ b/src/sirocco/core/_tasks/shell_task.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from sirocco.core.graph_items import Task -from sirocco.parsing._yaml_data_models import ConfigShellTaskSpecs +from sirocco.parsing.yaml_data_models import ConfigShellTaskSpecs @dataclass(kw_only=True) diff --git a/src/sirocco/core/graph_items.py b/src/sirocco/core/graph_items.py index 4ceba492..1103336b 100644 --- a/src/sirocco/core/graph_items.py +++ b/src/sirocco/core/graph_items.py @@ -4,7 +4,9 @@ from itertools import chain, product from typing import TYPE_CHECKING, Any, ClassVar, Self, TypeAlias, TypeVar, cast -from sirocco.parsing._yaml_data_models import ( +from sirocco.parsing.target_date import DateList, LagList, SameDate +from sirocco.parsing.when import WhenSpec +from sirocco.parsing.yaml_data_models import ( ConfigAvailableData, ConfigBaseDataSpecs, ConfigBaseTaskSpecs, @@ -12,12 +14,12 @@ if TYPE_CHECKING: from collections.abc import Iterator - from datetime import datetime from pathlib import Path from termcolor._types import Color - from sirocco.parsing._yaml_data_models import ( + from sirocco.parsing.cycling import CyclePoint + from sirocco.parsing.yaml_data_models import ( ConfigBaseData, ConfigCycleTask, ConfigCycleTaskWaitOn, @@ -73,8 +75,7 @@ class Task(ConfigBaseTaskSpecs, GraphItem): outputs: list[Data] = field(default_factory=list) wait_on: list[Task] = field(default_factory=list) config_rootdir: Path - start_date: datetime | None = None - end_date: datetime | None = None + cycle_point: CyclePoint _wait_on_specs: list[ConfigCycleTaskWaitOn] = field(default_factory=list, repr=False) @@ -90,8 +91,7 @@ def from_config( cls, config: ConfigTask, config_rootdir: Path, - start_date: datetime | None, - end_date: datetime | None, + cycle_point: CyclePoint, coordinates: dict[str, Any], datastore: Store, graph_spec: ConfigCycleTask, @@ -112,8 +112,7 @@ def from_config( new = plugin_cls( config_rootdir=config_rootdir, coordinates=coordinates, - start_date=start_date, - end_date=end_date, + cycle_point=cycle_point, inputs=inputs, outputs=outputs, **cls_config, @@ -187,29 +186,29 @@ def __getitem__(self, coordinates: dict) -> GRAPH_ITEM_T: key = tuple(coordinates[dim] for dim in self._dims) return self._dict[key] - def iter_from_cycle_spec(self, spec: TargetNodesBaseModel, reference: dict) -> Iterator[GRAPH_ITEM_T]: + def iter_from_cycle_spec(self, spec: TargetNodesBaseModel, ref_coordinates: dict) -> Iterator[GRAPH_ITEM_T]: # Check date references - if "date" not in self._dims and (spec.lag or spec.date): + if "date" not in self._dims and isinstance(spec.target_date, DateList | LagList): msg = f"Array {self._name} has no date dimension, cannot be referenced by dates" raise ValueError(msg) - if "date" in self._dims and reference.get("date") is None and len(spec.date) == 0: + if "date" in self._dims and ref_coordinates.get("date") is None and not isinstance(spec.target_date, DateList): msg = f"Array {self._name} has a date dimension, must be referenced by dates" raise ValueError(msg) - for key in product(*(self._resolve_target_dim(spec, dim, reference) for dim in self._dims)): + for key in product(*(self._resolve_target_dim(spec, dim, ref_coordinates) for dim in self._dims)): yield self._dict[key] - def _resolve_target_dim(self, spec: TargetNodesBaseModel, dim: str, reference: Any) -> Iterator[Any]: + def _resolve_target_dim(self, spec: TargetNodesBaseModel, dim: str, ref_coordinates: Any) -> Iterator[Any]: if dim == "date": - if not spec.lag and not spec.date: - yield reference["date"] - if spec.lag: - for lag in spec.lag: - yield reference["date"] + lag - if spec.date: - yield from spec.date + match spec.target_date: + case SameDate(): + yield ref_coordinates["date"] + case DateList(): + yield from spec.target_date.dates + case LagList(): + yield from spec.target_date.lags elif spec.parameters.get(dim) == "single": - yield reference[dim] + yield ref_coordinates[dim] else: yield from self._axes[dim] @@ -239,20 +238,12 @@ def __getitem__(self, key: tuple[str, dict]) -> GRAPH_ITEM_T: raise KeyError(msg) return self._dict[name][coordinates] - def iter_from_cycle_spec(self, spec: TargetNodesBaseModel, reference: dict) -> Iterator[GRAPH_ITEM_T]: - # Check if target items should be querried at all - if (when := spec.when) is not None: - if (ref_date := reference.get("date")) is None: - msg = "Cannot use a `when` specification without a `reference date`" - raise ValueError(msg) - if (at := when.at) is not None and at != ref_date: - return - if (before := when.before) is not None and before <= ref_date: - return - if (after := when.after) is not None and after >= ref_date: - return + def iter_from_cycle_spec(self, spec: TargetNodesBaseModel, ref_coordinates: dict) -> Iterator[GRAPH_ITEM_T]: + # Check if we need to skip this querry + if isinstance(spec.when, WhenSpec) and not spec.when.is_active(ref_coordinates.get("date")): + return # Yield items - yield from self._dict[spec.name].iter_from_cycle_spec(spec, reference) + yield from self._dict[spec.name].iter_from_cycle_spec(spec, ref_coordinates) def __iter__(self) -> Iterator[GRAPH_ITEM_T]: yield from chain(*(self._dict.values())) diff --git a/src/sirocco/core/workflow.py b/src/sirocco/core/workflow.py index dd6c0e44..4d81ff6a 100644 --- a/src/sirocco/core/workflow.py +++ b/src/sirocco/core/workflow.py @@ -4,17 +4,18 @@ from typing import TYPE_CHECKING, Self from sirocco.core.graph_items import Cycle, Data, Store, Task -from sirocco.parsing._yaml_data_models import ( +from sirocco.parsing.cycling import DateCyclePoint, OneOffPoint +from sirocco.parsing.yaml_data_models import ( ConfigBaseData, ConfigWorkflow, ) if TYPE_CHECKING: from collections.abc import Iterator - from datetime import datetime from pathlib import Path - from sirocco.parsing._yaml_data_models import ( + from sirocco.parsing.cycling import CyclePoint + from sirocco.parsing.yaml_data_models import ( ConfigCycle, ConfigData, ConfigTask, @@ -44,40 +45,40 @@ def __init__( task_dict: dict[str, ConfigTask] = {task.name: task for task in tasks} # Function to iterate over date and parameter combinations - def iter_coordinates(param_refs: list, date: datetime | None = None) -> Iterator[dict]: - space = ({} if date is None else {"date": [date]}) | {k: parameters[k] for k in param_refs} - yield from (dict(zip(space.keys(), x, strict=False)) for x in product(*space.values())) + def iter_coordinates(param_refs: list[str], cycle_point: CyclePoint) -> Iterator[dict]: + axes = {k: parameters[k] for k in param_refs} + if isinstance(cycle_point, DateCyclePoint): + axes["date"] = [cycle_point.begin_date] + yield from (dict(zip(axes.keys(), x, strict=False)) for x in product(*axes.values())) # 1 - create availalbe data nodes for available_data_config in data.available: - for coordinates in iter_coordinates(param_refs=available_data_config.parameters, date=None): + for coordinates in iter_coordinates(param_refs=available_data_config.parameters, cycle_point=OneOffPoint()): self.data.add(Data.from_config(config=available_data_config, coordinates=coordinates)) # 2 - create output data nodes for cycle_config in cycles: - for date in self.cycle_dates(cycle_config): + for cycle_point in cycle_config.cycling.iter_cycle_points(): for task_ref in cycle_config.tasks: for data_ref in task_ref.outputs: data_name = data_ref.name data_config = data_dict[data_name] - for coordinates in iter_coordinates(param_refs=data_config.parameters, date=date): + for coordinates in iter_coordinates(param_refs=data_config.parameters, cycle_point=cycle_point): self.data.add(Data.from_config(config=data_config, coordinates=coordinates)) # 3 - create cycles and tasks for cycle_config in cycles: cycle_name = cycle_config.name - for date in self.cycle_dates(cycle_config): + for cycle_point in cycle_config.cycling.iter_cycle_points(): cycle_tasks = [] for task_graph_spec in cycle_config.tasks: task_name = task_graph_spec.name task_config = task_dict[task_name] - - for coordinates in iter_coordinates(param_refs=task_config.parameters, date=date): + for coordinates in iter_coordinates(param_refs=task_config.parameters, cycle_point=cycle_point): task = Task.from_config( config=task_config, config_rootdir=self.config_rootdir, - start_date=cycle_config.start_date, - end_date=cycle_config.end_date, + cycle_point=cycle_point, coordinates=coordinates, datastore=self.data, graph_spec=task_graph_spec, @@ -88,7 +89,7 @@ def iter_coordinates(param_refs: list, date: datetime | None = None) -> Iterator Cycle( name=cycle_name, tasks=cycle_tasks, - coordinates={} if date is None else {"date": date}, + coordinates={} if isinstance(cycle_point, OneOffPoint) else {"date": cycle_point.begin_date}, ) ) @@ -96,13 +97,6 @@ def iter_coordinates(param_refs: list, date: datetime | None = None) -> Iterator for task in self.tasks: task.link_wait_on_tasks(self.tasks) - @staticmethod - def cycle_dates(cycle_config: ConfigCycle) -> Iterator[datetime | None]: - yield (date := cycle_config.start_date) - if cycle_config.period is not None and date is not None and cycle_config.end_date is not None: - while (date := date + cycle_config.period) < cycle_config.end_date: - yield date - @classmethod def from_config_file(cls: type[Self], config_path: str) -> Self: """ diff --git a/src/sirocco/parsing/__init__.py b/src/sirocco/parsing/__init__.py index 62e186fa..2a942c94 100644 --- a/src/sirocco/parsing/__init__.py +++ b/src/sirocco/parsing/__init__.py @@ -1,4 +1,4 @@ -from ._yaml_data_models import ( +from .yaml_data_models import ( ConfigWorkflow, ) diff --git a/src/sirocco/parsing/_yaml_data_models.py b/src/sirocco/parsing/_yaml_data_models.py deleted file mode 100644 index 657360ca..00000000 --- a/src/sirocco/parsing/_yaml_data_models.py +++ /dev/null @@ -1,759 +0,0 @@ -from __future__ import annotations - -import enum -import itertools -import time -import typing -from dataclasses import dataclass, field -from datetime import datetime -from io import StringIO -from pathlib import Path -from typing import Annotated, Any, ClassVar, Literal, Self - -from isoduration import parse_duration -from isoduration.types import Duration # pydantic needs type # noqa: TCH002 -from pydantic import ( - BaseModel, - BeforeValidator, - ConfigDict, - Discriminator, - Field, - Tag, - TypeAdapter, - field_validator, - model_validator, -) -from ruamel.yaml import YAML - -from sirocco.parsing._utils import TimeUtils - -ITEM_T = typing.TypeVar("ITEM_T") - - -def list_not_empty(value: list[ITEM_T]) -> list[ITEM_T]: - if len(value) < 1: - msg = "At least one element is required." - raise ValueError(msg) - return value - - -class _NamedBaseModel(BaseModel): - """ - Base model for reading names from yaml keys *or* keyword args to the constructor. - - Reading from key-value pairs in yaml is also supported in order to enable - the standard constructor usage from Python, as demonstrated in the below - examples. On it's own it is not considered desirable. - - Examples: - - >>> _NamedBaseModel(name="foo") - _NamedBaseModel(name='foo') - - >>> _NamedBaseModel(foo={}) - _NamedBaseModel(name='foo') - - >>> import textwrap - >>> validate_yaml_content( - ... _NamedBaseModel, - ... textwrap.dedent(''' - ... foo: - ... '''), - ... ) - _NamedBaseModel(name='foo') - - >>> validate_yaml_content( - ... _NamedBaseModel, - ... textwrap.dedent(''' - ... name: foo - ... '''), - ... ) - _NamedBaseModel(name='foo') - """ - - name: str - - @model_validator(mode="before") - @classmethod - def reformat_named_object(cls, data: Any) -> Any: - return cls.extract_merge_name(data) - - @classmethod - def extract_merge_name(cls, data: Any) -> Any: - if not isinstance(data, dict): - return data - if len(data) == 1: - key, value = next(iter(data.items())) - match key: - case str(): - match value: - case str() if key == "name": - pass - case dict() if "name" not in value: - data = value | {"name": key} - case None: - data = {"name": key} - case _: - msg = f"{cls.__name__} may only be used for named objects, not values (got {data})." - raise TypeError(msg) - case _: - msg = f"{cls.__name__} requires name to be a str (got {key})." - raise TypeError(msg) - return data - - -class _WhenBaseModel(BaseModel): - """Base class for when specifications""" - - before: datetime | None = None - after: datetime | None = None - at: datetime | None = None - - @model_validator(mode="before") - @classmethod - def check_before_after_at_combination(cls, data: Any) -> Any: - if "at" in data and any(k in data for k in ("before", "after")): - msg = "'at' key is incompatible with 'before' and after'" - raise ValueError(msg) - if not any(k in data for k in ("at", "before", "after")): - msg = "use at least one of 'at', 'before' or 'after' keys" - raise ValueError(msg) - return data - - @field_validator("before", "after", "at", mode="before") - @classmethod - def convert_datetime(cls, value: datetime | str | None) -> datetime | None: - if value is None or isinstance(value, datetime): - return value - return datetime.fromisoformat(value) - - -class TargetNodesBaseModel(_NamedBaseModel): - """class for targeting other task or data nodes in the graph - - When specifying cycle tasks, this class gathers the required information for - targeting other nodes, either input data or wait on tasks. - - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - date: list[datetime] = [] # this is safe in pydantic - lag: list[Duration] = [] # this is safe in pydantic - when: _WhenBaseModel | None = None - parameters: dict = {} - - @model_validator(mode="before") - @classmethod - def check_lag_xor_date_is_set(cls, data: Any) -> Any: - if "lag" in data and "date" in data: - msg = "Only one key 'lag' or 'date' is allowed. Not both." - raise ValueError(msg) - return data - - @field_validator("lag", mode="before") - @classmethod - def convert_durations(cls, value) -> list[Duration]: - if value is None: - return [] - values = value if isinstance(value, list) else [value] - return [parse_duration(value) for value in values] - - @field_validator("date", mode="before") - @classmethod - def convert_datetimes(cls, value) -> list[datetime]: - if value is None: - return [] - values = value if isinstance(value, list) else [value] - return [datetime.fromisoformat(value) for value in values] - - @field_validator("parameters", mode="before") - @classmethod - def check_parameters_spec(cls, params: dict) -> dict: - if not params: - return {} - for k, v in params.items(): - if v not in ("single", "all"): - msg = f"parameter {k}: reference can only be 'single' or 'all', got {v}" - raise ValueError(msg) - return params - - -class ConfigCycleTaskInput(TargetNodesBaseModel): - port: str | None = None - - -class ConfigCycleTaskWaitOn(TargetNodesBaseModel): - pass - - -class ConfigCycleTaskOutput(_NamedBaseModel): - """ - To create an instance of an output in a task in a cycle defined in a workflow file. - """ - - -NAMED_BASE_T = typing.TypeVar("NAMED_BASE_T", bound=_NamedBaseModel) - - -def make_named_model_list_converter( - cls: type[NAMED_BASE_T], -) -> typing.Callable[[list[NAMED_BASE_T | str | dict] | None], list[NAMED_BASE_T]]: - def convert_named_model_list(values: list[NAMED_BASE_T | str | dict] | None) -> list[NAMED_BASE_T]: - inputs: list[NAMED_BASE_T] = [] - if values is None: - return inputs - for value in values: - match value: - case str(): - inputs.append(cls(name=value)) - case dict(): - inputs.append(cls(**value)) - case _NamedBaseModel(): - inputs.append(value) - case _: - msg = "Unsupported Type" - raise TypeError(msg) - return inputs - - return convert_named_model_list - - -class ConfigCycleTask(_NamedBaseModel): - """ - To create an instance of a task in a cycle defined in a workflow file. - """ - - inputs: Annotated[ - list[ConfigCycleTaskInput], BeforeValidator(make_named_model_list_converter(ConfigCycleTaskInput)) - ] = [] - outputs: Annotated[ - list[ConfigCycleTaskOutput], BeforeValidator(make_named_model_list_converter(ConfigCycleTaskOutput)) - ] = [] - wait_on: Annotated[ - list[ConfigCycleTaskWaitOn], BeforeValidator(make_named_model_list_converter(ConfigCycleTaskWaitOn)) - ] = [] - - -class ConfigCycle(_NamedBaseModel): - """ - To create an instance of a cycle defined in a workflow file. - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - name: str - tasks: list[ConfigCycleTask] - start_date: datetime | None = None - end_date: datetime | None = None - period: Duration | None = None - - @field_validator("start_date", "end_date", mode="before") - @classmethod - def convert_datetime(cls, value) -> None | datetime: - return None if value is None else datetime.fromisoformat(value) - - @field_validator("period", mode="before") - @classmethod - def convert_duration(cls, value): - return None if value is None else parse_duration(value) - - @model_validator(mode="before") - @classmethod - def check_start_date_end_date_period_combination(cls, data: Any) -> Any: - if ("start_date" in data) ^ ("end_date" in data): - msg = f"in cycle {data['name']}: both start_date and end_date must be provided or none of them." - raise ValueError(msg) - if "period" in data and "start_date" not in data: - msg = f"in cycle {data['name']}: period provided without start and end dates." - return data - - @model_validator(mode="after") - def check_start_date_before_end_date(self) -> ConfigCycle: - if self.start_date is not None and self.end_date is not None and self.start_date > self.end_date: - msg = "For cycle {self._name!r} the start_date {start_date!r} lies after given end_date {end_date!r}." - raise ValueError(msg) - return self - - @model_validator(mode="after") - def check_period_is_not_negative_or_zero(self) -> ConfigCycle: - if self.period is not None and TimeUtils.duration_is_less_equal_zero(self.period): - msg = f"For cycle {self.name!r} the period {self.period!r} is negative or zero." - raise ValueError(msg) - return self - - -@dataclass(kw_only=True) -class ConfigBaseTaskSpecs: - computer: str | None = None - host: str | None = None - account: str | None = None - uenv: dict | None = None - nodes: int | None = None - walltime: str | None = None - - -class ConfigBaseTask(_NamedBaseModel, ConfigBaseTaskSpecs): - """ - config for genric task, no plugin specifics - """ - - parameters: list[str] = Field(default_factory=list) - - @field_validator("walltime") - @classmethod - def convert_to_struct_time(cls, value: str | None) -> time.struct_time | None: - """Converts a string of form "%H:%M:%S" to a time.time_struct""" - return None if value is None else time.strptime(value, "%H:%M:%S") - - -class ConfigRootTask(ConfigBaseTask): - plugin: ClassVar[Literal["_root"]] = "_root" - - -# By using a frozen class we only need to validate on initialization -@dataclass(frozen=True) -class ShellCliArgument: - """A holder for a CLI argument to simplify access. - - Stores CLI arguments of the form "file", "--init", "{file}" or "{--init file}". These examples translate into - ShellCliArguments ShellCliArgument(name="file", references_data_item=False, cli_option_of_data_item=None), - ShellCliArgument(name="--init", references_data_item=False, cli_option_of_data_item=None), - ShellCliArgument(name="file", references_data_item=True, cli_option_of_data_item=None), - ShellCliArgument(name="file", references_data_item=True, cli_option_of_data_item="--init") - - Attributes: - name: Name of the argument. For the examples it is "file", "--init", "file" and "file" - references_data_item: Specifies if the argument references a data item signified by enclosing it by curly - brackets. - cli_option_of_data_item: The CLI option associated to the data item. - """ - - name: str - references_data_item: bool - cli_option_of_data_item: str | None = None - - def __post_init__(self): - if self.cli_option_of_data_item is not None and not self.references_data_item: - msg = "data_item_option cannot be not None if cli_option_of_data_item is False" - raise ValueError(msg) - - @classmethod - def from_cli_argument(cls, arg: str) -> ShellCliArgument: - len_arg_with_option = 2 - len_arg_no_option = 1 - references_data_item = arg.startswith("{") and arg.endswith("}") - # remove curly brackets "{--init file}" -> "--init file" - arg_unwrapped = arg[1:-1] if arg.startswith("{") and arg.endswith("}") else arg - - # "--init file" -> ["--init", "file"] - input_arg = arg_unwrapped.split() - if len(input_arg) != len_arg_with_option and len(input_arg) != len_arg_no_option: - msg = f"Expected argument of format {{data}} or {{option data}} but found {arg}" - raise ValueError(msg) - name = input_arg[0] if len(input_arg) == len_arg_no_option else input_arg[1] - cli_option_of_data_item = input_arg[0] if len(input_arg) == len_arg_with_option else None - return cls(name, references_data_item, cli_option_of_data_item) - - -@dataclass(kw_only=True) -class ConfigShellTaskSpecs: - plugin: ClassVar[Literal["shell"]] = "shell" - command: str = "" - cli_arguments: list[ShellCliArgument] = field(default_factory=list) - env_source_files: list[str] = field(default_factory=list) - src: str | None = None - - -class ConfigShellTask(ConfigBaseTask, ConfigShellTaskSpecs): - command: str = "" - cli_arguments: list[ShellCliArgument] = Field(default_factory=list) - env_source_files: list[str] = Field(default_factory=list) - - @field_validator("cli_arguments", mode="before") - @classmethod - def validate_cli_arguments(cls, value: str) -> list[ShellCliArgument]: - return cls.parse_cli_arguments(value) - - @field_validator("env_source_files", mode="before") - @classmethod - def validate_env_source_files(cls, value: str | list[str]) -> list[str]: - return [value] if isinstance(value, str) else value - - @staticmethod - def split_cli_arguments(cli_arguments: str) -> list[str]: - """Splits the CLI arguments into a list of separate entities. - - Splits the CLI arguments by whitespaces except if the whitespace is contained within curly brackets. For example - the string - "-D --CMAKE_CXX_COMPILER=${CXX_COMPILER} {--init file}" - will be splitted into the list - ["-D", "--CMAKE_CXX_COMPILER=${CXX_COMPILER}", "{--init file}"] - """ - - nb_open_curly_brackets = 0 - last_split_idx = 0 - splits = [] - for i, char in enumerate(cli_arguments): - if char == " " and not nb_open_curly_brackets: - # we ommit the space in the splitting therefore we only store up to i but move the last_split_idx to i+1 - splits.append(cli_arguments[last_split_idx:i]) - last_split_idx = i + 1 - elif char == "{": - nb_open_curly_brackets += 1 - elif char == "}": - if nb_open_curly_brackets == 0: - msg = "Invalid input for cli_arguments. Found a closing curly bracket before an opening in {cli_argumentss!r}" - raise ValueError(msg) - nb_open_curly_brackets -= 1 - - if last_split_idx != len(cli_arguments): - splits.append(cli_arguments[last_split_idx : len(cli_arguments)]) - return splits - - @staticmethod - def parse_cli_arguments(cli_arguments: str) -> list[ShellCliArgument]: - return [ShellCliArgument.from_cli_argument(arg) for arg in ConfigShellTask.split_cli_arguments(cli_arguments)] - - -@dataclass(kw_only=True) -class ConfigNamelist: - """Class for namelist specifications - - - path is the path to the namelist file considered as template - - specs is a dictionnary containing the specifications of parameters - to change in the original namelist file - - Example: - - >>> path = "/some/path/to/icon.nml" - >>> specs = { - ... "first_nml_block": {"first_param": "a string value", "second_param": 0}, - ... "second_nml_block": {"third_param": False}, - ... } - >>> config_nml = ConfigNamelist(path=path, specs=specs) - """ - - path: Path - specs: dict | None = None - - -@dataclass(kw_only=True) -class ConfigIconTaskSpecs: - plugin: ClassVar[Literal["icon"]] = "icon" - namelists: dict[str, ConfigNamelist] - - -class ConfigIconTask(ConfigBaseTask, ConfigIconTaskSpecs): - """Class representing an ICON task configuration from a workflow file - - Examples: - - yaml snippet: - - >>> import textwrap - >>> snippet = textwrap.dedent( - ... ''' - ... ICON: - ... plugin: icon - ... namelists: - ... - path/to/icon_master.namelist - ... - path/to/case_nml: - ... block_1: - ... param_name: param_value - ... ''' - ... ) - >>> icon_task_cfg = validate_yaml_content(ConfigIconTask, snippet) - """ - - @field_validator("namelists", mode="before") - @classmethod - def check_nmls(cls, nmls: dict[str, ConfigNamelist] | list[Any]) -> dict[str, ConfigNamelist]: - # Make validator idempotent even if not used yet - if isinstance(nmls, dict): - return nmls - if not isinstance(nmls, list): - msg = f"expected a list got type {type(nmls).__name__}" - raise TypeError(msg) - namelists = {} - master_found = False - for nml in nmls: - msg = f"was expecting a dict of length 1 or a string, got {nml}" - if not isinstance(nml, str | dict): - raise TypeError(msg) - if isinstance(nml, dict) and len(nml) > 1: - raise TypeError(msg) - if isinstance(nml, str): - path, specs = Path(nml), None - else: - path, specs = next(iter(nml.items())) - path = Path(path) - namelists[path.name] = ConfigNamelist(path=path, specs=specs) - master_found = master_found or (path.name == "icon_master.namelist") - if not master_found: - msg = "icon_master.namelist not found" - raise ValueError(msg) - return namelists - - -class DataType(enum.StrEnum): - FILE = enum.auto() - DIR = enum.auto() - - -@dataclass(kw_only=True) -class ConfigBaseDataSpecs: - type: DataType - src: str - format: str | None = None - computer: str | None = None - - -class ConfigBaseData(_NamedBaseModel, ConfigBaseDataSpecs): - """ - To create an instance of a data defined in a workflow file. - - Examples: - - yaml snippet: - - >>> import textwrap - >>> snippet = textwrap.dedent( - ... ''' - ... foo: - ... type: "file" - ... src: "foo.txt" - ... ''' - ... ) - >>> validate_yaml_content(ConfigBaseData, snippet) - ConfigBaseData(type=, src='foo.txt', format=None, computer=None, name='foo', parameters=[]) - - - from python: - - >>> ConfigBaseData(name="foo", type=DataType.FILE, src="foo.txt") - ConfigBaseData(type=, src='foo.txt', format=None, computer=None, name='foo', parameters=[]) - """ - - parameters: list[str] = [] - - @field_validator("type") - @classmethod - def is_file_or_dir(cls, value: str) -> str: - """.""" - valid_types = ("file", "dir") - if value not in valid_types: - msg = f"Must be one of {valid_types}" - raise ValueError(msg) - return value - - -class ConfigAvailableData(ConfigBaseData): - pass - - -class ConfigGeneratedData(ConfigBaseData): - @field_validator("computer") - @classmethod - def invalid_field(cls, value: str | None) -> str | None: - if value is not None: - msg = "The field 'computer' can only be specified for available data." - raise ValueError(msg) - return value - - -class ConfigData(BaseModel): - """ - To create the container of available and generated data - - Example: - - yaml snippet: - - >>> import textwrap - >>> snippet = textwrap.dedent( - ... ''' - ... available: - ... - foo: - ... type: "file" - ... src: "foo.txt" - ... generated: - ... - bar: - ... type: "file" - ... src: "bar.txt" - ... ''' - ... ) - >>> data = validate_yaml_content(ConfigData, snippet) - >>> assert data.available[0].name == "foo" - >>> assert data.generated[0].name == "bar" - - from python: - - >>> ConfigData() - ConfigData(available=[], generated=[]) - """ - - available: list[ConfigAvailableData] = [] - generated: list[ConfigGeneratedData] = [] - - -def get_plugin_from_named_base_model( - data: dict | ConfigRootTask | ConfigShellTask | ConfigIconTask, -) -> str: - if isinstance(data, ConfigRootTask | ConfigShellTask | ConfigIconTask): - return data.plugin - name_and_specs = ConfigBaseTask.extract_merge_name(data) - if name_and_specs.get("name", None) == "ROOT": - return ConfigRootTask.plugin - plugin = name_and_specs.get("plugin", None) - if plugin is None: - msg = f"Could not find plugin name in {data}" - raise ValueError(msg) - return plugin - - -ConfigTask = Annotated[ - Annotated[ConfigRootTask, Tag(ConfigRootTask.plugin)] - | Annotated[ConfigIconTask, Tag(ConfigIconTask.plugin)] - | Annotated[ConfigShellTask, Tag(ConfigShellTask.plugin)], - Discriminator(get_plugin_from_named_base_model), -] - - -class ConfigWorkflow(BaseModel): - """ - The root of the configuration tree. - - Examples: - - minimal yaml to generate: - - >>> import textwrap - >>> content = textwrap.dedent( - ... ''' - ... name: minimal - ... rootdir: /location/of/config/file - ... cycles: - ... - minimal_cycle: - ... tasks: - ... - task_a: - ... tasks: - ... - task_a: - ... plugin: shell - ... data: - ... available: - ... - foo: - ... type: file - ... src: foo.txt - ... generated: - ... - bar: - ... type: dir - ... src: bar - ... ''' - ... ) - >>> wf = validate_yaml_content(ConfigWorkflow, content) - - minimum programmatically created instance - - >>> wf = ConfigWorkflow( - ... name="minimal", - ... rootdir=Path("/location/of/config/file"), - ... cycles=[ConfigCycle(minimal_cycle={"tasks": [ConfigCycleTask(task_a={})]})], - ... tasks=[ConfigShellTask(task_a={"plugin": "shell"})], - ... data=ConfigData( - ... available=[ - ... ConfigAvailableData(name="foo", type=DataType.FILE, src="foo.txt") - ... ], - ... generated=[ - ... ConfigGeneratedData(name="bar", type=DataType.DIR, src="bar") - ... ], - ... ), - ... parameters={}, - ... ) - - """ - - rootdir: Path - name: str - cycles: Annotated[list[ConfigCycle], BeforeValidator(list_not_empty)] - tasks: Annotated[list[ConfigTask], BeforeValidator(list_not_empty)] - data: ConfigData - parameters: dict[str, list] = {} - - @field_validator("parameters", mode="before") - @classmethod - def check_parameters_lists(cls, data) -> dict[str, list]: - for param_name, param_values in data.items(): - msg = f"""{param_name}: parameters must map a string to list of single values, got {param_values}""" - if isinstance(param_values, list): - for v in param_values: - if isinstance(v, dict | list): - raise TypeError(msg) - else: - raise TypeError(msg) - return data - - @model_validator(mode="after") - def check_parameters(self) -> ConfigWorkflow: - task_data_list = itertools.chain(self.tasks, self.data.generated, self.data.available) - for item in task_data_list: - for param_name in item.parameters: - if param_name not in self.parameters: - msg = f"parameter {param_name} in {item.name} specification not declared in parameters section" - raise ValueError(msg) - return self - - @classmethod - def from_config_file(cls, config_path: str) -> Self: - """Creates a ConfigWorkflow instance from a config file, a yaml with the workflow definition. - - Args: - config_path (str): The path of the config file to load from. - - Returns: - OBJECT_T: An instance of the specified class type with data parsed and - validated from the YAML content. - """ - config_filename = Path(config_path).stem - config_resolved_path = Path(config_path).resolve() - if not config_resolved_path.exists(): - msg = f"Workflow config file in path {config_resolved_path} does not exists." - raise FileNotFoundError(msg) - if not config_resolved_path.is_file(): - msg = f"Workflow config file in path {config_resolved_path} is not a file." - raise FileNotFoundError(msg) - - content = config_resolved_path.read_text() - # An empty workflow is parsed to None object so we catch this here for a more understandable error - if content == "": - msg = f"Workflow config file in path {config_resolved_path} is empty." - raise ValueError(msg) - reader = YAML(typ="safe", pure=True) - object_ = reader.load(StringIO(content)) - # If name was not specified, then we use filename without file extension - if "name" not in object_: - object_["name"] = config_filename - object_["rootdir"] = config_resolved_path.parent - adapter = TypeAdapter(cls) - return adapter.validate_python(object_) - - -OBJECT_T = typing.TypeVar("OBJECT_T") - - -def validate_yaml_content(cls: type[OBJECT_T], content: str) -> OBJECT_T: - """Parses the YAML content into a python object using generic types and subsequently validates it with pydantic. - - Args: - cls (type[OBJECT_T]): The class type to which the parsed yaml content should - be validated. It must be compatible with pydantic validation. - content (str): The yaml content as a string. - - Returns: - OBJECT_T: An instance of the specified class type with data parsed and - validated from the YAML content. - - Raises: - pydantic.ValidationError: If the YAML content cannot be validated - against the specified class type. - ruamel.yaml.YAMLError: If there is an error in parsing the YAML content. - """ - return TypeAdapter(cls).validate_python(YAML(typ="safe", pure=True).load(StringIO(content))) diff --git a/src/sirocco/pretty_print.py b/src/sirocco/pretty_print.py index 1dfa1ad8..811b22c7 100644 --- a/src/sirocco/pretty_print.py +++ b/src/sirocco/pretty_print.py @@ -86,11 +86,18 @@ def format_basic(self, obj: core.GraphItem) -> str: >>> from datetime import datetime >>> import pathlib + >>> from sirocco.parsing.cycling import DateCyclePoint >>> print( ... PrettyPrinter().format_basic( ... core.Task( ... name="foo", ... config_rootdir=pathlib.Path("."), + ... cycle_point=DateCyclePoint( + ... start_date=datetime(1000, 1, 1), + ... stop_date=datetime(1000, 1, 2), + ... begin_date=datetime(1000, 1, 1), + ... end_date=datetime(1000, 1, 2), + ... ), ... coordinates={"date": datetime(1000, 1, 1).date()}, ... ) ... ) diff --git a/tests/cases/large/config/config.yml b/tests/cases/large/config/config.yml index 54d355c4..63bd909f 100644 --- a/tests/cases/large/config/config.yml +++ b/tests/cases/large/config/config.yml @@ -1,6 +1,6 @@ --- start_date: &root_start_date '2025-01-01T00:00' -end_date: &root_end_date '2027-01-01T00:00' +stop_date: &root_stop_date '2027-01-01T00:00' cycles: - init: tasks: @@ -9,7 +9,7 @@ cycles: outputs: [extpar_file] - icon_bimonthly: start_date: *root_start_date - end_date: *root_end_date + stop_date: *root_stop_date period: 'P2M' tasks: - preproc: @@ -38,7 +38,7 @@ cycles: outputs: [stored_data_1] - yearly: start_date: *root_start_date - end_date: *root_end_date + stop_date: *root_stop_date period: 'P1Y' tasks: - postproc_2: diff --git a/tests/cases/large/data/config.txt b/tests/cases/large/data/config.txt index bdc9d1c7..86bbc43d 100644 --- a/tests/cases/large/data/config.txt +++ b/tests/cases/large/data/config.txt @@ -30,7 +30,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -49,7 +49,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -64,7 +64,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -81,7 +81,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -101,7 +101,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -121,7 +121,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -136,7 +136,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -153,7 +153,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -175,7 +175,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -195,7 +195,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -210,7 +210,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -227,7 +227,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -249,7 +249,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -269,7 +269,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -284,7 +284,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -301,7 +301,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -323,7 +323,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -343,7 +343,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -358,7 +358,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -375,7 +375,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -397,7 +397,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -417,7 +417,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -432,7 +432,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -449,7 +449,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -471,7 +471,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -491,7 +491,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -506,7 +506,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -523,7 +523,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -545,7 +545,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -565,7 +565,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -580,7 +580,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -597,7 +597,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -619,7 +619,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -639,7 +639,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -654,7 +654,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -671,7 +671,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -693,7 +693,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -713,7 +713,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -728,7 +728,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -745,7 +745,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -767,7 +767,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -787,7 +787,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -802,7 +802,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -819,7 +819,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -841,7 +841,7 @@ cycles: nodes: 4 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=2, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/cleanup.sh' cli arguments: [ShellCliArgument(name='extpar_file', references_data_item=True, cli_option_of_data_item='-p'), ShellCliArgument(name='ERA5', references_data_item=True, cli_option_of_data_item='-e'), ShellCliArgument(name='grid_file', references_data_item=True, cli_option_of_data_item=None)] @@ -861,7 +861,7 @@ cycles: nodes: 40 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=23, tm_min=59, tm_sec=59, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'icon' namelists: {'icon_master.namelist': ConfigNamelist(path=PosixPath('ICON/icon_master.namelist'), specs=None), 'NAMELIST_exclaim_ape_R02B04': ConfigNamelist(path=PosixPath('ICON/NAMELIST_exclaim_ape_R02B04'), specs={'parallel_nml': {'nproma': 96}, 'output_nml[1]': {'output_filename': 'atm_2d'}})} core namelists: {} @@ -876,7 +876,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_ocn.sh' cli arguments: [ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--input')] @@ -893,7 +893,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_1', references_data_item=True, cli_option_of_data_item='--input'), ShellCliArgument(name='stream_1', references_data_item=True, cli_option_of_data_item='--stream'), ShellCliArgument(name='icon_input', references_data_item=True, cli_option_of_data_item='--icon_input')] @@ -916,7 +916,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_atm.sh' cli arguments: [ShellCliArgument(name='stream_2', references_data_item=True, cli_option_of_data_item='--input')] @@ -938,7 +938,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_2', references_data_item=True, cli_option_of_data_item='--input')] @@ -961,7 +961,7 @@ cycles: nodes: 2 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=5, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/main_script_atm.sh' cli arguments: [ShellCliArgument(name='stream_2', references_data_item=True, cli_option_of_data_item='--input')] @@ -983,7 +983,7 @@ cycles: nodes: 1 walltime: time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=1, tm_sec=0, tm_wday=0, tm_yday=1, tm_isdst=-1) start date: 2025-01-01 00:00:00 - end date: 2027-01-01 00:00:00 + stop date: 2027-01-01 00:00:00 plugin: 'shell' command: 'scripts/post_clean.sh' cli arguments: [ShellCliArgument(name='postout_2', references_data_item=True, cli_option_of_data_item='--input')] diff --git a/tests/cases/parameters/config/config.yml b/tests/cases/parameters/config/config.yml index 374a1e6b..90c8afdc 100644 --- a/tests/cases/parameters/config/config.yml +++ b/tests/cases/parameters/config/config.yml @@ -1,11 +1,11 @@ --- start_date: &root_start_date '2026-01-01T00:00' -end_date: &root_end_date '2028-01-01T00:00' +stop_date: &root_stop_date '2028-01-01T00:00' cycles: - bimonthly_tasks: start_date: *root_start_date - end_date: *root_end_date + stop_date: *root_stop_date period: P6M tasks: - icon: @@ -33,7 +33,7 @@ cycles: outputs: [analysis_foo_bar] - yearly: start_date: *root_start_date - end_date: *root_end_date + stop_date: *root_stop_date period: P1Y tasks: - merge: diff --git a/tests/cases/parameters/data/config.txt b/tests/cases/parameters/data/config.txt index 21cccb92..ecc7fee1 100644 --- a/tests/cases/parameters/data/config.txt +++ b/tests/cases/parameters/data/config.txt @@ -11,7 +11,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2026, 1, 1, 0, 0), 'foo': 0, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -26,7 +26,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2026, 1, 1, 0, 0), 'foo': 1, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -40,7 +40,7 @@ cycles: name: 'statistics_foo' coordinates: {'date': datetime.datetime(2026, 1, 1, 0, 0), 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -53,7 +53,7 @@ cycles: name: 'statistics_foo_bar' coordinates: {'date': datetime.datetime(2026, 1, 1, 0, 0)} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -70,7 +70,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2026, 7, 1, 0, 0), 'foo': 0, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -85,7 +85,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2026, 7, 1, 0, 0), 'foo': 1, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -99,7 +99,7 @@ cycles: name: 'statistics_foo' coordinates: {'date': datetime.datetime(2026, 7, 1, 0, 0), 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -112,7 +112,7 @@ cycles: name: 'statistics_foo_bar' coordinates: {'date': datetime.datetime(2026, 7, 1, 0, 0)} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -129,7 +129,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2027, 1, 1, 0, 0), 'foo': 0, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -144,7 +144,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2027, 1, 1, 0, 0), 'foo': 1, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -158,7 +158,7 @@ cycles: name: 'statistics_foo' coordinates: {'date': datetime.datetime(2027, 1, 1, 0, 0), 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -171,7 +171,7 @@ cycles: name: 'statistics_foo_bar' coordinates: {'date': datetime.datetime(2027, 1, 1, 0, 0)} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -188,7 +188,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2027, 7, 1, 0, 0), 'foo': 0, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -203,7 +203,7 @@ cycles: name: 'icon' coordinates: {'date': datetime.datetime(2027, 7, 1, 0, 0), 'foo': 1, 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init'), ShellCliArgument(name='forcing', references_data_item=True, cli_option_of_data_item='--forcing')] @@ -217,7 +217,7 @@ cycles: name: 'statistics_foo' coordinates: {'date': datetime.datetime(2027, 7, 1, 0, 0), 'bar': 3.0} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -230,7 +230,7 @@ cycles: name: 'statistics_foo_bar' coordinates: {'date': datetime.datetime(2027, 7, 1, 0, 0)} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/statistics.py' cli arguments: [] @@ -246,7 +246,7 @@ cycles: name: 'merge' coordinates: {'date': datetime.datetime(2026, 1, 1, 0, 0)} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/merge.py' cli arguments: [] @@ -262,7 +262,7 @@ cycles: name: 'merge' coordinates: {'date': datetime.datetime(2027, 1, 1, 0, 0)} start date: 2026-01-01 00:00:00 - end date: 2028-01-01 00:00:00 + stop date: 2028-01-01 00:00:00 plugin: 'shell' command: 'scripts/merge.py' cli arguments: [] diff --git a/tests/cases/small/config/config.yml b/tests/cases/small/config/config.yml index 30abeaa2..483750ba 100644 --- a/tests/cases/small/config/config.yml +++ b/tests/cases/small/config/config.yml @@ -1,11 +1,12 @@ --- start_date: &root_start_date '2026-01-01T00:00' -end_date: &root_end_date '2026-06-01T00:00' +stop_date: &root_stop_date '2026-06-01T00:00' cycles: - bimonthly_tasks: - start_date: *root_start_date - end_date: *root_end_date - period: P2M + cycling: + start_date: *root_start_date + stop_date: *root_stop_date + period: P2M tasks: - icon: inputs: diff --git a/tests/cases/small/data/config.txt b/tests/cases/small/data/config.txt index 3e799e13..5bd8562a 100644 --- a/tests/cases/small/data/config.txt +++ b/tests/cases/small/data/config.txt @@ -12,7 +12,7 @@ cycles: coordinates: {'date': datetime.datetime(2026, 1, 1, 0, 0)} computer: 'localhost' start date: 2026-01-01 00:00:00 - end date: 2026-06-01 00:00:00 + stop date: 2026-06-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init')] @@ -30,7 +30,7 @@ cycles: coordinates: {'date': datetime.datetime(2026, 3, 1, 0, 0)} computer: 'localhost' start date: 2026-01-01 00:00:00 - end date: 2026-06-01 00:00:00 + stop date: 2026-06-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init')] @@ -48,7 +48,7 @@ cycles: coordinates: {'date': datetime.datetime(2026, 5, 1, 0, 0)} computer: 'localhost' start date: 2026-01-01 00:00:00 - end date: 2026-06-01 00:00:00 + stop date: 2026-06-01 00:00:00 plugin: 'shell' command: 'scripts/icon.py' cli arguments: [ShellCliArgument(name='icon_restart', references_data_item=True, cli_option_of_data_item='--restart'), ShellCliArgument(name='initial_conditions', references_data_item=True, cli_option_of_data_item='--init')] diff --git a/tests/conftest.py b/tests/conftest.py index 8cf8f129..5b7e2d31 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ from sirocco import pretty_print from sirocco.core import _tasks as core_tasks from sirocco.core import workflow -from sirocco.parsing import _yaml_data_models as models +from sirocco.parsing import yaml_data_models as models pytest_plugins = ["aiida.tools.pytest_fixtures"] diff --git a/tests/test_wc_workflow.py b/tests/test_wc_workflow.py index 4491af43..fa4046d9 100644 --- a/tests/test_wc_workflow.py +++ b/tests/test_wc_workflow.py @@ -4,7 +4,7 @@ from sirocco.core import Workflow from sirocco.core._tasks.icon_task import IconTask -from sirocco.parsing._yaml_data_models import ConfigShellTask, ShellCliArgument +from sirocco.parsing.yaml_data_models import ConfigShellTask, ShellCliArgument from sirocco.vizgraph import VizGraph from sirocco.workgraph import AiidaWorkGraph diff --git a/tests/unit_tests/parsing/test_yaml_data_models.py b/tests/unit_tests/parsing/test_yaml_data_models.py index 1549e508..6209bc0d 100644 --- a/tests/unit_tests/parsing/test_yaml_data_models.py +++ b/tests/unit_tests/parsing/test_yaml_data_models.py @@ -3,7 +3,7 @@ import pytest from pydantic import ValidationError -from sirocco.parsing import _yaml_data_models as models +from sirocco.parsing import yaml_data_models as models @pytest.mark.parametrize("data_type", ["file", "dir"])