From 1be1b2dd83b7ab9eeafede20f32326359d4fb974 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 23 Oct 2020 09:18:12 +0200 Subject: [PATCH 01/36] start to SimStore support --- paths_cli/param_core.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 3d379b58..c95029e8 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -85,9 +85,17 @@ def _workaround(self, name): st.close() def get(self, name): - import openpathsampling as paths - self._workaround(name) - return paths.Storage(name, mode=self.mode) + if name.endswith(".db") or name.endswith(".sql"): + from openpathsampling.experimental.simstore import \ + SQLStorageBackend + from openpathsampling.experimental.storage import Storage + backend = SQLStorageBackend(name, mode=self.mode) + storage = Storage.from_backend(backend) + else: + from openpathsampling import Storage + self._workaround(name) + storage = paths.Storage(name, self.mode) + return storage class OPSStorageLoadNames(AbstractLoader): From e773c1fcd60ee12a77315559fea37ac3403504e7 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 23 Oct 2020 10:02:11 +0200 Subject: [PATCH 02/36] fix tests --- paths_cli/param_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index c95029e8..14506b91 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -94,7 +94,7 @@ def get(self, name): else: from openpathsampling import Storage self._workaround(name) - storage = paths.Storage(name, self.mode) + storage = Storage(name, self.mode) return storage From 67c1b22b9cecf6632ffac064e5164bab0fc44ec0 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 27 Nov 2020 09:15:12 +0100 Subject: [PATCH 03/36] Tests for SimStore support --- paths_cli/param_core.py | 13 +++++++++++-- paths_cli/tests/test_parameters.py | 12 +++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 14506b91..226aa42a 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -76,16 +76,25 @@ def __init__(self, param, mode): super(StorageLoader, self).__init__(param) self.mode = mode + @staticmethod + def _is_simstore(name): + return name.endswith(".db") or name.endswith(".sql") + def _workaround(self, name): # this is messed up... for some reason, storage doesn't create a new # file in append mode. That may be a bug import openpathsampling as paths - if self.mode == 'a' and not os.path.exists(name): + needs_workaround = ( + self.mode == 'a' + and not os.path.exists(name) + and not self._is_simstore(name) + ) + if needs_workaround: st = paths.Storage(name, mode='w') st.close() def get(self, name): - if name.endswith(".db") or name.endswith(".sql"): + if self._is_simstore(name): from openpathsampling.experimental.simstore import \ SQLStorageBackend from openpathsampling.experimental.storage import Storage diff --git a/paths_cli/tests/test_parameters.py b/paths_cli/tests/test_parameters.py index af7a170a..bed3163f 100644 --- a/paths_cli/tests/test_parameters.py +++ b/paths_cli/tests/test_parameters.py @@ -376,21 +376,23 @@ def test_get(self, getter): self._getter_test(getter) -def test_OUTPUT_FILE(): +@pytest.mark.parametrize('ext', ['nc', 'db', 'sql']) +def test_OUTPUT_FILE(ext): tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "test_output_file.nc") + filename = os.path.join(tempdir, "test_output_file." + ext) assert not os.path.exists(filename) storage = OUTPUT_FILE.get(filename) assert os.path.exists(filename) os.remove(filename) os.rmdir(tempdir) -def test_APPEND_FILE(): +@pytest.mark.parametrize('ext', ['nc', 'db', 'sql']) +def test_APPEND_FILE(ext): tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "test_append_file.nc") + filename = os.path.join(tempdir, "test_append_file." + ext) assert not os.path.exists(filename) storage = APPEND_FILE.get(filename) - print(storage) + # print(storage) # potentially useful debug; keep assert os.path.exists(filename) traj = make_1d_traj([0.0, 1.0]) storage.tags['first_save'] = traj[0] From 22ca390d722c5670aa4bcbfae0ba434e62b33f9c Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sun, 6 Dec 2020 16:48:19 +0100 Subject: [PATCH 04/36] monkey patching and snapshots for SimStore --- paths_cli/param_core.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 226aa42a..9d5fa6a6 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -95,6 +95,10 @@ def _workaround(self, name): def get(self, name): if self._is_simstore(name): + import openpathsampling as paths + from openpathsampling.experimental.storage import \ + monkey_patch_all + paths = monkey_patch_all(paths) from openpathsampling.experimental.simstore import \ SQLStorageBackend from openpathsampling.experimental.storage import Storage @@ -217,10 +221,20 @@ class GetOnlySnapshot(Getter): def __init__(self, store_name="snapshots"): super().__init__(store_name) + def _min_num_snapshots(self, storage): + # For netcdfplus, we see 2 snapshots when there is only one + # (reversed copy gets saved). For SimStore, we see only one. + import openpathsampling as paths + if isinstance(storage, paths.netcdfplus.NetCDFPlus): + min_snaps = 2 + else: + min_snaps = 1 + return min_snaps + def __call__(self, storage): store = getattr(storage, self.store_name) - if len(store) == 2: - # this is really only 1 snapshot; reversed copy gets saved + min_snaps = self._min_num_snapshots(storage) + if len(store) == min_snaps: return store[0] From 132f23eeaa44781cb6c5c36a1a762f9d54e8e034 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sun, 6 Dec 2020 16:54:15 +0100 Subject: [PATCH 05/36] improve error message when missing objects --- paths_cli/param_core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 9d5fa6a6..03fb7103 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -301,6 +301,7 @@ def get(self, storage, name): result = _try_strategies(self.none_strategies, storage) if result is None: - raise RuntimeError("Couldn't find %s", name) + raise RuntimeError("Couldn't find %s in %s" % (name, + self.store)) return result From df83b710d0682a34cd8f121a66ff42a1f97cfae4 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 14:19:56 +0100 Subject: [PATCH 06/36] Only monkeypatch SimStore once; better error msg --- paths_cli/param_core.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 03fb7103..de619472 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -72,6 +72,7 @@ class StorageLoader(AbstractLoader): mode : 'r', 'w', or 'a' the mode for the file """ + has_simstore_patch = False def __init__(self, param, mode): super(StorageLoader, self).__init__(param) self.mode = mode @@ -97,11 +98,14 @@ def get(self, name): if self._is_simstore(name): import openpathsampling as paths from openpathsampling.experimental.storage import \ - monkey_patch_all - paths = monkey_patch_all(paths) + Storage, monkey_patch_all + + if not self.has_simstore_patch: + paths = monkey_patch_all(paths) + StorageLoader.has_simstore_patch = True + from openpathsampling.experimental.simstore import \ SQLStorageBackend - from openpathsampling.experimental.storage import Storage backend = SQLStorageBackend(name, mode=self.mode) storage = Storage.from_backend(backend) else: @@ -301,7 +305,13 @@ def get(self, storage, name): result = _try_strategies(self.none_strategies, storage) if result is None: - raise RuntimeError("Couldn't find %s in %s" % (name, - self.store)) + if name is None: + msg = "Couldn't guess which item to use from " + self.store + else: + msg = "Couldn't find {name} is {store}".format( + name=name, + store=self.store + ) + raise RuntimeError(msg) return result From 695ed8b973f73e6c28899316de0d6dc3e6272ea3 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 14:21:51 +0100 Subject: [PATCH 07/36] Add support for --table option in `contents` --- paths_cli/commands/contents.py | 36 +++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/paths_cli/commands/contents.py b/paths_cli/commands/contents.py index c1a31597..7cdcfd67 100644 --- a/paths_cli/commands/contents.py +++ b/paths_cli/commands/contents.py @@ -1,12 +1,22 @@ import click from paths_cli.parameters import INPUT_FILE +UNNAMED_SECTIONS = { + 'Steps': lambda storage: storage.steps, + 'Move Changes': lambda storage: storage.movechanges, + 'SampleSets': lambda storage: storage.samplesets, + 'Trajectories': lambda storage: storage.trajectories, + 'Snapshots': lambda storage: storage.snapshots +} + @click.command( 'contents', short_help="list named objects from an OPS .nc file", ) @INPUT_FILE.clicked(required=True) -def contents(input_file): +@click.option('--table', type=str, required=False, + help="table to show results from") +def contents(input_file, table): """List the names of named objects in an OPS .nc file. This is particularly useful when getting ready to use one of simulation @@ -14,6 +24,22 @@ def contents(input_file): """ storage = INPUT_FILE.get(input_file) print(storage) + if table is None: + report_all_tables(storage) + else: + table_attr = table.lower() + try: + store = getattr(storage, table_attr) + except AttributeError: + print("This needs to raise a good error; bad table name") + else: + if table_attr in UNNAMED_SECTIONS: + print(get_unnamed_section_string(table_attr, store)) + else: + print(get_section_string_nameable(table_attr, store, + _get_named_namedobj)) + +def report_all_tables(storage): store_section_mapping = { 'CVs': storage.cvs, 'Volumes': storage.volumes, 'Engines': storage.engines, 'Networks': storage.networks, @@ -26,12 +52,8 @@ def contents(input_file): print(get_section_string_nameable('Tags', storage.tags, _get_named_tags)) print("\nData Objects:") - unnamed_sections = { - 'Steps': storage.steps, 'Move Changes': storage.movechanges, - 'SampleSets': storage.samplesets, - 'Trajectories': storage.trajectories, 'Snapshots': storage.snapshots - } - for section, store in unnamed_sections.items(): + for section, store_func in UNNAMED_SECTIONS.items(): + store = store_func(storage) print(get_unnamed_section_string(section, store)) def _item_or_items(count): From f983f35113a66d6893346df8b8779e42e0b2746a Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 14:50:21 +0100 Subject: [PATCH 08/36] Steps toward allowing multiple --init_conds --- paths_cli/commands/equilibrate.py | 1 + paths_cli/parameters.py | 5 ++--- paths_cli/tests/commands/test_pathsampling.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/paths_cli/commands/equilibrate.py b/paths_cli/commands/equilibrate.py index 357635e4..41b74676 100644 --- a/paths_cli/commands/equilibrate.py +++ b/paths_cli/commands/equilibrate.py @@ -43,6 +43,7 @@ def equilibrate_main(output_storage, scheme, init_conds, multiplier, extra_steps): import openpathsampling as paths init_conds = scheme.initial_conditions_from_trajectories(init_conds) + scheme.assert_initial_conditions(init_conds) simulation = paths.PathSampling( storage=output_storage, move_scheme=scheme, diff --git a/paths_cli/parameters.py b/paths_cli/parameters.py index 70b8ec4c..58823795 100644 --- a/paths_cli/parameters.py +++ b/paths_cli/parameters.py @@ -19,9 +19,9 @@ ) INIT_CONDS = OPSStorageLoadSingle( - param=Option('-t', '--init-conds', + param=Option('-t', '--init-conds', multiple=True, help=("identifier for initial conditions " - + "(sample set or trajectory)")), + + "(sample set or trajectory)" + HELP_MULTIPLE)), store='samplesets', value_strategies=[GetByName('tags'), GetByNumber('samplesets'), GetByNumber('trajectories')], @@ -57,7 +57,6 @@ store='engines' ) - STATES = OPSStorageLoadNames( param=Option('-s', '--state', type=str, multiple=True, help='name of state' + HELP_MULTIPLE), diff --git a/paths_cli/tests/commands/test_pathsampling.py b/paths_cli/tests/commands/test_pathsampling.py index be7b272e..dc5034a5 100644 --- a/paths_cli/tests/commands/test_pathsampling.py +++ b/paths_cli/tests/commands/test_pathsampling.py @@ -26,10 +26,11 @@ def test_pathsampling(tps_fixture): results = runner.invoke(pathsampling, ['setup.nc', '-o', 'foo.nc', '-n', '1000']) - assert results.exit_code == 0 expected_output = (f"True\n{scheme.__uuid__}\n{init_conds.__uuid__}" "\n1000\n") + assert results.output == expected_output + assert results.exit_code == 0 def test_pathsampling_main(tps_fixture): From fbe73da6834d627eb73abba8ad41b39d391720e3 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 14:51:40 +0100 Subject: [PATCH 09/36] Add md command --- paths_cli/commands/md.py | 55 ++++++++++++++++++++++++++++++++++++++++ paths_cli/parameters.py | 5 ++++ 2 files changed, 60 insertions(+) create mode 100644 paths_cli/commands/md.py diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py new file mode 100644 index 00000000..09b9b80a --- /dev/null +++ b/paths_cli/commands/md.py @@ -0,0 +1,55 @@ +import functools +import operator + +import click + +from paths_cli.parameters import (INPUT_FILE, OUTPUT_FILE, ENGINE, + MULTI_ENSEMBLE, INIT_SNAP) + +@click.command( + "md", + short_help=("Run MD for fixed time or until a given ensemble is " + "satisfied"), +) +@INPUT_FILE.clicked(required=True) +@OUTPUT_FILE.clicked(required=True) +@ENGINE.clicked(required=False) +@MULTI_ENSEMBLE.clicked(required=False) +@click.option('-n', '--nsteps', type=int, + help="number of MD steps to run") +@INIT_SNAP.clicked(required=False) +def md(input_file, output_file, engine, ensemble, nsteps, init_frame): + """Run MD for for time of steps or until ensembles are satisfied. + """ + storage = INPUT_FILE.get(input_file) + md_main( + output_storage=OUTPUT_FILE.get(output_file), + engine=ENGINE.get(storage, engine), + ensembles=MULTI_ENSEMBLE.get(storage, ensemble), + nsteps=nsteps, + initial_frame=INIT_SNAP.get(storage, init_frame) + ) + + +def md_main(output_storage, engine, ensembles, nsteps, initial_frame): + import openpathsampling as paths + if nsteps is not None and ensembles: + raise RuntimeError("Options --ensemble and --nsteps cannot both be" + " used at once.") + + if nsteps is not None: + ens = paths.LengthEnsemble(nsteps) + else: + ens = functools.reduce(operator.and_, ensembles) + + trajectory = engine.generate(initial_frame, running=ens.can_append) + if output_storage is not None: + output_storage.save(trajectory) + output_storage.tags['final_conditions'] = trajectory + + return trajectory, None + +CLI = md +SECTION = "Simulation" +REQUIRES_OPS = (1, 0) + diff --git a/paths_cli/parameters.py b/paths_cli/parameters.py index 70b8ec4c..d154dcf1 100644 --- a/paths_cli/parameters.py +++ b/paths_cli/parameters.py @@ -57,6 +57,11 @@ store='engines' ) +MULTI_ENSEMBLE = OPSStorageLoadNames( + param=Option('--ensemble', type=str, multiple=True, + help='name of index of ensemble' + HELP_MULTIPLE), + store='ensembles' +) STATES = OPSStorageLoadNames( param=Option('-s', '--state', type=str, multiple=True, From 7948e37fc0210e6ab80da38819972cad37a32d9e Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 15:43:38 +0100 Subject: [PATCH 10/36] OPSStorageLoadMultiple for INIT_CONDS --- paths_cli/param_core.py | 48 +++++++++++++++++++++++++++++++++++++++++ paths_cli/parameters.py | 8 +++---- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 3d379b58..2e53ea12 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -273,3 +273,51 @@ def get(self, storage, name): raise RuntimeError("Couldn't find %s", name) return result + + +class OPSStorageLoadMultiple(OPSStorageLoadSingle): + """Objects that can guess a single object or have multiple specified. + + Parameters + ---------- + param : :class:`.AbstractParameter` + the Option or Argument wrapping a click decorator + store : Str + the name of the store to search + value_strategies : List[Callable[(:class:`.Storage`, Str), Any]] + The strategies to be used when the CLI provides a value for this + parameter. Each should be a callable taking a storage and the string + input from the CLI, and should return the desired object or None if + it cannot be found. + none_strategies : List[Callable[:class:`openpathsampling.Storage, Any]] + The strategies to be used when the CLI does not provide a value for + this parameter. Each should be a callable taking a storage, and + returning the desired object or None if it cannot be found. + """ + def get(self, storage, names): + """Load desired objects from storage. + + Parameters + ---------- + storage : openpathsampling.Storage + the input storage to search + names : List[Str] or None + strings from CLI providing the identifier (name or index) for + this object; None if not provided + """ + if names == tuple(): + names = None + + if names is None or isinstance(names, (str, int)): + listified = True + names = [names] + else: + listified = False + + results = [super(OPSStorageLoadMultiple, self).get(storage, name) + for name in names] + + if listified: + results = results[0] + + return results diff --git a/paths_cli/parameters.py b/paths_cli/parameters.py index 58823795..9f4c77a3 100644 --- a/paths_cli/parameters.py +++ b/paths_cli/parameters.py @@ -1,8 +1,8 @@ import click from paths_cli.param_core import ( - Option, Argument, OPSStorageLoadSingle, OPSStorageLoadNames, - StorageLoader, GetByName, GetByNumber, GetOnly, GetOnlySnapshot, - GetPredefinedName + Option, Argument, OPSStorageLoadSingle, OPSStorageLoadMultiple, + OPSStorageLoadNames, StorageLoader, GetByName, GetByNumber, GetOnly, + GetOnlySnapshot, GetPredefinedName ) @@ -18,7 +18,7 @@ store='schemes', ) -INIT_CONDS = OPSStorageLoadSingle( +INIT_CONDS = OPSStorageLoadMultiple( param=Option('-t', '--init-conds', multiple=True, help=("identifier for initial conditions " + "(sample set or trajectory)" + HELP_MULTIPLE)), From 3229fba306dc993c449b31283bb06db2d1ccedbe Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 16:54:10 +0100 Subject: [PATCH 11/36] Add test for getting multiple init conds --- paths_cli/tests/test_parameters.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/paths_cli/tests/test_parameters.py b/paths_cli/tests/test_parameters.py index af7a170a..72f66a0e 100644 --- a/paths_cli/tests/test_parameters.py +++ b/paths_cli/tests/test_parameters.py @@ -234,6 +234,14 @@ def test_get_none(self, num_in_file): obj = INIT_CONDS.get(st, None) assert obj == stored_things[num_in_file - 1] + def test_get_multiple(self): + filename = self.create_file('number-traj') + storage = paths.Storage(filename, mode='r') + traj0, traj1 = self.PARAMETER.get(storage, (0, 1)) + assert traj0 == self.traj + assert traj1 == self.other_traj + + class TestINIT_SNAP(ParamInstanceTest): PARAMETER = INIT_SNAP def setup(self): From 47e41afd5da0674d89ee1805a8defa25f4b44db6 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 8 Dec 2020 17:15:06 +0100 Subject: [PATCH 12/36] Tests for --table arg to contents --- paths_cli/commands/contents.py | 19 +++++++++------- paths_cli/tests/commands/test_contents.py | 27 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/paths_cli/commands/contents.py b/paths_cli/commands/contents.py index 7cdcfd67..7ddb848c 100644 --- a/paths_cli/commands/contents.py +++ b/paths_cli/commands/contents.py @@ -1,13 +1,8 @@ import click from paths_cli.parameters import INPUT_FILE -UNNAMED_SECTIONS = { - 'Steps': lambda storage: storage.steps, - 'Move Changes': lambda storage: storage.movechanges, - 'SampleSets': lambda storage: storage.samplesets, - 'Trajectories': lambda storage: storage.trajectories, - 'Snapshots': lambda storage: storage.snapshots -} +UNNAMED_SECTIONS = ['steps', 'movechanges', 'samplesets', 'trajectories', + 'snapshots'] @click.command( 'contents', @@ -52,7 +47,15 @@ def report_all_tables(storage): print(get_section_string_nameable('Tags', storage.tags, _get_named_tags)) print("\nData Objects:") - for section, store_func in UNNAMED_SECTIONS.items(): + data_object_mapping = { + 'Steps': lambda storage: storage.steps, + 'Move Changes': lambda storage: storage.movechanges, + 'SampleSets': lambda storage: storage.samplesets, + 'Trajectories': lambda storage: storage.trajectories, + 'Snapshots': lambda storage: storage.snapshots + } + + for section, store_func in data_object_mapping.items(): store = store_func(storage) print(get_unnamed_section_string(section, store)) diff --git a/paths_cli/tests/commands/test_contents.py b/paths_cli/tests/commands/test_contents.py index 246bafb9..05c953b1 100644 --- a/paths_cli/tests/commands/test_contents.py +++ b/paths_cli/tests/commands/test_contents.py @@ -40,3 +40,30 @@ def test_contents(tps_fixture): assert results.output.split('\n') == expected for truth, beauty in zip(expected, results.output.split('\n')): assert truth == beauty + +@pytest.mark.parametrize('table', ['volumes', 'trajectories']) +def test_contents_table(tps_fixture, table): + scheme, network, engine, init_conds = tps_fixture + runner = CliRunner() + with runner.isolated_filesystem(): + storage = paths.Storage("setup.nc", 'w') + for obj in tps_fixture: + storage.save(obj) + storage.tags['initial_conditions'] = init_conds + + results = runner.invoke(contents, ['setup.nc', '--table', table]) + cwd = os.getcwd() + expected = { + 'volumes': [ + f"Storage @ '{cwd}/setup.nc'", + "volumes: 8 items", "* A", "* B", "* plus 6 unnamed items", + "" + ], + 'trajectories': [ + f"Storage @ '{cwd}/setup.nc'", + "trajectories: 1 unnamed item", + "" + ] + }[table] + assert results.output.split("\n") == expected + assert results.exit_code == 0 From 98f38685285efe8574de73b9725f8f4c8a20677a Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 11:30:09 +0100 Subject: [PATCH 13/36] Add tests for md --- paths_cli/commands/md.py | 56 +++++++++++++++++-- paths_cli/tests/commands/test_md.py | 83 +++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 paths_cli/tests/commands/test_md.py diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py index 09b9b80a..fdfbc8a2 100644 --- a/paths_cli/commands/md.py +++ b/paths_cli/commands/md.py @@ -6,6 +6,9 @@ from paths_cli.parameters import (INPUT_FILE, OUTPUT_FILE, ENGINE, MULTI_ENSEMBLE, INIT_SNAP) +import logging +logger = logging.getLogger(__name__) + @click.command( "md", short_help=("Run MD for fixed time or until a given ensemble is " @@ -31,18 +34,63 @@ def md(input_file, output_file, engine, ensemble, nsteps, init_frame): ) +class EnsembleSatisfiedContinueConditions(object): + def __init__(self, ensembles): + self.satisfied = {ens: False for ens in ensembles} + + def _check_previous_frame(self, trajectory, start, unsatisfied): + # TODO: add some debug logging in here + if -start > len(trajectory): + # we've done the whole traj; don't keep going + return False + subtraj = trajectory[start:] + logger.debug(str(subtraj) + "/" + str(trajectory)) + for ens in unsatisfied: + if not ens.strict_can_prepend(subtraj, trusted=True): + # test if we can't prepend because we satsify + self.satisfied[ens] = ens(subtraj) or ens(subtraj[1:]) + unsatisfied.remove(ens) + return bool(unsatisfied) + + def _call_untrusted(self, trajectory): + self.satisfied = {ens: False for ens in self.satisfied} + for i in range(1, len(trajectory)): + keep_going = self(trajectory[:i], trusted=True) + if not keep_going: + return False + return self(trajectory, trusted=True) + + def __call__(self, trajectory, trusted=False): + if not trusted: + return self._call_untrusted(trajectory) + + # below here, trusted is True + unsatisfied = [ens for ens, done in self.satisfied.items() + if not done] + # TODO: update on how many ensembles left, what frame number we are + + if not unsatisfied: + return False + + start = -1 + while self._check_previous_frame(trajectory, start, unsatisfied): + start -= 1 + + return not all(self.satisfied.values()) + + def md_main(output_storage, engine, ensembles, nsteps, initial_frame): import openpathsampling as paths if nsteps is not None and ensembles: raise RuntimeError("Options --ensemble and --nsteps cannot both be" " used at once.") - if nsteps is not None: - ens = paths.LengthEnsemble(nsteps) + if ensembles: + continue_cond = EnsembleSatisfiedContinueConditions(ensembles) else: - ens = functools.reduce(operator.and_, ensembles) + continue_cond = paths.LengthEnsemble(nsteps).can_append - trajectory = engine.generate(initial_frame, running=ens.can_append) + trajectory = engine.generate(initial_frame, running=continue_cond) if output_storage is not None: output_storage.save(trajectory) output_storage.tags['final_conditions'] = trajectory diff --git a/paths_cli/tests/commands/test_md.py b/paths_cli/tests/commands/test_md.py new file mode 100644 index 00000000..d9d33139 --- /dev/null +++ b/paths_cli/tests/commands/test_md.py @@ -0,0 +1,83 @@ +import pytest +import os +import tempfile +from unittest.mock import patch, Mock +from click.testing import CliRunner + +from paths_cli.commands.md import * + +import openpathsampling as paths + +from openpathsampling.tests.test_helpers import \ + make_1d_traj, CalvinistDynamics + + +class TestEnsembleSatisfiedContinueConditions(object): + def setup(self): + cv = paths.CoordinateFunctionCV('x', lambda x: x.xyz[0][0]) + vol_A = paths.CVDefinedVolume(cv, float("-inf"), 0.0) + vol_B = paths.CVDefinedVolume(cv, 1.0, float("inf")) + ensembles = [ + paths.LengthEnsemble(1).named("len1"), + paths.LengthEnsemble(3).named("len3"), + paths.SequentialEnsemble([ + paths.LengthEnsemble(1) & paths.AllInXEnsemble(vol_A), + paths.AllOutXEnsemble(vol_A | vol_B), + paths.LengthEnsemble(1) & paths.AllInXEnsemble(vol_A) + ]).named('return'), + paths.SequentialEnsemble([ + paths.LengthEnsemble(1) & paths.AllInXEnsemble(vol_A), + paths.AllOutXEnsemble(vol_A | vol_B), + paths.LengthEnsemble(1) & paths.AllInXEnsemble(vol_B) + ]).named('transition'), + ] + self.ensembles = {ens.name: ens for ens in ensembles} + traj_vals = [-0.1, 1.1, 0.5, -0.2, 0.1, -0.3, 0.4, 1.4, -1.0] + self.trajectory = make_1d_traj(traj_vals) + self.engine = CalvinistDynamics(traj_vals) + self.satisfied_when_traj_len = { + "len1": 1, + "len3": 3, + "return": 6, + "transition": 8, + } + self.conditions = EnsembleSatisfiedContinueConditions(ensembles) + + + @pytest.mark.parametrize('trusted', [True, False]) + @pytest.mark.parametrize('traj_len,expected', [ + # expected = (num_calls, num_satisfied) + (0, (1, 0)), + (1, (2, 1)), + (2, (3, 1)), + (3, (3, 2)), + (5, (2, 2)), + (6, (3, 3)), + (7, (1, 3)), + (8, (3, 4)), + ]) + def test_call(self, traj_len, expected, trusted): + if trusted: + already_satisfied = [ + self.ensembles[key] + for key, val in self.satisfied_when_traj_len.items() + if traj_len > val + ] + for ens in already_satisfied: + self.conditions.satisfied[ens] = True + + traj = self.trajectory[:traj_len] + mock = Mock(wraps=self.conditions._check_previous_frame) + self.conditions._check_previous_frame = mock + expected_calls, expected_satisfied = expected + result = self.conditions(traj, trusted) + assert result == (expected_satisfied != 4) + assert sum(self.conditions.satisfied.values()) == expected_satisfied + if trusted: + # only test call count if we're trusted + assert mock.call_count == expected_calls + + def test_generate(self): + init_snap = self.trajectory[0] + traj = self.engine.generate(init_snap, self.conditions) + assert len(traj) == 8 From 830ef03aab26466e242252fc517a1a9594d98db9 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 12:02:42 +0100 Subject: [PATCH 14/36] cleanup --- paths_cli/commands/md.py | 11 +++-------- paths_cli/commands/visit_all.py | 6 +++--- paths_cli/utils.py | 4 ++++ 3 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 paths_cli/utils.py diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py index fdfbc8a2..34f7a7a4 100644 --- a/paths_cli/commands/md.py +++ b/paths_cli/commands/md.py @@ -1,8 +1,6 @@ -import functools -import operator - import click +import paths_cli.utils from paths_cli.parameters import (INPUT_FILE, OUTPUT_FILE, ENGINE, MULTI_ENSEMBLE, INIT_SNAP) @@ -39,7 +37,6 @@ def __init__(self, ensembles): self.satisfied = {ens: False for ens in ensembles} def _check_previous_frame(self, trajectory, start, unsatisfied): - # TODO: add some debug logging in here if -start > len(trajectory): # we've done the whole traj; don't keep going return False @@ -91,10 +88,8 @@ def md_main(output_storage, engine, ensembles, nsteps, initial_frame): continue_cond = paths.LengthEnsemble(nsteps).can_append trajectory = engine.generate(initial_frame, running=continue_cond) - if output_storage is not None: - output_storage.save(trajectory) - output_storage.tags['final_conditions'] = trajectory - + paths_cli.utils.tag_final_result(trajectory, output_storage, + 'final_conditions') return trajectory, None CLI = md diff --git a/paths_cli/commands/visit_all.py b/paths_cli/commands/visit_all.py index 2cb61c1a..4b686b56 100644 --- a/paths_cli/commands/visit_all.py +++ b/paths_cli/commands/visit_all.py @@ -1,5 +1,6 @@ import click +import paths_cli.utils from paths_cli.parameters import (INPUT_FILE, OUTPUT_FILE, ENGINE, STATES, INIT_SNAP) @@ -34,9 +35,8 @@ def visit_all_main(output_storage, states, engine, initial_frame): timestep = getattr(engine, 'timestep', None) visit_all_ens = paths.VisitAllStatesEnsemble(states, timestep=timestep) trajectory = engine.generate(initial_frame, [visit_all_ens.can_append]) - if output_storage is not None: - output_storage.save(trajectory) - output_storage.tags['final_conditions'] = trajectory + paths_cli.utils.tag_final_result(trajectory, output_storage, + 'final_conditions') return trajectory, None # no simulation object to return here diff --git a/paths_cli/utils.py b/paths_cli/utils.py new file mode 100644 index 00000000..cf9c0b92 --- /dev/null +++ b/paths_cli/utils.py @@ -0,0 +1,4 @@ +def tag_final_result(result, storage, tag='final_conditions'): + if storage: + storage.save(result) + storage.tags[tag] = result From 6125554375ccf296141001a676eb595f0d4274f7 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 12:51:27 +0100 Subject: [PATCH 15/36] tests for md command --- paths_cli/tests/commands/test_md.py | 61 ++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/paths_cli/tests/commands/test_md.py b/paths_cli/tests/commands/test_md.py index d9d33139..f00b01ad 100644 --- a/paths_cli/tests/commands/test_md.py +++ b/paths_cli/tests/commands/test_md.py @@ -11,7 +11,6 @@ from openpathsampling.tests.test_helpers import \ make_1d_traj, CalvinistDynamics - class TestEnsembleSatisfiedContinueConditions(object): def setup(self): cv = paths.CoordinateFunctionCV('x', lambda x: x.xyz[0][0]) @@ -81,3 +80,63 @@ def test_generate(self): init_snap = self.trajectory[0] traj = self.engine.generate(init_snap, self.conditions) assert len(traj) == 8 + + +@pytest.fixture() +def md_fixture(tps_fixture): + _, _, engine, sample_set = tps_fixture + snapshot = sample_set[0].trajectory[0] + ensemble = paths.LengthEnsemble(5).named('len5') + return engine, ensemble, snapshot + +def print_test(output_storage, engine, ensembles, nsteps, initial_frame): + print(isinstance(output_storage, paths.Storage)) + print(engine.__uuid__) + print([e.__uuid__ for e in ensembles]) # only 1? + print(nsteps) + print(initial_frame.__uuid__) + +@patch('paths_cli.commands.md.md_main', print_test) +def test_md(md_fixture): + engine, ensemble, snapshot = md_fixture + runner = CliRunner() + with runner.isolated_filesystem(): + storage = paths.Storage("setup.nc", 'w') + storage.save([ensemble, snapshot, engine]) + storage.tags['initial_snapshot'] = snapshot + storage.close() + + results = runner.invoke( + md, + ["setup.nc", '-o', 'foo.nc', '--ensemble', 'len5', '-f', + 'initial_snapshot'] + ) + expected_output = "\n".join([ "True", str(engine.__uuid__), + '[' + str(ensemble.__uuid__) + ']', + 'None', str(snapshot.__uuid__)]) + "\n" + + assert results.output == expected_output + assert results.exit_code == 0 + +def test_md_main(md_fixture): + tempdir = tempfile.mkdtemp() + try: + store_name = os.path.join(tempdir, "md.nc") + storage = paths.Storage(store_name, mode='w') + engine, ensemble, snapshot = md_fixture + traj, foo = md_main( + output_storage=storage, + engine=engine, + ensembles=[ensemble], + nsteps=None, + initial_frame=snapshot + ) + assert isinstance(traj, paths.Trajectory) + assert foo is None + assert len(traj) == 5 + assert len(storage.trajectories) == 1 + storage.close() + finally: + os.remove(store_name) + os.rmdir(tempdir) + From 00b83cc118453283ee827ce5485297a6265364e5 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 14:51:02 +0100 Subject: [PATCH 16/36] more test coverage --- paths_cli/commands/md.py | 3 --- paths_cli/tests/commands/test_md.py | 34 +++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py index 34f7a7a4..b20b8b95 100644 --- a/paths_cli/commands/md.py +++ b/paths_cli/commands/md.py @@ -66,9 +66,6 @@ def __call__(self, trajectory, trusted=False): if not done] # TODO: update on how many ensembles left, what frame number we are - if not unsatisfied: - return False - start = -1 while self._check_previous_frame(trajectory, start, unsatisfied): start -= 1 diff --git a/paths_cli/tests/commands/test_md.py b/paths_cli/tests/commands/test_md.py index f00b01ad..2877844e 100644 --- a/paths_cli/tests/commands/test_md.py +++ b/paths_cli/tests/commands/test_md.py @@ -31,9 +31,9 @@ def setup(self): ]).named('transition'), ] self.ensembles = {ens.name: ens for ens in ensembles} - traj_vals = [-0.1, 1.1, 0.5, -0.2, 0.1, -0.3, 0.4, 1.4, -1.0] - self.trajectory = make_1d_traj(traj_vals) - self.engine = CalvinistDynamics(traj_vals) + self.traj_vals = [-0.1, 1.1, 0.5, -0.2, 0.1, -0.3, 0.4, 1.4, -1.0] + self.trajectory = make_1d_traj(self.traj_vals) + self.engine = CalvinistDynamics(self.traj_vals) self.satisfied_when_traj_len = { "len1": 1, "len3": 3, @@ -76,6 +76,10 @@ def test_call(self, traj_len, expected, trusted): # only test call count if we're trusted assert mock.call_count == expected_calls + def test_long_traj_untrusted(self): + traj = make_1d_traj(self.traj_vals + [1.0, 1.2, 1.3, 1.4]) + assert self.conditions(traj) is False + def test_generate(self): init_snap = self.trajectory[0] traj = self.engine.generate(init_snap, self.conditions) @@ -118,17 +122,25 @@ def test_md(md_fixture): assert results.output == expected_output assert results.exit_code == 0 -def test_md_main(md_fixture): +@pytest.mark.parametrize('inp', ['nsteps', 'ensemble']) +def test_md_main(md_fixture, inp): tempdir = tempfile.mkdtemp() try: store_name = os.path.join(tempdir, "md.nc") storage = paths.Storage(store_name, mode='w') - engine, ensemble, snapshot = md_fixture + engine, ens, snapshot = md_fixture + if inp == 'nsteps': + nsteps, ensembles = 5, None + elif inp == 'ensemble': + nsteps, ensembles = None, [ens] + else: + raise RuntimeError("pytest went crazy") + traj, foo = md_main( output_storage=storage, engine=engine, - ensembles=[ensemble], - nsteps=None, + ensembles=ensembles, + nsteps=nsteps, initial_frame=snapshot ) assert isinstance(traj, paths.Trajectory) @@ -140,3 +152,11 @@ def test_md_main(md_fixture): os.remove(store_name) os.rmdir(tempdir) +def test_md_main_error(md_fixture): + engine, ensemble, snapshot = md_fixture + with pytest.raises(RuntimeError): + md_main(output_storage=None, + engine=engine, + ensembles=[ensemble], + nsteps=5, + initial_frame=snapshot) From dd4ec60b933baac84a3241b2e6353babf57fcd66 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 15:27:31 +0100 Subject: [PATCH 17/36] Some cleanup in contents; complete testing --- paths_cli/commands/contents.py | 36 +++++++++++++++++++---- paths_cli/tests/commands/test_contents.py | 8 +++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/paths_cli/commands/contents.py b/paths_cli/commands/contents.py index 7ddb848c..cf316f5a 100644 --- a/paths_cli/commands/contents.py +++ b/paths_cli/commands/contents.py @@ -4,6 +4,21 @@ UNNAMED_SECTIONS = ['steps', 'movechanges', 'samplesets', 'trajectories', 'snapshots'] +NAME_TO_ATTR = { + 'CVs': 'cvs', + 'Volumes': 'volumes', + 'Engines': 'engines', + 'Networks': 'networks', + 'Move Schemes': 'schemes', + 'Simulations': 'pathsimulators', + 'Tags': 'tags', + 'Steps': 'steps', + 'Move Changes': 'movechanges', + 'SampleSets': 'samplesets', + 'Trajectories': 'trajectories', + 'Snapshots': 'snapshots' +} + @click.command( 'contents', short_help="list named objects from an OPS .nc file", @@ -26,13 +41,22 @@ def contents(input_file, table): try: store = getattr(storage, table_attr) except AttributeError: - print("This needs to raise a good error; bad table name") + raise click.UsageError("Unknown table: '" + table_attr + "'") else: - if table_attr in UNNAMED_SECTIONS: - print(get_unnamed_section_string(table_attr, store)) - else: - print(get_section_string_nameable(table_attr, store, - _get_named_namedobj)) + print(get_section_string(table_attr, store)) + + +def get_section_string(label, store): + attr = NAME_TO_ATTR.get(label, label.lower()) + if attr in UNNAMED_SECTIONS: + string = get_unnamed_section_string(label, store) + elif attr in ['tag', 'tags']: + string = get_section_string_nameable(label, store, _get_named_tags) + else: + string = get_section_string_nameable(label, store, + _get_named_namedobj) + return string + def report_all_tables(storage): store_section_mapping = { diff --git a/paths_cli/tests/commands/test_contents.py b/paths_cli/tests/commands/test_contents.py index 05c953b1..a39385c1 100644 --- a/paths_cli/tests/commands/test_contents.py +++ b/paths_cli/tests/commands/test_contents.py @@ -67,3 +67,11 @@ def test_contents_table(tps_fixture, table): }[table] assert results.output.split("\n") == expected assert results.exit_code == 0 + +def test_contents_table_error(): + runner = CliRunner() + with runner.isolated_filesystem(): + storage = paths.Storage("temp.nc", mode='w') + storage.close() + results = runner.invoke(contents, ['temp.nc', '--table', 'foo']) + assert results.exit_code != 0 From 586f1a7bf596b4294e7c34bcd97d8d74e86d2085 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 15:53:34 +0100 Subject: [PATCH 18/36] docstrings --- paths_cli/commands/md.py | 19 +++++++++++++++++++ paths_cli/utils.py | 11 +++++++++++ 2 files changed, 30 insertions(+) diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py index b20b8b95..aefbf79f 100644 --- a/paths_cli/commands/md.py +++ b/paths_cli/commands/md.py @@ -21,6 +21,13 @@ @INIT_SNAP.clicked(required=False) def md(input_file, output_file, engine, ensemble, nsteps, init_frame): """Run MD for for time of steps or until ensembles are satisfied. + + This can either take a --nsteps or --ensemble, but not both. If the + --ensemble option is specfied more than once, then this will attempt to + run until all ensembles are satisfied by a subtrajectory. + + This still respects the maximum number of frames as set in the engine, + and will terminate if the trajectory gets longer than that. """ storage = INPUT_FILE.get(input_file) md_main( @@ -33,6 +40,18 @@ def md(input_file, output_file, engine, ensemble, nsteps, init_frame): class EnsembleSatisfiedContinueConditions(object): + """Continuation condition for including subtrajs for each ensemble. + + This object creates a continuation condition (a callable) analogous with + the ensemble ``can_append`` method. This will tell the trajectory to + keep running until, for each of the given ensembles, a subtrajectory has + been found that will satisfy the ensemble. + + Parameters + ---------- + ensembles: List[:class:`openpathsampling.Ensemble`] + the ensembles to satisfy + """ def __init__(self, ensembles): self.satisfied = {ens: False for ens in ensembles} diff --git a/paths_cli/utils.py b/paths_cli/utils.py index cf9c0b92..3c483809 100644 --- a/paths_cli/utils.py +++ b/paths_cli/utils.py @@ -1,4 +1,15 @@ def tag_final_result(result, storage, tag='final_conditions'): + """Save results to a tag in storage. + + Parameters + ---------- + result : UUIDObject + the result to store + storage : OPS storage + the output storage + tag : str + the name to tag it with; default is 'final_conditions' + """ if storage: storage.save(result) storage.tags[tag] = result From 586e6e4d2316784a0304bf6c56b5caaa5e55a2f2 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 15:57:04 +0100 Subject: [PATCH 19/36] add test for tags table --- paths_cli/tests/commands/test_contents.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/paths_cli/tests/commands/test_contents.py b/paths_cli/tests/commands/test_contents.py index a39385c1..0d136437 100644 --- a/paths_cli/tests/commands/test_contents.py +++ b/paths_cli/tests/commands/test_contents.py @@ -41,7 +41,7 @@ def test_contents(tps_fixture): for truth, beauty in zip(expected, results.output.split('\n')): assert truth == beauty -@pytest.mark.parametrize('table', ['volumes', 'trajectories']) +@pytest.mark.parametrize('table', ['volumes', 'trajectories', 'tags']) def test_contents_table(tps_fixture, table): scheme, network, engine, init_conds = tps_fixture runner = CliRunner() @@ -63,7 +63,13 @@ def test_contents_table(tps_fixture, table): f"Storage @ '{cwd}/setup.nc'", "trajectories: 1 unnamed item", "" - ] + ], + 'tags': [ + f"Storage @ '{cwd}/setup.nc'", + "tags: 1 item", + "* initial_conditions", + "" + ], }[table] assert results.output.split("\n") == expected assert results.exit_code == 0 From 8df4a543dff7622500ce4dd730daf9dd4084146b Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Wed, 9 Dec 2020 17:08:44 +0100 Subject: [PATCH 20/36] Add progress reporting to md command --- README.md | 1 + paths_cli/commands/md.py | 63 ++++++++++++++++++++++++++++++++++++++-- paths_cli/utils.py | 1 + 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8fead643..d4c5887b 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ miscellaneous operations on OPS output files. **Simulation Commands:** * `visit-all`: Run MD to generate initial trajectories +* `md`: Run MD for fixed time or until a given ensemble is satisfied * `equilibrate`: Run equilibration for path sampling * `pathsampling`: Run any path sampling simulation, including TIS variants diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py index aefbf79f..5a533f67 100644 --- a/paths_cli/commands/md.py +++ b/paths_cli/commands/md.py @@ -38,8 +38,34 @@ def md(input_file, output_file, engine, ensemble, nsteps, init_frame): initial_frame=INIT_SNAP.get(storage, init_frame) ) +class ProgressReporter(object): + def __init__(self, timestep, update_freq): + self.timestep = timestep + self.update_freq = update_freq -class EnsembleSatisfiedContinueConditions(object): + def steps_progress_string(self, n_steps): + report_str = "Ran {n_steps} frames" + if self.timestep is not None: + report_str += " [{}]".format(str(n_steps * timestep)) + report_str += '.' + return report_str + + def progress_string(self, n_steps): + report_str = self.steps_progress_string(n_steps) + "\n" + return report_str.format(n_steps=n_steps) + + + def report_progress(self, n_steps): + import openpathsampling as paths + if n_steps % self.update_freq == 0: + string = self.progress_string(n_steps) + paths.tools.refresh_output(string) + + def __call__(self, trajectory, trusted=False): + raise NotImplementedError() + + +class EnsembleSatisfiedContinueConditions(ProgressReporter): """Continuation condition for including subtrajs for each ensemble. This object creates a continuation condition (a callable) analogous with @@ -52,9 +78,24 @@ class EnsembleSatisfiedContinueConditions(object): ensembles: List[:class:`openpathsampling.Ensemble`] the ensembles to satisfy """ - def __init__(self, ensembles): + def __init__(self, ensembles, timestep=None, update_freq=10): + super().__init__(timestep, update_freq) self.satisfied = {ens: False for ens in ensembles} + def progress_string(self, n_steps): + report_str = self.steps_progress_string(n_steps) + report_str += (" Found ensembles [{found}]. " + "Looking for [{missing}].\n") + found = [ens.name for ens, done in self.satisfied.items() if done] + missing = [ens.name for ens, done in self.satisfied.items() + if not done] + found_str = ",".join(found) + missing_str = ",".join(missing) + return report_str.format(n_steps=n_steps, + found=found_str, + missing=missing_str) + + def _check_previous_frame(self, trajectory, start, unsatisfied): if -start > len(trajectory): # we've done the whole traj; don't keep going @@ -81,6 +122,8 @@ def __call__(self, trajectory, trusted=False): return self._call_untrusted(trajectory) # below here, trusted is True + self.report_progress(len(trajectory) - 1) + unsatisfied = [ens for ens, done in self.satisfied.items() if not done] # TODO: update on how many ensembles left, what frame number we are @@ -92,6 +135,20 @@ def __call__(self, trajectory, trusted=False): return not all(self.satisfied.values()) +class FixedLengthContinueCondition(ProgressReporter): + """ + """ + def __init__(self, length, timestep=None, update_freq=10): + super().__init__(timestep, update_freq) + self.length = length + + def __call__(self, trajectory, trusted=False): + len_traj = len(trajectory) + self.report_progress(len_traj - 1) + return len_traj < self.length + + + def md_main(output_storage, engine, ensembles, nsteps, initial_frame): import openpathsampling as paths if nsteps is not None and ensembles: @@ -101,7 +158,7 @@ def md_main(output_storage, engine, ensembles, nsteps, initial_frame): if ensembles: continue_cond = EnsembleSatisfiedContinueConditions(ensembles) else: - continue_cond = paths.LengthEnsemble(nsteps).can_append + continue_cond = FixedLengthContinueCondition(nsteps) trajectory = engine.generate(initial_frame, running=continue_cond) paths_cli.utils.tag_final_result(trajectory, output_storage, diff --git a/paths_cli/utils.py b/paths_cli/utils.py index 3c483809..5504e702 100644 --- a/paths_cli/utils.py +++ b/paths_cli/utils.py @@ -11,5 +11,6 @@ def tag_final_result(result, storage, tag='final_conditions'): the name to tag it with; default is 'final_conditions' """ if storage: + print("Saving results to output file....") storage.save(result) storage.tags[tag] = result From 7726d8c8036b4a1bb77151666efa496c3842a993 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 11 Dec 2020 18:58:59 +0100 Subject: [PATCH 21/36] Tell InterfaceSet to use SimStore when needed --- paths_cli/param_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paths_cli/param_core.py b/paths_cli/param_core.py index 895b3404..1ec2540c 100644 --- a/paths_cli/param_core.py +++ b/paths_cli/param_core.py @@ -102,6 +102,7 @@ def get(self, name): if not self.has_simstore_patch: paths = monkey_patch_all(paths) + paths.InterfaceSet.simstore = True StorageLoader.has_simstore_patch = True from openpathsampling.experimental.simstore import \ From 8e9fa467a3f53996a76a55a82548beba136f7085 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 15 Dec 2020 15:33:25 +0100 Subject: [PATCH 22/36] more tests; docstrings --- paths_cli/commands/md.py | 44 +++++++++++++++++++++++++---- paths_cli/tests/commands/test_md.py | 26 +++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/paths_cli/commands/md.py b/paths_cli/commands/md.py index 5a533f67..22f36240 100644 --- a/paths_cli/commands/md.py +++ b/paths_cli/commands/md.py @@ -39,25 +39,43 @@ def md(input_file, output_file, engine, ensemble, nsteps, init_frame): ) class ProgressReporter(object): + """Generic class for a callable that reports progress. + + Base class for ends-with-ensemble and fixed-length tricks. + + Parameters + ---------- + timestep : Any + timestep, optionally with units + update_freq : int + how often to report updates + """ def __init__(self, timestep, update_freq): self.timestep = timestep self.update_freq = update_freq def steps_progress_string(self, n_steps): + """Return string for number of frames run and time elapsed + + Not newline-terminated. + """ report_str = "Ran {n_steps} frames" if self.timestep is not None: - report_str += " [{}]".format(str(n_steps * timestep)) + report_str += " [{}]".format(str(n_steps * self.timestep)) report_str += '.' - return report_str + return report_str.format(n_steps=n_steps) def progress_string(self, n_steps): + """Return the progress string. Subclasses may override. + """ report_str = self.steps_progress_string(n_steps) + "\n" return report_str.format(n_steps=n_steps) - - def report_progress(self, n_steps): + def report_progress(self, n_steps, force=False): + """Report the progress to the terminal. + """ import openpathsampling as paths - if n_steps % self.update_freq == 0: + if (n_steps % self.update_freq == 0) or force: string = self.progress_string(n_steps) paths.tools.refresh_output(string) @@ -77,6 +95,10 @@ class EnsembleSatisfiedContinueConditions(ProgressReporter): ---------- ensembles: List[:class:`openpathsampling.Ensemble`] the ensembles to satisfy + timestep : Any + timestep, optionally with units + update_freq : int + how often to report updates """ def __init__(self, ensembles, timestep=None, update_freq=10): super().__init__(timestep, update_freq) @@ -136,7 +158,16 @@ def __call__(self, trajectory, trusted=False): class FixedLengthContinueCondition(ProgressReporter): - """ + """Continuation condition for fixed-length runs. + + Parameters + ---------- + length : int + final length of the trajectory in frames + timestep : Any + timestep, optionally with units + update_freq : int + how often to report updates """ def __init__(self, length, timestep=None, update_freq=10): super().__init__(timestep, update_freq) @@ -161,6 +192,7 @@ def md_main(output_storage, engine, ensembles, nsteps, initial_frame): continue_cond = FixedLengthContinueCondition(nsteps) trajectory = engine.generate(initial_frame, running=continue_cond) + continue_cond.report_progress(len(trajectory) - 1, force=True) paths_cli.utils.tag_final_result(trajectory, output_storage, 'final_conditions') return trajectory, None diff --git a/paths_cli/tests/commands/test_md.py b/paths_cli/tests/commands/test_md.py index 2877844e..f4aab68f 100644 --- a/paths_cli/tests/commands/test_md.py +++ b/paths_cli/tests/commands/test_md.py @@ -11,6 +11,32 @@ from openpathsampling.tests.test_helpers import \ make_1d_traj, CalvinistDynamics +class TestProgressReporter(object): + def setup(self): + self.progress = ProgressReporter(timestep=None, update_freq=5) + + @pytest.mark.parametrize('timestep', [None, 0.1]) + def test_progress_string(self, timestep): + progress = ProgressReporter(timestep, update_freq=5) + expected = "Ran 25 frames" + if timestep is not None: + expected += " [2.5]" + expected += '.\n' + assert progress.progress_string(25) == expected + + @pytest.mark.parametrize('n_steps', [0, 5, 6]) + @pytest.mark.parametrize('force', [True, False]) + @patch('openpathsampling.tools.refresh_output', + lambda s: print(s, end='')) + def test_report_progress(self, n_steps, force, capsys): + self.progress.report_progress(n_steps, force) + expected = "Ran {n_steps} frames.\n".format(n_steps=n_steps) + out, err = capsys.readouterr() + if (n_steps in [0, 5]) or force: + assert out == expected + else: + assert out == "" + class TestEnsembleSatisfiedContinueConditions(object): def setup(self): cv = paths.CoordinateFunctionCV('x', lambda x: x.xyz[0][0]) From 5650b496299baf82bfbc4f6cadab7af2f54d3836 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Tue, 22 Dec 2020 06:55:18 +0100 Subject: [PATCH 23/36] Bump to 0.2.0.dev0; update autorelease --- .github/workflows/autorelease-default-env.sh | 2 +- .github/workflows/autorelease-deploy.yml | 6 ++++-- .github/workflows/autorelease-gh-rel.yml | 4 +++- .github/workflows/autorelease-prep.yml | 8 ++++++-- setup.cfg | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/autorelease-default-env.sh b/.github/workflows/autorelease-default-env.sh index 74dd4615..f1da159b 100644 --- a/.github/workflows/autorelease-default-env.sh +++ b/.github/workflows/autorelease-default-env.sh @@ -1,4 +1,4 @@ -INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3" +INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.6" if [ -f autorelease-env.sh ]; then source autorelease-env.sh fi diff --git a/.github/workflows/autorelease-deploy.yml b/.github/workflows/autorelease-deploy.yml index 89aaeac3..9a8f03bc 100644 --- a/.github/workflows/autorelease-deploy.yml +++ b/.github/workflows/autorelease-deploy.yml @@ -14,7 +14,9 @@ jobs: python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - cat autorelease-env.sh >> $GITHUB_ENV + if [ -f "autorelease-env.sh" ]; then + cat autorelease-env.sh >> $GITHUB_ENV + fi eval $INSTALL_AUTORELEASE name: "Install autorelease" - run: | @@ -27,5 +29,5 @@ jobs: - uses: pypa/gh-action-pypi-publish@master with: password: ${{ secrets.pypi_password }} - name: "Deploy to testpypi" + name: "Deploy to pypi" diff --git a/.github/workflows/autorelease-gh-rel.yml b/.github/workflows/autorelease-gh-rel.yml index f9e294eb..bb5cd276 100644 --- a/.github/workflows/autorelease-gh-rel.yml +++ b/.github/workflows/autorelease-gh-rel.yml @@ -15,7 +15,9 @@ jobs: python-version: "3.7" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - cat autorelease-env.sh >> $GITHUB_ENV + if [ -f "autorelease-env.sh" ]; then + cat autorelease-env.sh >> $GITHUB_ENV + fi eval $INSTALL_AUTORELEASE name: "Install autorelease" - run: | diff --git a/.github/workflows/autorelease-prep.yml b/.github/workflows/autorelease-prep.yml index 48c82ba8..49674133 100644 --- a/.github/workflows/autorelease-prep.yml +++ b/.github/workflows/autorelease-prep.yml @@ -19,7 +19,9 @@ jobs: python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - cat autorelease-env.sh >> $GITHUB_ENV + if [ -f "autorelease-env.sh" ]; then + cat autorelease-env.sh >> $GITHUB_ENV + fi eval $INSTALL_AUTORELEASE name: "Install autorelease" - run: | @@ -49,7 +51,9 @@ jobs: python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - cat autorelease-env.sh >> $GITHUB_ENV + if [ -f "autorelease-env.sh" ]; then + cat autorelease-env.sh >> $GITHUB_ENV + fi eval $INSTALL_AUTORELEASE name: "Install autorelease" - run: test-testpypi diff --git a/setup.cfg b/setup.cfg index 2ca6e44d..c6e305f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = openpathsampling-cli -version = 0.1.1 +version = 0.2.0.dev0 # version should end in .dev0 if this isn't to be released description = Command line tool for OpenPathSampling long_description = file: README.md From 869567681fa73e161ab8ea9be5c49545c5f318fb Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 25 Dec 2020 11:39:38 +0100 Subject: [PATCH 24/36] add reqs for SimStore --- devtools/tests_require.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/devtools/tests_require.txt b/devtools/tests_require.txt index 75b415c2..e5f44cc0 100644 --- a/devtools/tests_require.txt +++ b/devtools/tests_require.txt @@ -3,3 +3,6 @@ nose pytest pytest-cov coveralls +# following are for SimStore integration +dill +sqlalchemy From 107f1db3207887e31514863791026599647b3741 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 1 Jan 2021 18:42:53 +0100 Subject: [PATCH 25/36] Undo SimStore monkey patches after tests --- paths_cli/tests/commands/test_append.py | 6 ++-- paths_cli/tests/test_parameters.py | 40 ++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/paths_cli/tests/commands/test_append.py b/paths_cli/tests/commands/test_append.py index 40d50994..df8a4b90 100644 --- a/paths_cli/tests/commands/test_append.py +++ b/paths_cli/tests/commands/test_append.py @@ -8,13 +8,13 @@ import openpathsampling as paths def make_input_file(tps_network_and_traj): - input_file = paths.Storage("setup.py", mode='w') + input_file = paths.Storage("setup.nc", mode='w') for obj in tps_network_and_traj: input_file.save(obj) input_file.tags['template'] = input_file.snapshots[0] input_file.close() - return "setup.py" + return "setup.nc" def test_append(tps_network_and_traj): runner = CliRunner() @@ -22,8 +22,8 @@ def test_append(tps_network_and_traj): in_file = make_input_file(tps_network_and_traj) result = runner.invoke(append, [in_file, '-a', 'output.nc', '--volume', 'A', '--volume', 'B']) - assert result.exit_code == 0 assert result.exception is None + assert result.exit_code == 0 storage = paths.Storage('output.nc', mode='r') assert len(storage.volumes) == 2 assert len(storage.snapshots) == 0 diff --git a/paths_cli/tests/test_parameters.py b/paths_cli/tests/test_parameters.py index 5b4c6664..0f9adf37 100644 --- a/paths_cli/tests/test_parameters.py +++ b/paths_cli/tests/test_parameters.py @@ -2,10 +2,44 @@ import tempfile import os -import openpathsampling as paths +import paths_cli from openpathsampling.tests.test_helpers import make_1d_traj from paths_cli.parameters import * +import openpathsampling as paths + + +def pre_monkey_patch(): + # store things that get monkey-patched; ensure we un-patch + stored_functions = {} + CallableCV = paths.CallableCV + PseudoAttr = paths.netcdfplus.FunctionPseudoAttribute + stored_functions['CallableCV.from'] = CallableCV.from_dict + stored_functions['PseudoAttr.from'] = PseudoAttr.from_dict + stored_functions['TPSNetwork.from'] = paths.TPSNetwork.from_dict + stored_functions['MISTISNetwork.from'] = paths.MISTISNetwork.from_dict + stored_functions['PseudoAttr.to'] = PseudoAttr.to_dict + stored_functions['TPSNetwork.to'] = paths.TPSNetwork.to_dict + stored_functions['MISTISNetwork.to'] = paths.MISTISNetwork.to_dict + return stored_functions + +def undo_monkey_patch(stored_functions): + CallableCV = paths.CallableCV + PseudoAttr = paths.netcdfplus.FunctionPseudoAttribute + CallableCV.from_dict = stored_functions['CallableCV.from'] + PseudoAttr.from_dict = stored_functions['PseudoAttr.from'] + paths.TPSNetwork.from_dict = stored_functions['TPSNetwork.from'] + paths.MISTISNetwork.from_dict = stored_functions['MISTISNetwork.from'] + PseudoAttr.to_dict = stored_functions['PseudoAttr.to'] + paths.TPSNetwork.to_dict = stored_functions['TPSNetwork.to'] + paths.MISTISNetwork.to_dict = stored_functions['MISTISNetwork.to'] + paths_cli.param_core.StorageLoader.has_simstore_patch = False + paths.InterfaceSet.simstore = False + import importlib + importlib.reload(paths.netcdfplus) + importlib.reload(paths.collectivevariable) + importlib.reload(paths) + class ParameterTest(object): @@ -386,6 +420,7 @@ def test_get(self, getter): @pytest.mark.parametrize('ext', ['nc', 'db', 'sql']) def test_OUTPUT_FILE(ext): + stored_functions = pre_monkey_patch() tempdir = tempfile.mkdtemp() filename = os.path.join(tempdir, "test_output_file." + ext) assert not os.path.exists(filename) @@ -393,9 +428,11 @@ def test_OUTPUT_FILE(ext): assert os.path.exists(filename) os.remove(filename) os.rmdir(tempdir) + undo_monkey_patch(stored_functions) @pytest.mark.parametrize('ext', ['nc', 'db', 'sql']) def test_APPEND_FILE(ext): + stored_functions = pre_monkey_patch() tempdir = tempfile.mkdtemp() filename = os.path.join(tempdir, "test_append_file." + ext) assert not os.path.exists(filename) @@ -414,3 +451,4 @@ def test_APPEND_FILE(ext): storage.close() os.remove(filename) os.rmdir(tempdir) + undo_monkey_patch(stored_functions) From 17a031950fc6e79b2b19f9d775c430cf98d17ba2 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 1 Jan 2021 20:03:23 +0100 Subject: [PATCH 26/36] add a couple tests to complete coverage --- paths_cli/tests/test_parameters.py | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/paths_cli/tests/test_parameters.py b/paths_cli/tests/test_parameters.py index 0f9adf37..b55c3714 100644 --- a/paths_cli/tests/test_parameters.py +++ b/paths_cli/tests/test_parameters.py @@ -148,6 +148,17 @@ def setup(self): def test_get(self, getter): self._getter_test(getter) + def test_cannot_guess(self): + filename = self._filename('no-guess') + storage = paths.Storage(filename, 'w') + storage.save(self.engine) + storage.save(self.other_engine.named('other')) + storage.close() + + storage = paths.Storage(filename, mode='r') + with pytest.raises(RuntimeError): + self.PARAMETER.get(storage, None) + class TestSCHEME(ParamInstanceTest): PARAMETER = SCHEME @@ -275,6 +286,17 @@ def test_get_multiple(self): assert traj0 == self.traj assert traj1 == self.other_traj + def test_cannot_guess(self): + filename = self._filename('no-guess') + storage = paths.Storage(filename, 'w') + storage.save(self.traj) + storage.save(self.other_traj) + storage.close() + + storage = paths.Storage(filename, 'r') + with pytest.raises(RuntimeError): + self.PARAMETER.get(storage, None) + class TestINIT_SNAP(ParamInstanceTest): PARAMETER = INIT_SNAP @@ -312,6 +334,18 @@ def test_get(self, getter): obj = self.PARAMETER.get(storage, get_arg) assert obj == expected + def test_simstore_single_snapshot(self): + stored_functions = pre_monkey_patch() + filename = os.path.join(self.tempdir, "simstore.db") + storage = APPEND_FILE.get(filename) + storage.save(self.init_snap) + storage.close() + + storage = INPUT_FILE.get(filename) + snap = self.PARAMETER.get(storage, None) + assert snap == self.init_snap + undo_monkey_patch(stored_functions) + class MultiParamInstanceTest(ParamInstanceTest): def _getter_test(self, getter): From 3daed14d3f9732b0b35cf3ceb962d16aac2e3362 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 1 Jan 2021 20:09:50 +0100 Subject: [PATCH 27/36] Release 0.2.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index c6e305f0..4969019c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = openpathsampling-cli -version = 0.2.0.dev0 +version = 0.2.0 # version should end in .dev0 if this isn't to be released description = Command line tool for OpenPathSampling long_description = file: README.md From aa58da03493af51cdebda0b12877b00e4a782c22 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 1 Jan 2021 20:28:07 +0100 Subject: [PATCH 28/36] use custom test-testpypi to add simstore reqs --- autorelease-env.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/autorelease-env.sh b/autorelease-env.sh index 3b488eb7..123ccc3f 100644 --- a/autorelease-env.sh +++ b/autorelease-env.sh @@ -1,2 +1,6 @@ INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose" PACKAGE_IMPORT_NAME=paths_cli +AUTORELEASE_TEST_TESTPYPI=" +python -m pip install sqlalchemy dill pytest; +py.test --pyargs $PACKAGE_IMPORTNAME +" From b7b22e776040318a411d3e0349c0f270b324705a Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 1 Jan 2021 20:32:20 +0100 Subject: [PATCH 29/36] one-liner of test-testpypi these should be optional scripts with default behaviors --- autorelease-env.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/autorelease-env.sh b/autorelease-env.sh index 123ccc3f..5a72c8a9 100644 --- a/autorelease-env.sh +++ b/autorelease-env.sh @@ -1,6 +1,3 @@ INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose" PACKAGE_IMPORT_NAME=paths_cli -AUTORELEASE_TEST_TESTPYPI=" -python -m pip install sqlalchemy dill pytest; -py.test --pyargs $PACKAGE_IMPORTNAME -" +AUTORELEASE_TEST_TESTPYPI="python -m pip install sqlalchemy dill pytest; py.test --pyargs $PACKAGE_IMPORTNAME" From 7fdf2cf6b406a54936a8c7998ef1bf842e18c71d Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Fri, 1 Jan 2021 22:41:53 +0100 Subject: [PATCH 30/36] will && work? --- autorelease-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorelease-env.sh b/autorelease-env.sh index 5a72c8a9..50f6bf62 100644 --- a/autorelease-env.sh +++ b/autorelease-env.sh @@ -1,3 +1,3 @@ INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose" PACKAGE_IMPORT_NAME=paths_cli -AUTORELEASE_TEST_TESTPYPI="python -m pip install sqlalchemy dill pytest; py.test --pyargs $PACKAGE_IMPORTNAME" +AUTORELEASE_TEST_TESTPYPI="python -m pip install sqlalchemy dill pytest && py.test --pyargs $PACKAGE_IMPORTNAME" From 3931c993319174fe3a54d0eea1db49c4ac4c2521 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sat, 2 Jan 2021 23:28:51 +0100 Subject: [PATCH 31/36] Fix bash variable --- autorelease-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autorelease-env.sh b/autorelease-env.sh index 50f6bf62..80ef1470 100644 --- a/autorelease-env.sh +++ b/autorelease-env.sh @@ -1,3 +1,3 @@ INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose" PACKAGE_IMPORT_NAME=paths_cli -AUTORELEASE_TEST_TESTPYPI="python -m pip install sqlalchemy dill pytest && py.test --pyargs $PACKAGE_IMPORTNAME" +AUTORELEASE_TEST_TESTPYPI="python -m pip install sqlalchemy dill pytest && py.test --pyargs $PACKAGE_IMPORT_NAME" From ecc32f2da5751788fe7374a681c82f936c8e5571 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sat, 2 Jan 2021 23:37:29 +0100 Subject: [PATCH 32/36] Try updating autorelease vendoring --- .github/workflows/autorelease-default-env.sh | 2 +- .github/workflows/autorelease-prep.yml | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autorelease-default-env.sh b/.github/workflows/autorelease-default-env.sh index f1da159b..74dd4615 100644 --- a/.github/workflows/autorelease-default-env.sh +++ b/.github/workflows/autorelease-default-env.sh @@ -1,4 +1,4 @@ -INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.6" +INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3" if [ -f autorelease-env.sh ]; then source autorelease-env.sh fi diff --git a/.github/workflows/autorelease-prep.yml b/.github/workflows/autorelease-prep.yml index 49674133..2df2c796 100644 --- a/.github/workflows/autorelease-prep.yml +++ b/.github/workflows/autorelease-prep.yml @@ -1,3 +1,5 @@ +# File vencdored from Autorelease; specific version information should be in +# the INSTALL_AUTORELEASE variable in autorelease-default-env.sh name: "Autorelease" on: pull_request: @@ -51,7 +53,9 @@ jobs: python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - if [ -f "autorelease-env.sh" ]; then + if [ -f ".autorelease/test-testpypi.sh" ]; then + source ./autorelease/test-testpypi.sh + elif [ -f "autorelease-env.sh" ]; then cat autorelease-env.sh >> $GITHUB_ENV fi eval $INSTALL_AUTORELEASE From e36d2019bd5482f4d062adb4cecf2028a15b5636 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sat, 2 Jan 2021 23:49:03 +0100 Subject: [PATCH 33/36] don't forget to add the file! --- .autorelease/test-testpypi.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .autorelease/test-testpypi.sh diff --git a/.autorelease/test-testpypi.sh b/.autorelease/test-testpypi.sh new file mode 100644 index 00000000..d4509bb2 --- /dev/null +++ b/.autorelease/test-testpypi.sh @@ -0,0 +1,2 @@ +python -m pip install sqlalchemy dill pytest +py.test --pyargs paths_cli From a2a10aa328aeed24304925022a06bb17a6e50130 Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Sun, 3 Jan 2021 00:16:19 +0100 Subject: [PATCH 34/36] fix missing dots --- .github/workflows/autorelease-default-env.sh | 2 +- .github/workflows/autorelease-prep.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/autorelease-default-env.sh b/.github/workflows/autorelease-default-env.sh index 74dd4615..f1da159b 100644 --- a/.github/workflows/autorelease-default-env.sh +++ b/.github/workflows/autorelease-default-env.sh @@ -1,4 +1,4 @@ -INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3" +INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.6" if [ -f autorelease-env.sh ]; then source autorelease-env.sh fi diff --git a/.github/workflows/autorelease-prep.yml b/.github/workflows/autorelease-prep.yml index 2df2c796..6c6b42c1 100644 --- a/.github/workflows/autorelease-prep.yml +++ b/.github/workflows/autorelease-prep.yml @@ -53,8 +53,8 @@ jobs: python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - if [ -f ".autorelease/test-testpypi.sh" ]; then - source ./autorelease/test-testpypi.sh + if [ -f "./.autorelease/test-testpypi.sh" ]; then + source ./.autorelease/test-testpypi.sh elif [ -f "autorelease-env.sh" ]; then cat autorelease-env.sh >> $GITHUB_ENV fi From b77dac4f0cad819b3f700d0d46c48d996553550d Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Mon, 4 Jan 2021 15:01:50 +0100 Subject: [PATCH 35/36] install extras in autorelease install --- .github/workflows/autorelease-prep.yml | 6 +----- autorelease-env.sh | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/autorelease-prep.yml b/.github/workflows/autorelease-prep.yml index 6c6b42c1..49674133 100644 --- a/.github/workflows/autorelease-prep.yml +++ b/.github/workflows/autorelease-prep.yml @@ -1,5 +1,3 @@ -# File vencdored from Autorelease; specific version information should be in -# the INSTALL_AUTORELEASE variable in autorelease-default-env.sh name: "Autorelease" on: pull_request: @@ -53,9 +51,7 @@ jobs: python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh - if [ -f "./.autorelease/test-testpypi.sh" ]; then - source ./.autorelease/test-testpypi.sh - elif [ -f "autorelease-env.sh" ]; then + if [ -f "autorelease-env.sh" ]; then cat autorelease-env.sh >> $GITHUB_ENV fi eval $INSTALL_AUTORELEASE diff --git a/autorelease-env.sh b/autorelease-env.sh index 80ef1470..a6e6bc5b 100644 --- a/autorelease-env.sh +++ b/autorelease-env.sh @@ -1,3 +1,3 @@ -INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose" +# include extra packages when installing autorelease +INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose sqlalchemy dill" PACKAGE_IMPORT_NAME=paths_cli -AUTORELEASE_TEST_TESTPYPI="python -m pip install sqlalchemy dill pytest && py.test --pyargs $PACKAGE_IMPORT_NAME" From 3fb2f493485e417f74963f76e93f5cc9224eb36f Mon Sep 17 00:00:00 2001 From: "David W.H. Swenson" Date: Mon, 4 Jan 2021 15:05:14 +0100 Subject: [PATCH 36/36] remove comment --- autorelease-env.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/autorelease-env.sh b/autorelease-env.sh index a6e6bc5b..9074cb38 100644 --- a/autorelease-env.sh +++ b/autorelease-env.sh @@ -1,3 +1,2 @@ -# include extra packages when installing autorelease INSTALL_AUTORELEASE="python -m pip install autorelease==0.2.3 nose sqlalchemy dill" PACKAGE_IMPORT_NAME=paths_cli