From 0eb26a6a375f9f430219ade19319f502ad026725 Mon Sep 17 00:00:00 2001 From: Tony Fast Date: Tue, 8 May 2018 00:20:00 -0400 Subject: [PATCH] Add a file watcher with watchdog (#29) * Create a watchdog trick for modules that is sensitive to notebook paths Fixes #28 * Catch an error when a file is created in the watcher * Use the watcher while developing importnb * Rename the unittests to something more canonical. * Add the github pages deployment to travis. * Fix the watcher instructions in the readme. * Add the watchdog requirement for the setup * Manually install pyyaml from to pass 3.7dev * Add context to the readme * Deploy pages on any travis_branch * Escape the profile read error int he ipython extension * add pip to the cache on travis --- readme.ipynb | 209 +++++++++++++++----- readme.md | 162 ++++++++++----- requirements-tox.txt | 3 + setup.py | 1 + src/importnb/exporter.py | 12 +- src/importnb/loader.py | 31 ++- src/importnb/utils/__init__.py | 29 +++ src/importnb/utils/ipython.py | 49 ++--- src/importnb/utils/watch.py | 34 ++++ src/notebooks/exporter.ipynb | 40 +--- src/notebooks/loader.ipynb | 89 ++++++--- src/notebooks/utils/__init__.ipynb | 80 ++++++++ src/notebooks/utils/__init__.py | 26 +++ src/notebooks/utils/ipython.ipynb | 10 +- src/notebooks/utils/watch.ipynb | 93 +++++++++ tests/__init__.py | 4 +- tests/test_importnb.ipynb | 208 +++++++++++++------ tests/{test_.ipynb => test_unittests.ipynb} | 0 tox.ini | 18 +- tricks.yaml | 14 +- 20 files changed, 822 insertions(+), 290 deletions(-) create mode 100644 requirements-tox.txt create mode 100644 src/importnb/utils/watch.py create mode 100644 src/notebooks/utils/__init__.ipynb create mode 100644 src/notebooks/utils/__init__.py create mode 100644 src/notebooks/utils/watch.ipynb rename tests/{test_.ipynb => test_unittests.ipynb} (100%) diff --git a/readme.ipynb b/readme.ipynb index fcdc211..05539a2 100644 --- a/readme.ipynb +++ b/readme.ipynb @@ -4,9 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "__importnb__ supports the ability to use Jupyter notebooks as python source.\n", + "__importnb__ imports notebooks as modules & packages.\n", + "\n", + "[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/deathbeds/importnb/master?urlpath=lab/tree/readme.ipynb)[![Build Status](https://travis-ci.org/deathbeds/importnb.svg?branch=master)](https://travis-ci.org/deathbeds/importnb)[![PyPI version](https://badge.fury.io/py/importnb.svg)](https://badge.fury.io/py/importnb)![PyPI - Python Version](https://img.shields.io/pypi/pyversions/importnb.svg)![PyPI - Format](https://img.shields.io/pypi/format/importnb.svg)![PyPI - Format](https://img.shields.io/pypi/l/importnb.svg)\n", + "\n", "\n", - "[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/deathbeds/importnb/master?urlpath=lab/tree/readme.ipynb)[![Build Status](https://travis-ci.org/deathbeds/importnb.svg?branch=master)](https://travis-ci.org/deathbeds/importnb)\n", "\n", " pip install importnb" ] @@ -15,9 +17,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Jupyter Extension\n", + "# `importnb` works in Python and IPython\n", "\n", - " %load_ext importnb " + "Use the `Notebook` context manager." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### For brevity" ] }, { @@ -26,17 +35,15 @@ "metadata": {}, "outputs": [], "source": [ - " foo = 42\n", - " import readme\n", - " assert readme.foo is 42\n", - " assert readme.__file__.endswith('.ipynb')" + " with __import__('importnb').Notebook(): \n", + " import readme" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Notebooks maybe reloaded with the standard Python Import machinery." + "#### or explicity " ] }, { @@ -45,34 +52,40 @@ "metadata": {}, "outputs": [], "source": [ - " from importnb import Notebook, reload\n", - " reload(readme);" + " from importnb import Notebook\n", + " with Notebook(): \n", + " import readme" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 3, "metadata": {}, + "outputs": [], "source": [ - "## Unload the extension\n", - "\n", - " %unload_ext importnb" + " foo = 42\n", + " import readme\n", + " assert readme.foo is 42\n", + " assert readme.__file__.endswith('.ipynb')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Context Manager" + "### Modules may be reloaded \n", + "\n", + "The context manager is required to `reload` a module." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - " with Notebook(): \n", - " import readme\n", + " from importlib import reload\n", + " with Notebook():\n", " reload(readme)" ] }, @@ -80,9 +93,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Integrations\n", - "\n", - "`importnb` integrates with IPython, py.test, and setuptools.\n" + "## Integrations\n" ] }, { @@ -91,24 +102,31 @@ "source": [ "### IPython\n", "\n", - "`importnb` may allow notebooks to import by default with \n", + "#### Extension\n", "\n", - " ipython -c \"__import__('importnb').utils.ipython.install()\"\n", + "Avoid the use of the context manager using loading importnb as IPython extension.\n", + "\n", + " %load_ext importnb\n", " \n", - "This extension will install a script into the default IPython profile startup that is called each time an IPython session is created. \n", + "`%unload_ext importnb` will unload the extension.\n", + "\n", + "#### Default Extension\n", "\n", - "#### Command\n", + "`importnb` may allow notebooks to import by default with \n", "\n", - "After the `importnb` extension is created notebooks can be execute from the command line.\n", + " importnb-install\n", + " \n", + "This extension will install a script into the default IPython profile startup that is called each time an IPython session is created. \n", "\n", - " ipython -m readme\n", + "Uninstall the extension with `importnb-install`.\n", "\n", - "### Unloading the Extension\n", + "##### Run a notebook as a module\n", "\n", - "The default settings may be discarded temporarily with\n", + "When the default extension is loaded any notebook can be run from the command line. After the `importnb` extension is created notebooks can be execute from the command line.\n", "\n", - " %unload_ext importnb\n", - " " + " ipython -m readme\n", + " \n", + "> See the [deploy step in the travis build](https://github.com/deathbeds/importnb/blob/docs/.travis.yml#L19)." ] }, { @@ -135,6 +153,30 @@ " ...,)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### [Watchdog](https://github.com/gorakhargosh/watchdog/tree/master/src/watchdog/tricks)\n", + "\n", + "`importnb` exports a watchdog trick to watch files and apply command like operations on their module path.\n", + "\n", + "#### Tricks File\n", + "\n", + "For example, create a file called `tricks.yaml` containing\n", + "\n", + " tricks:\n", + " - importnb.utils.watch.ModuleTrick:\n", + " patterns: ['*.ipynb']\n", + " shell_command: ipython -m ${watch_dest_path}\n", + " \n", + "#### Run the watcher in a terminal\n", + "\n", + " watchmedo tricks tricks.yaml\n", + " \n", + "> [`tricks.yaml`](tricks.yaml) is a concrete implementation of `tricks.yaml`" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -142,26 +184,58 @@ "## Developer" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* [Tests](tests/test_importnb.ipynb)\n", + "* [Source Notebooks](src/notebooks/)\n", + "* [Transpiled Python Source](src/importnb/)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Format and test the Source Code" + ] + }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "src/notebooks/compiler_ipython.ipynb\n", + "src/notebooks/compiler_python.ipynb\n", + "src/notebooks/decoder.ipynb\n", + "src/notebooks/exporter.ipynb\n", + "src/notebooks/loader.ipynb\n", + "src/notebooks/utils/__init__.ipynb\n", + "src/notebooks/utils/ipython.ipynb\n", + "src/notebooks/utils/pytest_plugin.ipynb\n", + "src/notebooks/utils/setup.ipynb\n", + "src/notebooks/utils/watch.ipynb\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "test_import (tests.test_.TestContext) ... ok\n", - "test_reload_with_context (tests.test_.TestContext) ... ok\n", - "test_reload_without_context (tests.test_.TestContext) ... skipped 'importnb is probably installed'\n", - "test_failure (tests.test_.TestExtension) ... expected failure\n", - "test_import (tests.test_.TestExtension) ... ok\n", - "test_exception (tests.test_.TestPartial) ... ok\n", - "test_traceback (tests.test_.TestPartial) ... ok\n", - "test_imports (tests.test_.TestRemote) ... skipped 'requires IP'\n", + "test_import (tests.test_unittests.TestContext) ... ok\n", + "test_reload_with_context (tests.test_unittests.TestContext) ... ok\n", + "test_reload_without_context (tests.test_unittests.TestContext) ... skipped 'importnb is probably installed'\n", + "test_failure (tests.test_unittests.TestExtension) ... expected failure\n", + "test_import (tests.test_unittests.TestExtension) ... ok\n", + "test_exception (tests.test_unittests.TestPartial) ... ok\n", + "test_traceback (tests.test_unittests.TestPartial) ... ok\n", + "test_imports (tests.test_unittests.TestRemote) ... skipped 'requires IP'\n", "\n", "----------------------------------------------------------------------\n", - "Ran 8 tests in 2.021s\n", + "Ran 8 tests in 2.023s\n", "\n", "OK (skipped=2, expected failures=1)\n" ] @@ -171,16 +245,57 @@ " if __name__ == '__main__':\n", " from pathlib import Path\n", " import black\n", - " from nbconvert.exporters.markdown import MarkdownExporter\n", " from importnb.compiler_python import ScriptExporter\n", " for path in Path('src/notebooks/').rglob(\"\"\"*.ipynb\"\"\"):\n", - " \n", - " 'checkpoint' not in str(path) and (Path('src/importnb') / path.with_suffix('.py').relative_to('src/notebooks')).write_text(\n", + " if 'checkpoint' not in str(path):\n", + " print(path)\n", + " (Path('src/importnb') / path.with_suffix('.py').relative_to('src/notebooks')).write_text(\n", " black.format_str(ScriptExporter().from_filename(path)[0], 100))\n", + " \n", + " __import__('unittest').main(module='tests', argv=\"discover --verbose\".split(), exit=False) \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Format the Github markdown files" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + " if __name__ == '__main__':\n", + " from nbconvert.exporters.markdown import MarkdownExporter\n", " for path in map(Path, ('readme.ipynb', 'changelog.ipynb')):\n", - " path.with_suffix('.md').write_text(MarkdownExporter().from_filename(path)[0])\n", + " path.with_suffix('.md').write_text(MarkdownExporter().from_filename(path)[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Format the Github Pages documentation\n", "\n", - " __import__('unittest').main(module='tests', argv=\"discover --verbose\".split(), exit=False)\n" + "We use `/docs` as the `local_dir`." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + " if __name__ == '__main__':\n", + " from nbconvert.exporters.markdown import MarkdownExporter\n", + " files = 'readme.ipynb', 'changelog.ipynb', 'tests/test_importnb.ipynb'\n", + " for doc in map(Path, files):\n", + " to = ('docs' / doc.with_suffix('.md'))\n", + " to.parent.mkdir(exist_ok=True)\n", + " to.write_text(MarkdownExporter().from_filename(doc)[0])" ] }, { diff --git a/readme.md b/readme.md index 2061a80..931da3e 100644 --- a/readme.md +++ b/readme.md @@ -1,68 +1,82 @@ -__importnb__ supports the ability to use Jupyter notebooks as python source. +__importnb__ imports notebooks as modules & packages. + +[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/deathbeds/importnb/master?urlpath=lab/tree/readme.ipynb)[![Build Status](https://travis-ci.org/deathbeds/importnb.svg?branch=master)](https://travis-ci.org/deathbeds/importnb)[![PyPI version](https://badge.fury.io/py/importnb.svg)](https://badge.fury.io/py/importnb)![PyPI - Python Version](https://img.shields.io/pypi/pyversions/importnb.svg)![PyPI - Format](https://img.shields.io/pypi/format/importnb.svg)![PyPI - Format](https://img.shields.io/pypi/l/importnb.svg) + -[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/deathbeds/importnb/master?urlpath=lab/tree/readme.ipynb)[![Build Status](https://travis-ci.org/deathbeds/importnb.svg?branch=master)](https://travis-ci.org/deathbeds/importnb) pip install importnb -## Jupyter Extension +# `importnb` works in Python and IPython - %load_ext importnb +Use the `Notebook` context manager. + +### For brevity ```python - foo = 42 - import readme - assert readme.foo is 42 - assert readme.__file__.endswith('.ipynb') + with __import__('importnb').Notebook(): + import readme ``` -Notebooks maybe reloaded with the standard Python Import machinery. +#### or explicity ```python - from importnb import Notebook, reload - reload(readme); + from importnb import Notebook + with Notebook(): + import readme ``` -## Unload the extension - %unload_ext importnb +```python + foo = 42 + import readme + assert readme.foo is 42 + assert readme.__file__.endswith('.ipynb') +``` + +### Modules may be reloaded -## Context Manager +The context manager is required to `reload` a module. ```python - with Notebook(): - import readme + from importlib import reload + with Notebook(): reload(readme) ``` ## Integrations -`importnb` integrates with IPython, py.test, and setuptools. - ### IPython -`importnb` may allow notebooks to import by default with +#### Extension - ipython -c "__import__('importnb').utils.ipython.install()" +Avoid the use of the context manager using loading importnb as IPython extension. + + %load_ext importnb -This extension will install a script into the default IPython profile startup that is called each time an IPython session is created. +`%unload_ext importnb` will unload the extension. -#### Command +#### Default Extension -After the `importnb` extension is created notebooks can be execute from the command line. +`importnb` may allow notebooks to import by default with - ipython -m readme + importnb-install + +This extension will install a script into the default IPython profile startup that is called each time an IPython session is created. -### Unloading the Extension +Uninstall the extension with `importnb-install`. -The default settings may be discarded temporarily with +##### Run a notebook as a module - %unload_ext importnb +When the default extension is loaded any notebook can be run from the command line. After the `importnb` extension is created notebooks can be execute from the command line. + + ipython -m readme +> See the [deploy step in the travis build](https://github.com/deathbeds/importnb/blob/docs/.travis.yml#L19). ### py.test @@ -78,37 +92,97 @@ The default settings may be discarded temporarily with cmdclass=dict(build_py=build_ipynb) ...,) +### [Watchdog](https://github.com/gorakhargosh/watchdog/tree/master/src/watchdog/tricks) + +`importnb` exports a watchdog trick to watch files and apply command like operations on their module path. + +#### Tricks File + +For example, create a file called `tricks.yaml` containing + + tricks: + - importnb.utils.watch.ModuleTrick: + patterns: ['*.ipynb'] + shell_command: ipython -m ${watch_dest_path} + +#### Run the watcher in a terminal + + watchmedo tricks tricks.yaml + +> [`tricks.yaml`](tricks.yaml) is a concrete implementation of `tricks.yaml` + ## Developer +* [Tests](tests/test_importnb.ipynb) +* [Source Notebooks](src/notebooks/) +* [Transpiled Python Source](src/importnb/) + +### Format and test the Source Code + ```python if __name__ == '__main__': from pathlib import Path import black - from nbconvert.exporters.markdown import MarkdownExporter from importnb.compiler_python import ScriptExporter for path in Path('src/notebooks/').rglob("""*.ipynb"""): - - 'checkpoint' not in str(path) and (Path('src/importnb') / path.with_suffix('.py').relative_to('src/notebooks')).write_text( + if 'checkpoint' not in str(path): + print(path) + (Path('src/importnb') / path.with_suffix('.py').relative_to('src/notebooks')).write_text( black.format_str(ScriptExporter().from_filename(path)[0], 100)) - for path in map(Path, ('readme.ipynb', 'changelog.ipynb')): - path.with_suffix('.md').write_text(MarkdownExporter().from_filename(path)[0]) - - __import__('unittest').main(module='tests', argv="discover --verbose".split(), exit=False) + + __import__('unittest').main(module='tests', argv="discover --verbose".split(), exit=False) ``` - test_import (tests.test_.TestContext) ... ok - test_reload_with_context (tests.test_.TestContext) ... ok - test_reload_without_context (tests.test_.TestContext) ... skipped 'importnb is probably installed' - test_failure (tests.test_.TestExtension) ... expected failure - test_import (tests.test_.TestExtension) ... ok - test_exception (tests.test_.TestPartial) ... ok - test_traceback (tests.test_.TestPartial) ... ok - test_imports (tests.test_.TestRemote) ... skipped 'requires IP' + src/notebooks/compiler_ipython.ipynb + src/notebooks/compiler_python.ipynb + src/notebooks/decoder.ipynb + src/notebooks/exporter.ipynb + src/notebooks/loader.ipynb + src/notebooks/utils/__init__.ipynb + src/notebooks/utils/ipython.ipynb + src/notebooks/utils/pytest_plugin.ipynb + src/notebooks/utils/setup.ipynb + src/notebooks/utils/watch.ipynb + + + test_import (tests.test_unittests.TestContext) ... ok + test_reload_with_context (tests.test_unittests.TestContext) ... ok + test_reload_without_context (tests.test_unittests.TestContext) ... skipped 'importnb is probably installed' + test_failure (tests.test_unittests.TestExtension) ... unexpected success + test_import (tests.test_unittests.TestExtension) ... ok + test_exception (tests.test_unittests.TestPartial) ... ok + test_traceback (tests.test_unittests.TestPartial) ... ok + test_imports (tests.test_unittests.TestRemote) ... skipped 'requires IP' ---------------------------------------------------------------------- - Ran 8 tests in 2.021s + Ran 8 tests in 1.013s - OK (skipped=2, expected failures=1) + FAILED (skipped=2, unexpected successes=1) + + +### Format the Github markdown files + +```python + if __name__ == '__main__': + from nbconvert.exporters.markdown import MarkdownExporter + for path in map(Path, ('readme.ipynb', 'changelog.ipynb')): + path.with_suffix('.md').write_text(MarkdownExporter().from_filename(path)[0]) +``` + +### Format the Github Pages documentation + +We use `/docs` as the `local_dir`. + + +```python + if __name__ == '__main__': + from nbconvert.exporters.markdown import MarkdownExporter + files = 'readme.ipynb', 'changelog.ipynb', 'tests/test_importnb.ipynb' + for doc in map(Path, files): + to = ('docs' / doc.with_suffix('.md')) + to.parent.mkdir(exist_ok=True) + to.write_text(MarkdownExporter().from_filename(doc)[0]) +``` diff --git a/requirements-tox.txt b/requirements-tox.txt new file mode 100644 index 0000000..776f4a0 --- /dev/null +++ b/requirements-tox.txt @@ -0,0 +1,3 @@ +pytest +pytest-cov +git+https://github.com/yaml/pyyaml.git diff --git a/setup.py b/setup.py index 10664a0..7854b6c 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires=[ "dataclasses", "nbconvert", + "watchdog", ], include_package_data=True, packages=setuptools.find_packages(where="src"), diff --git a/src/importnb/exporter.py b/src/importnb/exporter.py index 6563617..407c0f1 100644 --- a/src/importnb/exporter.py +++ b/src/importnb/exporter.py @@ -1,21 +1,13 @@ try: from .decoder import LineNoDecoder + from .utils import __IPYTHON__ except: from decoder import LineNoDecoder + from utils import __IPYTHON__ __file__ = globals().get("__file__", "exporter.ipynb") __nb__ = __file__.replace("src/importnb", "src/notebooks") -__IPYTHON__ = False -try: - from IPython import get_ipython - - if not get_ipython(): - raise ValueError("""There is no interactive IPython shell""") - __IPYTHON__ = True -except: - ... - if __IPYTHON__: try: from .compiler_ipython import Compiler, NotebookNode, NotebookExporter diff --git a/src/importnb/loader.py b/src/importnb/loader.py index 00628bb..2afa253 100644 --- a/src/importnb/loader.py +++ b/src/importnb/loader.py @@ -1,7 +1,9 @@ try: from .exporter import Compile, AST + from .utils import __IPYTHON__, export except: from exporter import Compile, AST + from utils import __IPYTHON__, export import inspect, sys from importlib.machinery import SourceFileLoader from importlib._bootstrap_external import FileFinder @@ -9,19 +11,16 @@ from traceback import print_exc from contextlib import contextmanager -__IPYTHON__ = False -try: - from IPython import get_ipython - - if not get_ipython(): - raise ValueError("""There is no interactive IPython shell""") - __IPYTHON__ = True -except: - ... +__all__ = "Notebook", "Partial", "reload", @contextmanager def modify_file_finder_details(): + """yield the FileFinder in the sys.path_hooks that loads Python files and assure + the import cache is cleared afterwards. + + Everything goes to shit if the import cache is not cleared.""" + for id, hook in enumerate(sys.path_hooks): try: closure = inspect.getclosurevars(hook).nonlocals @@ -51,7 +50,9 @@ def remove_one_path_hook(loader): break -class ImportContextMixin: +class ImportContextManagerMixin: + """A context maanager to add and remove loader_details from the path_hooks. + """ def __enter__(self, position=0): add_path_hooks(type(self), self.EXTENSION_SUFFIXES, position=position) @@ -60,7 +61,7 @@ def __exit__(self, exception_type=None, exception_value=None, traceback=None): remove_one_path_hook(type(self)) -class Notebook(SourceFileLoader, ImportContextMixin): +class Notebook(SourceFileLoader, ImportContextManagerMixin): """A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.""" EXTENSION_SUFFIXES = ".ipynb", @@ -121,11 +122,5 @@ def unload_ipython_extension(ip=None): if __name__ == "__main__": - try: - from .compiler_python import ScriptExporter - except: - from compiler_python import ScriptExporter - from pathlib import Path - - Path("../importnb/loader.py").write_text(ScriptExporter().from_filename("loader.ipynb")[0]) + export("loader.ipynb", "../importnb/loader.py") __import__("doctest").testmod() diff --git a/src/importnb/utils/__init__.py b/src/importnb/utils/__init__.py index e69de29..57d68eb 100644 --- a/src/importnb/utils/__init__.py +++ b/src/importnb/utils/__init__.py @@ -0,0 +1,29 @@ +__IPYTHON__ = False +try: + try: + from . import ipython + except: + import ipython + if not ipython.get_ipython(): + raise ValueError("""There is no interactive IPython shell""") + __IPYTHON__ = True +except: + ... + +from pathlib import Path + +try: + from ..compiler_python import ScriptExporter + +except: + from importnb.compiler_python import ScriptExporter + + +def export(src, dst): + Path(dst).write_text(ScriptExporter().from_filename(src)[0]) + + +if __name__ == "__main__": + + export("__init__.ipynb", "../../importnb/utils/__init__.py") + export("__init__.ipynb", "__init__.py") diff --git a/src/importnb/utils/ipython.py b/src/importnb/utils/ipython.py index 4605707..707200b 100644 --- a/src/importnb/utils/ipython.py +++ b/src/importnb/utils/ipython.py @@ -3,70 +3,57 @@ from pathlib import Path import json - def get_config(): ip = get_ipython() - return Path(ip.profile_dir.location if ip else paths.locate_profile()) / "ipython_config.json" - + return Path(ip.profile_dir.location if ip else paths.locate_profile()) / "ipython_config.json" def load_config(): location = get_config() - with open(location) as file: - try: + try: + with open(location) as file: config = json.load(file) - except (FileNotFoundError, json.JSONDecodeError): - config = {} + except (FileNotFoundError, json.JSONDecodeError): + config = {} - if "InteractiveShellApp" not in config: - config["InteractiveShellApp"] = {} + if 'InteractiveShellApp' not in config: + config['InteractiveShellApp'] = {} - if "extensions" not in config["InteractiveShellApp"]: - config["InteractiveShellApp"]["extensions"] = [] + if 'extensions' not in config['InteractiveShellApp']: + config['InteractiveShellApp']['extensions'] = [] return config, location - def install(ip=None): config, location = load_config() - if "importnb" not in config["InteractiveShellApp"]["extensions"]: - config["InteractiveShellApp"]["extensions"].append("importnb.utils.ipython") + if 'importnb' not in config['InteractiveShellApp']['extensions']: + config['InteractiveShellApp']['extensions'].append('importnb.utils.ipython') - with open(location, "w") as file: + with open(location, 'w') as file: json.dump(config, file) - def installed(): config = load_config() - return "importnb.utils.ipython" in config.get("InteractiveShellApp", {}).get("extensions", []) - + return 'importnb.utils.ipython' in config.get('InteractiveShellApp', {}).get('extensions', []) def uninstall(ip=None): config, location = load_config() - config["InteractiveShellApp"]["extensions"] = [ - ext - for ext in config["InteractiveShellApp"]["extensions"] - if ext != "importnb.utils.ipython" + config['InteractiveShellApp']['extensions'] = [ + ext for ext in config['InteractiveShellApp']['extensions'] if ext != 'importnb.utils.ipython' ] - with open(location, "w") as file: - json.dump(config, file) - + with open(location, 'w') as file: json.dump(config, file) def load_ipython_extension(ip): from ..loader import Notebook - Notebook().__enter__(position=-1) - -if __name__ == "__main__": +if __name__ == '__main__': try: from .compiler_python import ScriptExporter except: from importnb.compiler_python import ScriptExporter from pathlib import Path + Path('../../importnb/utils/ipython.py').write_text(ScriptExporter().from_filename('ipython.ipynb')[0]) - Path("../../importnb/utils/ipython.py").write_text( - ScriptExporter().from_filename("ipython.ipynb")[0] - ) diff --git a/src/importnb/utils/watch.py b/src/importnb/utils/watch.py new file mode 100644 index 0000000..c99f3ac --- /dev/null +++ b/src/importnb/utils/watch.py @@ -0,0 +1,34 @@ +import os +from watchdog.tricks import ShellCommandTrick + + +class ModuleTrick(ShellCommandTrick): + """ModuleTrick is a watchdog trick that """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._ignore_patterns = self._ignore_patterns or [] + self._ignore_patterns.extend(("*-checkpoint.ipynb", "*.~*")) + + def on_any_event(self, event): + try: + event.dest_path = event.src_path.lstrip(".").lstrip(os.sep).rstrip(".ipynb").rstrip( + ".py" + ).replace( + os.sep, "." + ) + super().on_any_event(event) + except AttributeError: + ... + + +if __name__ == "__main__": + try: + from .compiler_python import ScriptExporter + except: + from importnb.compiler_python import ScriptExporter + from pathlib import Path + + Path("../../importnb/utils/watch.py").write_text( + ScriptExporter().from_filename("watch.ipynb")[0] + ) diff --git a/src/notebooks/exporter.ipynb b/src/notebooks/exporter.ipynb index 137dc3a..a63c526 100644 --- a/src/notebooks/exporter.ipynb +++ b/src/notebooks/exporter.ipynb @@ -4,28 +4,14 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'decoder'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m----------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLineNoDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named '__main__.decoder'; '__main__' is not a package", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLineNoDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdecoder\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLineNoDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0m__file__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglobals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'__file__'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'exporter.ipynb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'decoder'" - ] - } - ], + "outputs": [], "source": [ " try:\n", " from .decoder import LineNoDecoder\n", + " from .utils import __IPYTHON__\n", " except:\n", " from decoder import LineNoDecoder \n", + " from utils import __IPYTHON__\n", " \n", " __file__ = globals().get('__file__', 'exporter.ipynb')\n", " __nb__ = __file__.replace('src/importnb', 'src/notebooks')" @@ -46,18 +32,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - " __IPYTHON__ = False\n", - " try:\n", - " from IPython import get_ipython\n", - " if not get_ipython():\n", - " raise ValueError(\"\"\"There is no interactive IPython shell\"\"\")\n", - " __IPYTHON__ = True\n", - " except: ...\n", - " \n", " if __IPYTHON__:\n", " try:\n", " from .compiler_ipython import Compiler, NotebookNode, NotebookExporter\n", @@ -73,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -85,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -121,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -140,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -154,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "scrolled": false }, diff --git a/src/notebooks/loader.ipynb b/src/notebooks/loader.ipynb index 8f6c050..95048ad 100644 --- a/src/notebooks/loader.ipynb +++ b/src/notebooks/loader.ipynb @@ -4,7 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# The [Import Loader](https://docs.python.org/3/reference/import.html#loaders)" + "# The [Import Loader](https://docs.python.org/3/reference/import.html#loaders)\n", + "\n", + "`importnb` provides the ability to import Notebooks in Python packages and as modules." ] }, { @@ -15,21 +17,25 @@ "source": [ " try:\n", " from .exporter import Compile, AST\n", + " from .utils import __IPYTHON__, export\n", " except:\n", " from exporter import Compile, AST\n", + " from utils import __IPYTHON__, export\n", " import inspect, sys\n", " from importlib.machinery import SourceFileLoader\n", " from importlib._bootstrap_external import FileFinder\n", " from importlib import reload\n", " from traceback import print_exc\n", " from contextlib import contextmanager\n", - " __IPYTHON__ = False\n", - " try:\n", - " from IPython import get_ipython\n", - " if not get_ipython():\n", - " raise ValueError(\"\"\"There is no interactive IPython shell\"\"\")\n", - " __IPYTHON__ = True\n", - " except: ...\n" + " \n", + " __all__ = 'Notebook', 'Partial', 'reload'," + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `sys.path_hook` modifiers" ] }, { @@ -40,6 +46,11 @@ "source": [ " @contextmanager\n", " def modify_file_finder_details():\n", + " \"\"\"yield the FileFinder in the sys.path_hooks that loads Python files and assure\n", + " the import cache is cleared afterwards. \n", + " \n", + " Everything goes to shit if the import cache is not cleared.\"\"\"\n", + " \n", " for id, hook in enumerate(sys.path_hooks):\n", " try:\n", " closure = inspect.getclosurevars(hook).nonlocals\n", @@ -54,23 +65,23 @@ ] }, { - "cell_type": "code", - "execution_count": 3, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - " def add_path_hooks(loader: SourceFileLoader, extensions, *, position=0, lazy=False):\n", - " \"\"\"Update the FileFinder loader in sys.path_hooks to accomodate a {loader} with the {extensions}\"\"\"\n", - " with modify_file_finder_details() as details:\n", - " details.insert(position, (loader, extensions))" + "Update the file_finder details with functions to append and remove the [loader details](https://docs.python.org/3.7/library/importlib.html#importlib.machinery.FileFinder)." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ + " def add_path_hooks(loader: SourceFileLoader, extensions, *, position=0, lazy=False):\n", + " \"\"\"Update the FileFinder loader in sys.path_hooks to accomodate a {loader} with the {extensions}\"\"\"\n", + " with modify_file_finder_details() as details:\n", + " details.insert(position, (loader, extensions))\n", + "\n", " def remove_one_path_hook(loader):\n", " with modify_file_finder_details() as details:\n", " _details = list(details)\n", @@ -80,13 +91,24 @@ " break" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Context Manager\n", + "\n", + "`importnb` uses a context manager to assure that the traditional import system behaviors as expected. If the loader is permenantly available then it may create some unexpected import behaviors." + ] + }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - " class ImportContextMixin:\n", + " class ImportContextManagerMixin:\n", + " \"\"\"A context maanager to add and remove loader_details from the path_hooks.\n", + " \"\"\"\n", " def __enter__(self, position=0): \n", " add_path_hooks(type(self), self.EXTENSION_SUFFIXES, position=position)\n", " def __exit__(self, exception_type=None, exception_value=None, traceback=None): remove_one_path_hook(type(self))" @@ -94,11 +116,11 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - " class Notebook(SourceFileLoader, ImportContextMixin):\n", + " class Notebook(SourceFileLoader, ImportContextManagerMixin):\n", " \"\"\"A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.\"\"\"\n", " EXTENSION_SUFFIXES = '.ipynb',\n", " \n", @@ -128,9 +150,16 @@ " return Compile().from_file(stream, filename=Notebook.path, name=Notebook.name)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Partial Loader" + ] + }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -156,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -173,21 +202,23 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "scrolled": false }, "outputs": [], "source": [ " if __name__ == '__main__':\n", - " try:\n", - " from .compiler_python import ScriptExporter\n", - " except:\n", - " from compiler_python import ScriptExporter\n", - " from pathlib import Path\n", - " Path('../importnb/loader.py').write_text(ScriptExporter().from_filename('loader.ipynb')[0])\n", + " export('loader.ipynb', '../importnb/loader.py')\n", " __import__('doctest').testmod()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/notebooks/utils/__init__.ipynb b/src/notebooks/utils/__init__.ipynb new file mode 100644 index 0000000..486e1c8 --- /dev/null +++ b/src/notebooks/utils/__init__.ipynb @@ -0,0 +1,80 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "__IPYTHON__ = False\n", + "try:\n", + " try:\n", + " from . import ipython\n", + " except:\n", + " import ipython\n", + " if not ipython.get_ipython():\n", + " raise ValueError(\"\"\"There is no interactive IPython shell\"\"\")\n", + " __IPYTHON__ = True\n", + "except: ...\n", + " \n", + "from pathlib import Path" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " from ..compiler_python import ScriptExporter\n", + " \n", + "except:\n", + " from importnb.compiler_python import ScriptExporter" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def export(src, dst):\n", + " Path(dst).write_text(ScriptExporter().from_filename(src)[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == '__main__':\n", + " \n", + " export('__init__.ipynb', '../../importnb/utils/__init__.py')\n", + " export('__init__.ipynb', '__init__.py')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "p6", + "language": "python", + "name": "other-env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/notebooks/utils/__init__.py b/src/notebooks/utils/__init__.py new file mode 100644 index 0000000..6338111 --- /dev/null +++ b/src/notebooks/utils/__init__.py @@ -0,0 +1,26 @@ +__IPYTHON__ = False +try: + try: + from . import ipython + except: + import ipython + if not ipython.get_ipython(): + raise ValueError("""There is no interactive IPython shell""") + __IPYTHON__ = True +except: ... + +from pathlib import Path + +try: + from ..compiler_python import ScriptExporter + +except: + from importnb.compiler_python import ScriptExporter + +def export(src, dst): + Path(dst).write_text(ScriptExporter().from_filename(src)[0]) + +if __name__ == '__main__': + + export('__init__.ipynb', '../../importnb/utils/__init__.py') + export('__init__.ipynb', '__init__.py') \ No newline at end of file diff --git a/src/notebooks/utils/ipython.ipynb b/src/notebooks/utils/ipython.ipynb index 43f0078..7c98f2f 100644 --- a/src/notebooks/utils/ipython.ipynb +++ b/src/notebooks/utils/ipython.ipynb @@ -38,11 +38,11 @@ "source": [ "def load_config():\n", " location = get_config()\n", - " with open(location) as file:\n", - " try:\n", + " try:\n", + " with open(location) as file: \n", " config = json.load(file)\n", - " except (FileNotFoundError, json.JSONDecodeError):\n", - " config = {}\n", + " except (FileNotFoundError, json.JSONDecodeError):\n", + " config = {}\n", "\n", " if 'InteractiveShellApp' not in config:\n", " config['InteractiveShellApp'] = {}\n", @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ diff --git a/src/notebooks/utils/watch.ipynb b/src/notebooks/utils/watch.ipynb new file mode 100644 index 0000000..d385e24 --- /dev/null +++ b/src/notebooks/utils/watch.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Watchdog for modules\n", + "\n", + "Creates a module path from a source file to watch and execute file changes.\n", + "\n", + " tricks:\n", + " patterns:\n", + " - *.ipynb\n", + " shell_command: ipython -m ${watch_dest_path}\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from watchdog.tricks import ShellCommandTrick" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class ModuleTrick(ShellCommandTrick):\n", + " \"\"\"ModuleTrick is a watchdog trick that \"\"\"\n", + " def __init__(self, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self._ignore_patterns = self._ignore_patterns or []\n", + " self._ignore_patterns.extend((\n", + " '*-checkpoint.ipynb', '*.~*'\n", + " ))\n", + " def on_any_event(self, event):\n", + " try:\n", + " event.dest_path = event.src_path.lstrip('.').lstrip(os.sep).rstrip('.ipynb').rstrip('.py').replace(os.sep, '.')\n", + " super().on_any_event(event)\n", + " except AttributeError: ..." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == '__main__':\n", + " try:\n", + " from .compiler_python import ScriptExporter\n", + " except:\n", + " from importnb.compiler_python import ScriptExporter\n", + " from pathlib import Path\n", + " Path('../../importnb/utils/watch.py').write_text(ScriptExporter().from_filename('watch.ipynb')[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "p6", + "language": "python", + "name": "other-env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/__init__.py b/tests/__init__.py index 95c3837..803c9c8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,5 @@ with __import__('importnb').Notebook(): try: - from .test_ import * + from .test_unittests import * except: - from test_ import * \ No newline at end of file + from test_unittests import * diff --git a/tests/test_importnb.ipynb b/tests/test_importnb.ipynb index 6e3a38d..3035ad7 100644 --- a/tests/test_importnb.ipynb +++ b/tests/test_importnb.ipynb @@ -2,11 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 70, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ - "from importnb import Notebook, Partial, reload, load_ipython_extension, unload_ipython_extension\n", + "from importnb import Notebook, reload\n", "from nbformat import v4\n", "from pathlib import Path\n", "import shutil, os, functools\n", @@ -15,46 +15,60 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 73, "metadata": {}, "outputs": [], "source": [ - "source =\"\"\"\n", - "foo = 42\n", - "assert {}\n", - "bar= 100\n", - "\"\"\"" + "def new_notebook(str='foo') -> str:\n", + " \"\"\"Stringify a new notebook to test with a simple set of instructions that may be formatter.\n", + " \n", + " >>> assert isinstance(new_notebook(), str)\n", + " \"\"\"\n", + " return v4.writes(v4.new_notebook(cells=[\n", + " v4.new_code_cell(\"\"\"foo = 42\\nassert {}\\nbar= 100\"\"\".format(str))]))" ] }, { - "cell_type": "code", - "execution_count": 66, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "def new_notebook(str='foo'):\n", - " return v4.writes(v4.new_notebook(cells=[\n", - " v4.new_code_cell(source.format(str))\n", - " ]))" + "# Test Single File Modules\n", + "\n", + "Single file modules mimic common Untitled notebooks. An author should be able to trivially import notebooks in their working directory." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Single File Fixtures" ] }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ "@fixture(scope='function')\n", "def single_file(request):\n", + " \"\"\"A fixture to write a new notebook to disk and delete after each function call.\"\"\"\n", " file = Path('foobar.ipynb')\n", " file.write_text(new_notebook())\n", " request.addfinalizer(functools.partial(os.remove, file))\n", " return file" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each time a file is imported we should clear up the sys path to reset our imports and assure the validity of our tests." + ] + }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -68,20 +82,10 @@ ] }, { - "cell_type": "code", - "execution_count": 72, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "def validate_reload(module):\n", - " try:\n", - " reload(module)\n", - " assert False, \"\"\"The reload should fail.\"\"\"\n", - " except:\n", - " assert True, \"\"\"Cannot reload a file outside of a context manager\"\"\"\n", - "\n", - " with Notebook():\n", - " assert reload(module)" + "`importnb`'s most generic use is as a context manager. `with Notebook()` will update the `sys.path_hooks` to import notebooks as modules." ] }, { @@ -98,16 +102,35 @@ " validate_reload(foobar)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each time we test a notebook import we should test the ability to reload the module. `importnb` expresses the ability to use the normal Python import system, and a notebook must reload for interactive development." + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 77, "metadata": {}, "outputs": [], "source": [ - "@fixture\n", - "def extension(clean_up_file, request):\n", - " load_ipython_extension()\n", - " request.addfinalizer(unload_ipython_extension)" + "def validate_reload(module):\n", + " try:\n", + " reload(module)\n", + " assert False, \"\"\"The reload should have fail.\"\"\"\n", + " except:\n", + " assert True, \"\"\"Cannot reload a file outside of a context manager\"\"\"\n", + "\n", + " with Notebook():\n", + " assert reload(module)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A notebook will not import without the context manager or [IPython extension](#IPython-extension)." ] }, { @@ -116,14 +139,21 @@ "metadata": {}, "outputs": [], "source": [ - "def test_single_with_extension(extension):\n", - " import foobar\n", - " assert foobar.foo == 42 and foobar.bar == 100" + "@mark.xfail\n", + "def test_single_file_without_context():\n", + " import foobar" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the `__main__` context, relative imports are not allowed. " ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -133,9 +163,66 @@ " from . import foobar" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Commonly, we use the `try` statement to allow the ability to use relative imports in a package while developing interactively.\n", + "\n", + " try:\n", + " from . import a_module\n", + " except:\n", + " import a_module" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## IPython extension\n", + "\n", + "In general, an author would use IPython sugar to load an extension\n", + "\n", + " %load_ext importnb\n", + " \n", + "For testing purposes we use the explicit functions to create the extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [], + "source": [ + "from importnb import load_ipython_extension, unload_ipython_extension" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [], + "source": [ + "@fixture\n", + "def extension(clean_up_file, request):\n", + " load_ipython_extension()\n", + " request.addfinalizer(unload_ipython_extension)" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [], + "source": [ + "def test_single_with_extension(extension):\n", + " import foobar\n", + " assert foobar.foo == 42 and foobar.bar == 100" + ] + }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ @@ -152,18 +239,7 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@mark.xfail\n", - "def test_single_file_without_context():\n", - " import foobar" - ] - }, - { - "cell_type": "code", - "execution_count": 53, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -179,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -189,9 +265,25 @@ " from a_test_package import failure" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Partial Imports." + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [], + "source": [ + "from importnb import Partial" + ] + }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 91, "metadata": {}, "outputs": [], "source": [ @@ -206,15 +298,9 @@ " from io import StringIO\n", " s = StringIO()\n", " print_tb(failure.__exception__.__traceback__, file=s)\n", - " assert \"\"\"a_test_package/failure.ipynb\", line 11, in \\n\"\"\" in s.getvalue(), \"\"\"Traceback is not satisfied\"\"\"" + " print(s.getvalue())\n", + " assert \"\"\"a_test_package/failure.ipynb\", line 10, in \\n\"\"\" in s.getvalue(), \"\"\"Traceback is not satisfied\"\"\"" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tests/test_.ipynb b/tests/test_unittests.ipynb similarity index 100% rename from tests/test_.ipynb rename to tests/test_unittests.ipynb diff --git a/tox.ini b/tox.ini index 0a9d65d..3c940f5 100644 --- a/tox.ini +++ b/tox.ini @@ -3,18 +3,18 @@ envlist = ipython, python [testenv:python] deps= - pytest - pytest-cov + -rrequirements-tox.txt commands= - python setup.py pytest --addopts "--doctest-modules --cov=importnb --verbose" - + pip install git+https://github.com/yaml/pyyaml.git --upgrade + python setup.py pytest --addopts "--cov=importnb --verbose" + [testenv:ipython] deps= - pytest - pytest-cov + -rrequirements-tox.txt jupyter nbconvert commands= - # mkdir -p tests/PythonDataScienceHandbook/notebooks - # wget https://raw.githubusercontent.com/jakevdp/PythonDataScienceHandbook/master/notebooks/02.02-The-Basics-Of-NumPy-Arrays.ipynb -O tests/PythonDataScienceHandbook/notebooks/_02_The_Basics_Of_NumPy_Arrays.ipynb --no-check-certificate - ipython setup.py -- pytest --addopts "--doctest-modules --cov=importnb --verbose" + # mkdir -p tests/PythonDataScienceHandbook/notebooks + # wget https://raw.githubusercontent.com/jakevdp/PythonDataScienceHandbook/master/notebooks/02.02-The-Basics-Of-NumPy-Arrays.ipynb -O tests/PythonDataScienceHandbook/notebooks/_02_The_Basics_Of_NumPy_Arrays.ipynb --no-check-certificate + pip install git+https://github.com/yaml/pyyaml.git --upgrade + ipython setup.py -- pytest --addopts "--cov=importnb --verbose" diff --git a/tricks.yaml b/tricks.yaml index d5495d9..390cf1c 100644 --- a/tricks.yaml +++ b/tricks.yaml @@ -1,6 +1,14 @@ tricks: -- watchdog.tricks.ShellCommandTrick: +- importnb.utils.watch.ModuleTrick: patterns: - - ./tests/test_*.ipynb + - '*test_*.ipynb' shell_command: | - pytest tests/test_importnb.ipynb --verbose --capture=no + pytest ${watch_src_path} --verbose --capture=no + +- importnb.utils.watch.ModuleTrick: + patterns: + - '*.ipynb' + ignore_patterns: + - '*test_*.ipynb' + shell_command: | + echo ${watch_src_path} ${watch_dest_path} && ipython -m ${watch_dest_path}