From edb9c66a365aa6e18e3267ac2b115295287cf1a8 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 08:36:57 -0500 Subject: [PATCH 01/27] Drop Python 2 support --- .gitignore | 1 + .pipignore | 5 ----- .travis.yml | 6 +++--- .vscode/settings.json | 3 --- README.md | 20 ++++++++++++-------- frontmatter/__init__.py | 30 ++++++++++++------------------ requirements.txt | 1 - setup.py | 11 +++++------ test.py | 16 ++++++---------- 9 files changed, 39 insertions(+), 54 deletions(-) delete mode 100644 .pipignore delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index f380c2d..3a71124 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ docs/_build/ target/ .DS_Store +.vscode/ \ No newline at end of file diff --git a/.pipignore b/.pipignore deleted file mode 100644 index 7e6ec6b..0000000 --- a/.pipignore +++ /dev/null @@ -1,5 +0,0 @@ -gnureadline -ipython -pip-tools -readline -python-frontmatter \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 79c9eed..9a207cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,15 @@ language: python dist: xenial python: - - "2.7" - - "3.4" - "3.5" - "3.6" - "3.7" + - "3.8" + - "3.9" # command to install dependencies install: # use pyaml for tests above py26 - pip install -r requirements.txt - "python setup.py install" # command to run tests -script: python setup.py test +script: python test diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index de288e1..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.formatting.provider": "black" -} \ No newline at end of file diff --git a/README.md b/README.md index cce1709..6dd0dd2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Python Frontmatter -================== +# Python Frontmatter [Jekyll](http://jekyllrb.com/)-style YAML front matter offers a useful way to add arbitrary, structured metadata to text documents, regardless of type. @@ -7,14 +6,11 @@ This is a small package to load and parse files (or just text) with YAML front m [![Build Status](https://travis-ci.org/eyeseast/python-frontmatter.svg?branch=master)](https://travis-ci.org/eyeseast/python-frontmatter) -Install: --------- +## Install: pip install python-frontmatter - -Usage: ------- +## Usage: ```python import frontmatter @@ -31,6 +27,7 @@ Or a file (or file-like object): ```python >>> with open('tests/hello-world.markdown') as f: ... post = frontmatter.load(f) + ``` Or load from text: @@ -38,6 +35,7 @@ Or load from text: ```python >>> with open('tests/hello-world.markdown') as f: ... post = frontmatter.loads(f.read()) + ``` Access content: @@ -49,6 +47,7 @@ Well, hello there, world. # this works, too >>> print(post) Well, hello there, world. + ``` Use metadata (metadata gets proxied as post keys): @@ -56,6 +55,7 @@ Use metadata (metadata gets proxied as post keys): ```python >>> print(post['title']) Hello, world! + ``` Metadata is a dictionary, with some handy proxies: @@ -68,6 +68,7 @@ Metadata is a dictionary, with some handy proxies: >>> post['excerpt'] = 'tl;dr' >>> pprint(post.metadata) {'excerpt': 'tl;dr', 'layout': 'post', 'title': 'Hello, world!'} + ``` If you don't need the whole post object, just parse: @@ -77,6 +78,7 @@ If you don't need the whole post object, just parse: ... metadata, content = frontmatter.parse(f.read()) >>> print(metadata['title']) Hello, world! + ``` Write back to plain text, too: @@ -90,6 +92,8 @@ title: Hello, world! --- Well, hello there, world. +``` + Or write to a file (or file-like object): ```python @@ -103,5 +107,5 @@ layout: post title: Hello, world! --- Well, hello there, world. -``` +``` diff --git a/frontmatter/__init__.py b/frontmatter/__init__.py index 9108fdb..55e4a99 100644 --- a/frontmatter/__init__.py +++ b/frontmatter/__init__.py @@ -7,7 +7,6 @@ import codecs import re -import six from .util import u from .default_handlers import YAMLHandler, JSONHandler, TOMLHandler @@ -38,7 +37,7 @@ def detect_format(text, handlers): ``text`` should be unicode text about to be parsed. - ``handlers`` is a dictionary where keys are opening delimiters + ``handlers`` is a dictionary where keys are opening delimiters and values are handler instances. """ for pattern, handler in handlers.items(): @@ -93,9 +92,9 @@ def parse(text, encoding="utf-8", handler=None, **defaults): def check(fd, encoding="utf-8"): """ - Check if a file-like object or filename has a frontmatter, + Check if a file-like object or filename has a frontmatter, return True if exists, False otherwise. - + If it contains a frontmatter but it is empty, return True as well. :: @@ -118,7 +117,7 @@ def checks(text, encoding="utf-8"): """ Check if a text (binary or unicode) has a frontmatter, return True if exists, False otherwise. - + If it contains a frontmatter but it is empty, return True as well. :: @@ -134,7 +133,7 @@ def checks(text, encoding="utf-8"): def load(fd, encoding="utf-8", handler=None, **defaults): """ - Load and parse a file-like object or filename, + Load and parse a file-like object or filename, return a :py:class:`post `. :: @@ -202,12 +201,12 @@ def dump(post, fd, encoding="utf-8", handler=None, **kwargs): def dumps(post, handler=None, **kwargs): """ - Serialize a :py:class:`post ` to a string and return text. + Serialize a :py:class:`post ` to a string and return text. This always returns unicode text, which can then be encoded. Passing ``handler`` will change how metadata is turned into text. A handler - passed as an argument will override ``post.handler``, with - :py:class:`YAMLHandler ` used as + passed as an argument will override ``post.handler``, with + :py:class:`YAMLHandler ` used as a default. :: @@ -239,15 +238,15 @@ def dumps(post, handler=None, **kwargs): class Post(object): """ A post contains content and metadata from Front Matter. This is what gets - returned by :py:func:`load ` and :py:func:`loads `. - Passing this to :py:func:`dump ` or :py:func:`dumps ` + returned by :py:func:`load ` and :py:func:`loads `. + Passing this to :py:func:`dump ` or :py:func:`dumps ` will turn it back into text. - For convenience, metadata values are available as proxied item lookups. + For convenience, metadata values are available as proxied item lookups. """ def __init__(self, content, handler=None, **metadata): - self.content = u(content) + self.content = str(content) self.metadata = metadata self.handler = handler @@ -271,11 +270,6 @@ def __bytes__(self): return self.content.encode("utf-8") def __str__(self): - if six.PY2: - return self.__bytes__() - return self.content - - def __unicode__(self): return self.content def get(self, key, default=None): diff --git a/requirements.txt b/requirements.txt index 3106a37..5c0107e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ PyYAML -six toml \ No newline at end of file diff --git a/setup.py b/setup.py index 7de427f..6b14e4b 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,6 @@ readme = f.read() -requirements = ["PyYAML", "six"] - VERSION = "0.5.0" @@ -27,7 +25,9 @@ url="https://github.com/eyeseast/python-frontmatter", packages=["frontmatter"], include_package_data=True, - install_requires=requirements, + install_requires=["PyYAML"], + extras_require={"test": ["pytest", "toml"]}, + tests_require=["python-frontmatter[test]"], license="MIT", zip_safe=False, keywords="frontmatter", @@ -36,13 +36,12 @@ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Natural Language :: English", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], test_suite="test", ) diff --git a/test.py b/test.py index 47197bf..c0bcc50 100644 --- a/test.py +++ b/test.py @@ -15,8 +15,6 @@ import textwrap import unittest -import six - import frontmatter from frontmatter.default_handlers import YAMLHandler, JSONHandler, TOMLHandler @@ -56,7 +54,7 @@ def test_unicode_post(self): output = frontmatter.dumps(chinese) zh = "中文" - self.assertTrue(isinstance(chinese.content, six.text_type)) + self.assertTrue(isinstance(chinese.content, str)) # check that we're dumping out unicode metadata, too self.assertTrue(zh in output) @@ -88,7 +86,7 @@ def test_no_frontmatter(self): def test_empty_frontmatter(self): "Frontmatter, but no metadata" post = frontmatter.load("tests/empty-frontmatter.txt") - content = six.text_type("I have frontmatter but no metadata.") + content = "I have frontmatter but no metadata." self.assertEqual(post.metadata, {}) self.assertEqual(post.content, content) @@ -96,9 +94,7 @@ def test_empty_frontmatter(self): def test_extra_space(self): "Extra space in frontmatter delimiter" post = frontmatter.load("tests/extra-space.txt") - content = six.text_type( - "This file has an extra space on the opening line of the frontmatter." - ) + content = "This file has an extra space on the opening line of the frontmatter." self.assertEqual(post.content, content) metadata = {"something": "else", "test": "tester"} @@ -121,8 +117,8 @@ def test_to_string(self): # test unicode and bytes text = "Well, hello there, world." - self.assertEqual(six.text_type(post), text) - self.assertEqual(six.binary_type(post), text.encode("utf-8")) + self.assertEqual(str(post), text) + self.assertEqual(bytes(post), text.encode("utf-8")) def test_pretty_dumping(self): "Use pyaml to dump nicer" @@ -420,7 +416,7 @@ def setUp(self): if __name__ == "__main__": - doctest.testfile("README.md") + doctest.testfile("README.md", extraglobs={"frontmatter": frontmatter}) doctest.testmod( frontmatter.default_handlers, extraglobs={"frontmatter": frontmatter} ) From bf546679e047257ceb69b7207002d4c8c30e1bab Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 08:42:26 -0500 Subject: [PATCH 02/27] No six --- frontmatter/default_handlers.py | 6 +++--- frontmatter/util.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/frontmatter/default_handlers.py b/frontmatter/default_handlers.py index 3b50da8..b2890cc 100644 --- a/frontmatter/default_handlers.py +++ b/frontmatter/default_handlers.py @@ -141,7 +141,7 @@ class BaseHandler(object): """ - BaseHandler lays out all the steps to detecting, splitting, parsing and + BaseHandler lays out all the steps to detecting, splitting, parsing and exporting front matter metadata. All default handlers are subclassed from BaseHandler. @@ -169,7 +169,7 @@ def detect(self, text): Decide whether this handler can parse the given ``text``, and return True or False. - Note that this is *not* called when passing a handler instance to + Note that this is *not* called when passing a handler instance to :py:func:`frontmatter.load ` or :py:func:`loads `. """ if self.FM_BOUNDARY.match(text): @@ -207,7 +207,7 @@ class YAMLHandler(BaseHandler): def load(self, fm, **kwargs): """ - Parse YAML front matter. This uses yaml.SafeLoader by default. + Parse YAML front matter. This uses yaml.SafeLoader by default. """ kwargs.setdefault("Loader", SafeLoader) return yaml.load(fm, **kwargs) diff --git a/frontmatter/util.py b/frontmatter/util.py index 13e60b6..602f8fd 100644 --- a/frontmatter/util.py +++ b/frontmatter/util.py @@ -2,13 +2,12 @@ """ Utilities for handling unicode and other repetitive bits """ -import six def u(text, encoding="utf-8"): "Return unicode text, no matter what" - if isinstance(text, six.binary_type): + if isinstance(text, bytes): text = text.decode(encoding) # it's already unicode From 21f72b0fdb38c0d42f64353251ec319bdf43cf80 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 08:52:56 -0500 Subject: [PATCH 03/27] typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9a207cb..bcaec6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,4 +12,4 @@ install: - pip install -r requirements.txt - "python setup.py install" # command to run tests -script: python test +script: python test.py From 99b682b8ae6982551698721f8e29f6034aefab36 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 08:56:31 -0500 Subject: [PATCH 04/27] Add actions --- .github/workflows/test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e439c1b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: Test + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[test]' + - name: Run tests + run: | + python test.py From 7266263a433f0741535cfdcc9843b1177a885c82 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 11:00:58 -0500 Subject: [PATCH 05/27] publish workflow --- .github/workflows/publish.yml | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..86343c5 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,56 @@ +name: Publish Python Package + +on: + release: + types: [created] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[test]' + - name: Run tests + run: python test.py + deploy: + runs-on: ubuntu-latest + needs: [test] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-publish-pip- + - name: Install dependencies + run: | + pip install setuptools wheel twine + - name: Publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload --verbose dist/* From 13cdbc8a13ca1938d02f61f2c185ddb90a39fe29 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 13:30:46 -0500 Subject: [PATCH 06/27] Remove future imports --- frontmatter/__init__.py | 1 - frontmatter/default_handlers.py | 1 - requirements.txt | 3 ++- test.py | 15 +++++---------- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/frontmatter/__init__.py b/frontmatter/__init__.py index 55e4a99..032ac76 100644 --- a/frontmatter/__init__.py +++ b/frontmatter/__init__.py @@ -2,7 +2,6 @@ """ Python Frontmatter: Parse and manage posts with YAML frontmatter """ -from __future__ import unicode_literals import codecs import re diff --git a/frontmatter/default_handlers.py b/frontmatter/default_handlers.py index b2890cc..0fe4773 100644 --- a/frontmatter/default_handlers.py +++ b/frontmatter/default_handlers.py @@ -112,7 +112,6 @@ """ -from __future__ import unicode_literals import json import re diff --git a/requirements.txt b/requirements.txt index 5c0107e..eb10adb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ PyYAML -toml \ No newline at end of file +pyaml +toml diff --git a/test.py b/test.py index c0bcc50..43e1dbd 100644 --- a/test.py +++ b/test.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import unicode_literals -from __future__ import print_function import codecs import doctest @@ -123,7 +121,7 @@ def test_to_string(self): def test_pretty_dumping(self): "Use pyaml to dump nicer" # pyaml only runs on 2.7 and above - if sys.version_info > (2, 6) and pyaml is not None: + if pyaml is not None: with codecs.open("tests/unpretty.md", "r", "utf-8") as f: data = f.read() @@ -138,8 +136,6 @@ def test_pretty_dumping(self): self.assertTrue(yaml in dump) def test_with_crlf_string(self): - import codecs - markdown_bytes = b'---\r\ntitle: "my title"\r\ncontent_type: "post"\r\npublished: no\r\n---\r\n\r\nwrite your content in markdown here' loaded = frontmatter.loads(markdown_bytes, "utf-8") self.assertEqual(loaded["title"], "my title") @@ -277,18 +273,17 @@ def setUp(self): } def read_from_tests(self): - with open(self.data["filename"]) as fil: - return fil.read() + with open(self.data["filename"]) as f: + return f.read() def test_external(self): filename = self.data["filename"] content = self.data["content"] metadata = self.data["metadata"] - content_stripped = content.strip() post = frontmatter.load(filename) - self.assertEqual(post.content, content_stripped) + self.assertEqual(post.content, content.strip()) for k, v in metadata.items(): self.assertEqual(post[k], v) @@ -330,7 +325,7 @@ def test_split_load(self): if any_fail: self.fail(failmsg) - @unittest.skip("metadata can be reordered") + # @unittest.skip("metadata can be reordered") def test_split_export(self): text = self.read_from_tests() fm, content = self.handler.split(text) From 77f43c55fe04ab9979e78e9bf07f93f2ef8597ea Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 13:45:34 -0500 Subject: [PATCH 07/27] Still skip that --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 43e1dbd..77c481a 100644 --- a/test.py +++ b/test.py @@ -325,7 +325,7 @@ def test_split_load(self): if any_fail: self.fail(failmsg) - # @unittest.skip("metadata can be reordered") + @unittest.skip("metadata can be reordered") def test_split_export(self): text = self.read_from_tests() fm, content = self.handler.split(text) From 95a00d61f77ca741c3815cdd5596ae5ced9ccf73 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 21:07:39 -0500 Subject: [PATCH 08/27] Stubbing out a new testing approach --- setup.py | 2 +- tests/{ => empty}/empty-frontmatter.txt | 0 tests/{ => empty}/no-frontmatter.txt | 0 .../hello-json.md} | 0 tests/test_docs.py | 10 +++++ tests/test_files.py | 40 +++++++++++++++++++ .../hello-toml.md} | 0 tests/{ => yaml}/chinese.txt | 0 tests/{ => yaml}/extra-dash.txt | 0 tests/{ => yaml}/extra-space.txt | 0 .../hello-markdown.md} | 0 .../hello-world.txt} | 0 .../network-diagrams.md} | 0 tests/{ => yaml}/unpretty.md | 0 14 files changed, 51 insertions(+), 1 deletion(-) rename tests/{ => empty}/empty-frontmatter.txt (100%) rename tests/{ => empty}/no-frontmatter.txt (100%) rename tests/{hello-json.markdown => json/hello-json.md} (100%) create mode 100644 tests/test_docs.py create mode 100644 tests/test_files.py rename tests/{hello-toml.markdown => toml/hello-toml.md} (100%) rename tests/{ => yaml}/chinese.txt (100%) rename tests/{ => yaml}/extra-dash.txt (100%) rename tests/{ => yaml}/extra-space.txt (100%) rename tests/{hello-markdown.markdown => yaml/hello-markdown.md} (100%) rename tests/{hello-world.markdown => yaml/hello-world.txt} (100%) rename tests/{network-diagrams.markdown => yaml/network-diagrams.md} (100%) rename tests/{ => yaml}/unpretty.md (100%) diff --git a/setup.py b/setup.py index 6b14e4b..675b3dc 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ packages=["frontmatter"], include_package_data=True, install_requires=["PyYAML"], - extras_require={"test": ["pytest", "toml"]}, + extras_require={"test": ["pytest", "toml", "pyaml"]}, tests_require=["python-frontmatter[test]"], license="MIT", zip_safe=False, diff --git a/tests/empty-frontmatter.txt b/tests/empty/empty-frontmatter.txt similarity index 100% rename from tests/empty-frontmatter.txt rename to tests/empty/empty-frontmatter.txt diff --git a/tests/no-frontmatter.txt b/tests/empty/no-frontmatter.txt similarity index 100% rename from tests/no-frontmatter.txt rename to tests/empty/no-frontmatter.txt diff --git a/tests/hello-json.markdown b/tests/json/hello-json.md similarity index 100% rename from tests/hello-json.markdown rename to tests/json/hello-json.md diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..02d9c08 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,10 @@ +# all the doctests here +import doctest +import frontmatter + + +def test_docs(): + doctest.testfile("../README.md", extraglobs={"frontmatter": frontmatter}) + doctest.testmod( + frontmatter.default_handlers, extraglobs={"frontmatter": frontmatter} + ) diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 0000000..2ad6d93 --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,40 @@ +""" +Test individual files with frontmatter against expected results. +Files should be in a subdirectory under `tests`, usually sorted by format (yaml, toml, json). + +For a file called hello-world.markdown, there should be a corresponding file called hello-world.json +matching the expected output. +""" +import os +from itertools import chain +from pathlib import Path + +import frontmatter +import pytest + + +def files(): + tests = Path(__file__).parent + md = tests.glob("**/*.md") + txt = tests.glob("**/*.txt") + return chain(md, txt) + + +def get_result_filename(path): + root, ext = os.path.splitext(path) + return f"{root}.result.json" + + +@pytest.mark.parametrize("filename", list(files())) +def test_can_parse(filename): + "Check we can load every file in our test directories without raising an error" + for filename in files(): + post = frontmatter.load(filename) + assert isinstance(post, frontmatter.Post) + + +@pytest.mark.parametrize("filename", list(files())) +def test_file(filename): + result = Path(get_result_filename(filename)) + if not result.exists(): + pytest.fail(f"{result.name} does not exist") diff --git a/tests/hello-toml.markdown b/tests/toml/hello-toml.md similarity index 100% rename from tests/hello-toml.markdown rename to tests/toml/hello-toml.md diff --git a/tests/chinese.txt b/tests/yaml/chinese.txt similarity index 100% rename from tests/chinese.txt rename to tests/yaml/chinese.txt diff --git a/tests/extra-dash.txt b/tests/yaml/extra-dash.txt similarity index 100% rename from tests/extra-dash.txt rename to tests/yaml/extra-dash.txt diff --git a/tests/extra-space.txt b/tests/yaml/extra-space.txt similarity index 100% rename from tests/extra-space.txt rename to tests/yaml/extra-space.txt diff --git a/tests/hello-markdown.markdown b/tests/yaml/hello-markdown.md similarity index 100% rename from tests/hello-markdown.markdown rename to tests/yaml/hello-markdown.md diff --git a/tests/hello-world.markdown b/tests/yaml/hello-world.txt similarity index 100% rename from tests/hello-world.markdown rename to tests/yaml/hello-world.txt diff --git a/tests/network-diagrams.markdown b/tests/yaml/network-diagrams.md similarity index 100% rename from tests/network-diagrams.markdown rename to tests/yaml/network-diagrams.md diff --git a/tests/unpretty.md b/tests/yaml/unpretty.md similarity index 100% rename from tests/unpretty.md rename to tests/yaml/unpretty.md From ca54ed247e6c64ee882a634334822c7b3cd1612f Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 21:33:34 -0500 Subject: [PATCH 09/27] Add content-based tests --- tests/empty/empty-frontmatter.result.json | 3 +++ tests/empty/no-frontmatter.result.json | 3 +++ tests/json/hello-json.result.json | 6 ++++++ tests/stub_tests.py | 21 +++++++++++++++++++++ tests/test_files.py | 6 ++++++ tests/toml/hello-toml.result.json | 6 ++++++ tests/yaml/chinese.result.json | 5 +++++ tests/yaml/extra-dash.result.json | 5 +++++ tests/yaml/extra-space.result.json | 5 +++++ tests/yaml/hello-markdown.result.json | 6 ++++++ tests/yaml/hello-world.result.json | 5 +++++ tests/yaml/network-diagrams.result.json | 9 +++++++++ tests/yaml/unpretty.result.json | 23 +++++++++++++++++++++++ 13 files changed, 103 insertions(+) create mode 100644 tests/empty/empty-frontmatter.result.json create mode 100644 tests/empty/no-frontmatter.result.json create mode 100644 tests/json/hello-json.result.json create mode 100755 tests/stub_tests.py create mode 100644 tests/toml/hello-toml.result.json create mode 100644 tests/yaml/chinese.result.json create mode 100644 tests/yaml/extra-dash.result.json create mode 100644 tests/yaml/extra-space.result.json create mode 100644 tests/yaml/hello-markdown.result.json create mode 100644 tests/yaml/hello-world.result.json create mode 100644 tests/yaml/network-diagrams.result.json create mode 100644 tests/yaml/unpretty.result.json diff --git a/tests/empty/empty-frontmatter.result.json b/tests/empty/empty-frontmatter.result.json new file mode 100644 index 0000000..f9b8ff1 --- /dev/null +++ b/tests/empty/empty-frontmatter.result.json @@ -0,0 +1,3 @@ +{ + "content": "I have frontmatter but no metadata." +} \ No newline at end of file diff --git a/tests/empty/no-frontmatter.result.json b/tests/empty/no-frontmatter.result.json new file mode 100644 index 0000000..ac41f67 --- /dev/null +++ b/tests/empty/no-frontmatter.result.json @@ -0,0 +1,3 @@ +{ + "content": "I have no frontmatter." +} \ No newline at end of file diff --git a/tests/json/hello-json.result.json b/tests/json/hello-json.result.json new file mode 100644 index 0000000..3abb5fb --- /dev/null +++ b/tests/json/hello-json.result.json @@ -0,0 +1,6 @@ +{ + "test": "tester", + "author": "bob", + "something": "else", + "content": "Title\n=====\n\ntitle2\n------\n\nHello.\n\nJust need three dashes\n---\n\nAnd this might break." +} \ No newline at end of file diff --git a/tests/stub_tests.py b/tests/stub_tests.py new file mode 100755 index 0000000..2b7cc89 --- /dev/null +++ b/tests/stub_tests.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +""" +Generate result files for test content. Won't overwrite any that exit. +""" +import json +from pathlib import Path + +import frontmatter +from test_files import files, get_result_filename + + +def main(): + for path in files(): + result = Path(get_result_filename(path)) + if not result.exists(): + post = frontmatter.loads(path.read_text()) + result.write_text(json.dumps(post.to_dict(), indent=2), "utf-8") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test_files.py b/tests/test_files.py index 2ad6d93..430cef8 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -6,6 +6,7 @@ matching the expected output. """ import os +import json from itertools import chain from pathlib import Path @@ -38,3 +39,8 @@ def test_file(filename): result = Path(get_result_filename(filename)) if not result.exists(): pytest.fail(f"{result.name} does not exist") + + post = frontmatter.load(filename) + result = json.loads(result.read_text()) + + assert post.to_dict() == result \ No newline at end of file diff --git a/tests/toml/hello-toml.result.json b/tests/toml/hello-toml.result.json new file mode 100644 index 0000000..00f3a24 --- /dev/null +++ b/tests/toml/hello-toml.result.json @@ -0,0 +1,6 @@ +{ + "author": "bob", + "something": "else", + "test": "tester", + "content": "Title\n=====\n\ntitle2\n------\n\nHello.\n\nJust need three dashes\n---\n\nAnd this shouldn't break." +} \ No newline at end of file diff --git a/tests/yaml/chinese.result.json b/tests/yaml/chinese.result.json new file mode 100644 index 0000000..7920675 --- /dev/null +++ b/tests/yaml/chinese.result.json @@ -0,0 +1,5 @@ +{ + "title": "Let's try unicode", + "language": "\u4e2d\u6587", + "content": "\u6b22\u8fce\u6765\u5230\u5927\u8fde\u6c34\u4ea7\u5b66\u9662\uff01" +} \ No newline at end of file diff --git a/tests/yaml/extra-dash.result.json b/tests/yaml/extra-dash.result.json new file mode 100644 index 0000000..e874ce3 --- /dev/null +++ b/tests/yaml/extra-dash.result.json @@ -0,0 +1,5 @@ +{ + "test": "bob", + "else": "kate", + "content": "Here's some content." +} \ No newline at end of file diff --git a/tests/yaml/extra-space.result.json b/tests/yaml/extra-space.result.json new file mode 100644 index 0000000..842fc7a --- /dev/null +++ b/tests/yaml/extra-space.result.json @@ -0,0 +1,5 @@ +{ + "test": "tester", + "something": "else", + "content": "This file has an extra space on the opening line of the frontmatter." +} \ No newline at end of file diff --git a/tests/yaml/hello-markdown.result.json b/tests/yaml/hello-markdown.result.json new file mode 100644 index 0000000..90d62a0 --- /dev/null +++ b/tests/yaml/hello-markdown.result.json @@ -0,0 +1,6 @@ +{ + "test": "tester", + "author": "bob", + "something": "else", + "content": "Title\n=====\n\ntitle2\n------\n\nHello.\n\nJust need three dashes\n---\n\nAnd this shouldn't break." +} \ No newline at end of file diff --git a/tests/yaml/hello-world.result.json b/tests/yaml/hello-world.result.json new file mode 100644 index 0000000..2f4bb56 --- /dev/null +++ b/tests/yaml/hello-world.result.json @@ -0,0 +1,5 @@ +{ + "title": "Hello, world!", + "layout": "post", + "content": "Well, hello there, world." +} \ No newline at end of file diff --git a/tests/yaml/network-diagrams.result.json b/tests/yaml/network-diagrams.result.json new file mode 100644 index 0000000..2ae4877 --- /dev/null +++ b/tests/yaml/network-diagrams.result.json @@ -0,0 +1,9 @@ +{ + "title": "TODO: Understand Network Diagrams", + "layout": "post", + "published": true, + "tags": [ + "todo" + ], + "content": "Kim Rees, sitting in for Nathan Yau at [Flowing Data](http://flowingdata.com), has been posting examples of [network diagrams](http://flowingdata.com/category/visualization/network-visualization/) lately. I have to confess I'm stumped by most of them them. Or maybe I'm just overwhelmed. Maybe I'm [not alone](http://flowingdata.com/2012/05/28/network-diagrams-simplified/):\n\n> Network diagrams are notoriously messy. Even a small number of nodes can be overwhelmed by their chaotic placement and relationships. [Cody Dunne](http://www.cs.umd.edu/~cdunne/) of [HCIL](http://www.cs.umd.edu/hcil/) showed off [his new work in simplifying these complex structures](http://www.cs.umd.edu/localphp/hcil/tech-reports-search.php?number=2012-11). In essence, he aggregates leaf nodes into a fan glyph that describes the underlying data in its size, arc, and color. Span nodes are similarly captured into crescent glyphs. The result is an easy to read, high level look at the network. You can easily compare different sections of the network, understand areas that may have been occluded by the lines in a traditional diagram, and see relationships far more quickly.\n\nThis seems like the kind of thing that could be useful for news, where we're often trying to understand and illustrate complex relationships. I'll have to find a good dataset to play with." +} \ No newline at end of file diff --git a/tests/yaml/unpretty.result.json b/tests/yaml/unpretty.result.json new file mode 100644 index 0000000..c826041 --- /dev/null +++ b/tests/yaml/unpretty.result.json @@ -0,0 +1,23 @@ +{ + "destination": { + "encoding": { + "xz": { + "enabled": true, + "min_size": 5120, + "options": null, + "path_filter": null + } + }, + "result": { + "append_to_file": null, + "append_to_lafs_dir": null, + "print_to_stdout": true + }, + "url": "http://localhost:3456/uri" + }, + "filter": [ + "\u0414\u043b\u0438\u043d\u043d\u044b\u0439 \u0441\u0442\u0440\u0438\u043d\u0433 \u043d\u0430 \u0440\u0443\u0441\u0441\u043a\u043e\u043c", + "\u0415\u0449\u0435 \u043e\u0434\u043d\u0430 \u0434\u043b\u0438\u043d\u043d\u0430\u044f \u0441\u0442\u0440\u043e\u043a\u0430" + ], + "content": "This is a test of both unicode and prettier dumping. The above metadata comes from the [pretty-yaml](https://github.com/mk-fg/pretty-yaml) readme file.\n\nMetadata should be dumped in order." +} \ No newline at end of file From 21cdbcad374727b774ab9969281c4f3ef3ceaee4 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 21:40:37 -0500 Subject: [PATCH 10/27] actually use pytest --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 3 +-- .travis.yml | 6 ++---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 86343c5..29a03fa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,7 +27,7 @@ jobs: run: | pip install -e '.[test]' - name: Run tests - run: python test.py + run: pytest deploy: runs-on: ubuntu-latest needs: [test] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e439c1b..f9b7030 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,5 +25,4 @@ jobs: run: | pip install -e '.[test]' - name: Run tests - run: | - python test.py + run: pytest diff --git a/.travis.yml b/.travis.yml index bcaec6c..044d643 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ python: - "3.9" # command to install dependencies install: - # use pyaml for tests above py26 - - pip install -r requirements.txt - - "python setup.py install" + - "pip install -e .[test]" # command to run tests -script: python test.py +script: pytest From d256d995d3574e5955ab6cd9f4c897d9a7764638 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 6 Mar 2021 21:44:27 -0500 Subject: [PATCH 11/27] f strings started in 3.6 --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 2 +- .travis.yml | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 29a03fa..320a9ab 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f9b7030..9f48607 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.travis.yml b/.travis.yml index 044d643..d1f8a23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python dist: xenial python: - - "3.5" - "3.6" - "3.7" - "3.8" From 47b44693b2f575bab5b3c88ef245a714d898c336 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 7 Mar 2021 14:40:57 -0500 Subject: [PATCH 12/27] Got all the doctests sorted --- README.md | 8 ++-- docs/conf.py | 4 +- docs/index.rst | 8 ++-- frontmatter/__init__.py | 68 +++++++++++++++++++++++++-------- frontmatter/default_handlers.py | 7 +++- setup.py | 2 +- test.py | 61 ++++++++++++++--------------- tests/test_docs.py | 9 ++++- 8 files changed, 106 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 6dd0dd2..30a797c 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,13 @@ import frontmatter Load a post from a filename: ```python -post = frontmatter.load('tests/hello-world.markdown') +post = frontmatter.load('tests/yaml/hello-world.txt') ``` Or a file (or file-like object): ```python ->>> with open('tests/hello-world.markdown') as f: +>>> with open('tests/yaml/hello-world.txt') as f: ... post = frontmatter.load(f) ``` @@ -33,7 +33,7 @@ Or a file (or file-like object): Or load from text: ```python ->>> with open('tests/hello-world.markdown') as f: +>>> with open('tests/yaml/hello-world.txt') as f: ... post = frontmatter.loads(f.read()) ``` @@ -74,7 +74,7 @@ Metadata is a dictionary, with some handy proxies: If you don't need the whole post object, just parse: ```python ->>> with open('tests/hello-world.markdown') as f: +>>> with open('tests/yaml/hello-world.txt') as f: ... metadata, content = frontmatter.parse(f.read()) >>> print(metadata['title']) Hello, world! diff --git a/docs/conf.py b/docs/conf.py index c3697ca..ef09deb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = u"0.4.2" +version = u"0.5.0" # The full version, including alpha/beta/rc tags. -release = u"0.4.2" +release = u"0.5.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 80c45f8..2cc131b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,20 +34,20 @@ Load a post from a filename: :: - >>> post = frontmatter.load('tests/hello-world.markdown') + >>> post = frontmatter.load('tests/yaml/hello-world.txt') Or a file (or file-like object): :: - >>> with open('tests/hello-world.markdown') as f: + >>> with open('tests/yaml/hello-world.txt') as f: ... post = frontmatter.load(f) Or load from text: :: - >>> with open('tests/hello-world.markdown') as f: + >>> with open('tests/yaml/hello-world.txt') as f: ... post = frontmatter.loads(f.read()) Access content: @@ -84,7 +84,7 @@ If you don't need the whole post object, just parse: :: - >>> with open('tests/hello-world.markdown') as f: + >>> with open('tests/yaml/hello-world.txt') as f: ... metadata, content = frontmatter.parse(f.read()) >>> print(metadata['title']) Hello, world! diff --git a/frontmatter/__init__.py b/frontmatter/__init__.py index 032ac76..b4b290c 100644 --- a/frontmatter/__init__.py +++ b/frontmatter/__init__.py @@ -55,9 +55,13 @@ def parse(text, encoding="utf-8", handler=None, **defaults): If frontmatter is not found, returns an empty metadata dictionary (or defaults) and original text content. - :: + .. testsetup:: * + + import frontmatter - >>> with open('tests/hello-world.markdown') as f: + .. doctest:: + + >>> with open('../tests/yaml/hello-world.txt') as f: ... metadata, content = frontmatter.parse(f.read()) >>> print(metadata['title']) Hello, world! @@ -96,9 +100,9 @@ def check(fd, encoding="utf-8"): If it contains a frontmatter but it is empty, return True as well. - :: + .. doctest:: - >>> frontmatter.check('tests/hello-world.markdown') + >>> frontmatter.check('../tests/yaml/hello-world.txt') True """ @@ -119,9 +123,9 @@ def checks(text, encoding="utf-8"): If it contains a frontmatter but it is empty, return True as well. - :: + .. doctest:: - >>> with open('tests/hello-world.markdown') as f: + >>> with open('../tests/yaml/hello-world.txt') as f: ... frontmatter.checks(f.read()) True @@ -135,10 +139,10 @@ def load(fd, encoding="utf-8", handler=None, **defaults): Load and parse a file-like object or filename, return a :py:class:`post `. - :: + .. doctest:: - >>> post = frontmatter.load('tests/hello-world.markdown') - >>> with open('tests/hello-world.markdown') as f: + >>> post = frontmatter.load('../tests/yaml/hello-world.txt') + >>> with open('../tests/yaml/hello-world.txt') as f: ... post = frontmatter.load(f) """ @@ -157,9 +161,9 @@ def loads(text, encoding="utf-8", handler=None, **defaults): """ Parse text (binary or unicode) and return a :py:class:`post `. - :: + .. doctest:: - >>> with open('tests/hello-world.markdown') as f: + >>> with open('../tests/yaml/hello-world.txt') as f: ... post = frontmatter.loads(f.read()) """ @@ -177,17 +181,35 @@ def dump(post, fd, encoding="utf-8", handler=None, **kwargs): :: >>> from io import BytesIO + >>> post = frontmatter.load('../tests/yaml/hello-world.txt') >>> f = BytesIO() >>> frontmatter.dump(post, f) - >>> print(f.getvalue()) + >>> print(f.getvalue().decode('utf-8')) --- - excerpt: tl;dr layout: post title: Hello, world! --- + Well, hello there, world. + .. testcode:: + + from io import BytesIO + post = frontmatter.load('../tests/yaml/hello-world.txt') + f = BytesIO() + frontmatter.dump(post, f) + print(f.getvalue().decode('utf-8')) + + .. testoutput:: + + --- + layout: post + title: Hello, world! + --- + + Well, hello there, world. + """ content = dumps(post, handler, **kwargs) if hasattr(fd, "write"): @@ -207,14 +229,30 @@ def dumps(post, handler=None, **kwargs): passed as an argument will override ``post.handler``, with :py:class:`YAMLHandler ` used as a default. + :: - >>> print(frontmatter.dumps(post)) + >>> post = frontmatter.load('../tests/yaml/hello-world.txt') + >>> print(frontmatter.dumps(post)) # doctest: +NORMALIZE_WHITESPACE --- - excerpt: tl;dr layout: post title: Hello, world! --- + + Well, hello there, world. + + .. testcode:: + + post = frontmatter.load('../tests/yaml/hello-world.txt') + print(frontmatter.dumps(post)) + + .. testoutput:: + + --- + layout: post + title: Hello, world! + --- + Well, hello there, world. """ diff --git a/frontmatter/default_handlers.py b/frontmatter/default_handlers.py index 0fe4773..46b40be 100644 --- a/frontmatter/default_handlers.py +++ b/frontmatter/default_handlers.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- """ +.. testsetup:: handlers + + import frontmatter + By default, ``frontmatter`` reads and writes YAML metadata. But maybe you don't like YAML. Maybe enjoy writing metadata in JSON, or TOML, or some other exotic markup not yet invented. For this, there are handlers. @@ -31,11 +35,12 @@ object. By default, calling :py:func:`frontmatter.dumps ` on the post will use the attached handler. + :: >>> import frontmatter >>> from frontmatter.default_handlers import YAMLHandler, TOMLHandler - >>> post = frontmatter.load('tests/hello-toml.markdown', handler=TOMLHandler()) + >>> post = frontmatter.load('tests/toml/hello-toml.markdown', handler=TOMLHandler()) >>> post.handler #doctest: +ELLIPSIS diff --git a/setup.py b/setup.py index 675b3dc..ac02e5b 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ packages=["frontmatter"], include_package_data=True, install_requires=["PyYAML"], - extras_require={"test": ["pytest", "toml", "pyaml"]}, + extras_require={"test": ["pytest", "toml", "pyaml"], "docs": ["sphinx"]}, tests_require=["python-frontmatter[test]"], license="MIT", zip_safe=False, diff --git a/test.py b/test.py index 77c481a..b3e0541 100644 --- a/test.py +++ b/test.py @@ -33,14 +33,9 @@ class FrontmatterTest(unittest.TestCase): maxDiff = None - def test_all_the_tests(self): - "Sanity check that everything in the tests folder loads without errors" - for filename in glob.glob("tests/*"): - frontmatter.load(filename) - def test_with_markdown_content(self): "Parse frontmatter and only the frontmatter" - post = frontmatter.load("tests/hello-markdown.markdown") + post = frontmatter.load("tests/yaml/hello-markdown.md") metadata = {"author": "bob", "something": "else", "test": "tester"} for k, v in metadata.items(): @@ -48,7 +43,7 @@ def test_with_markdown_content(self): def test_unicode_post(self): "Ensure unicode is parsed correctly" - chinese = frontmatter.load("tests/chinese.txt", "utf-8") + chinese = frontmatter.load("tests/yaml/chinese.txt", "utf-8") output = frontmatter.dumps(chinese) zh = "中文" @@ -62,20 +57,20 @@ def test_unicode_post(self): def test_check_no_frontmatter(self): "Checks if a file does not have a frontmatter" - ret = frontmatter.check("tests/no-frontmatter.txt") + ret = frontmatter.check("tests/empty/no-frontmatter.txt") self.assertEqual(ret, False) def test_check_empty_frontmatter(self): "Checks if a file has a frontmatter (empty or not)" - ret = frontmatter.check("tests/empty-frontmatter.txt") + ret = frontmatter.check("tests/empty/empty-frontmatter.txt") self.assertEqual(ret, True) def test_no_frontmatter(self): "This is not a zen exercise." - post = frontmatter.load("tests/no-frontmatter.txt") - with codecs.open("tests/no-frontmatter.txt", "r", "utf-8") as f: + post = frontmatter.load("tests/empty/no-frontmatter.txt") + with codecs.open("tests/empty/no-frontmatter.txt", "r", "utf-8") as f: content = f.read().strip() self.assertEqual(post.metadata, {}) @@ -83,7 +78,7 @@ def test_no_frontmatter(self): def test_empty_frontmatter(self): "Frontmatter, but no metadata" - post = frontmatter.load("tests/empty-frontmatter.txt") + post = frontmatter.load("tests/empty/empty-frontmatter.txt") content = "I have frontmatter but no metadata." self.assertEqual(post.metadata, {}) @@ -91,7 +86,7 @@ def test_empty_frontmatter(self): def test_extra_space(self): "Extra space in frontmatter delimiter" - post = frontmatter.load("tests/extra-space.txt") + post = frontmatter.load("tests/yaml/extra-space.txt") content = "This file has an extra space on the opening line of the frontmatter." self.assertEqual(post.content, content) @@ -101,7 +96,7 @@ def test_extra_space(self): def test_to_dict(self): "Dump a post as a dict, for serializing" - post = frontmatter.load("tests/network-diagrams.markdown") + post = frontmatter.load("tests/yaml/network-diagrams.md") post_dict = post.to_dict() for k, v in post.metadata.items(): @@ -111,7 +106,7 @@ def test_to_dict(self): def test_to_string(self): "Calling str(post) returns post.content" - post = frontmatter.load("tests/hello-world.markdown") + post = frontmatter.load("tests/yaml/hello-world.txt") # test unicode and bytes text = "Well, hello there, world." @@ -123,10 +118,10 @@ def test_pretty_dumping(self): # pyaml only runs on 2.7 and above if pyaml is not None: - with codecs.open("tests/unpretty.md", "r", "utf-8") as f: + with codecs.open("tests/yaml/unpretty.md", "r", "utf-8") as f: data = f.read() - post = frontmatter.load("tests/unpretty.md") + post = frontmatter.load("tests/yaml/unpretty.md") yaml = pyaml.dump(post.metadata) # the unsafe dumper gives you nicer output, for times you want that @@ -142,14 +137,14 @@ def test_with_crlf_string(self): def test_dumping_with_custom_delimiters(self): "dump with custom delimiters" - post = frontmatter.load("tests/hello-world.markdown") + post = frontmatter.load("tests/yaml/hello-world.txt") dump = frontmatter.dumps(post, start_delimiter="+++", end_delimiter="+++") self.assertTrue("+++" in dump) def test_dump_to_file(self): "dump post to filename" - post = frontmatter.load("tests/hello-world.markdown") + post = frontmatter.load("tests/yaml/hello-world.txt") tempdir = tempfile.mkdtemp() filename = os.path.join(tempdir, "hello.md") @@ -168,9 +163,9 @@ class HandlerTest(unittest.TestCase): """ TEST_FILES = { - "tests/hello-world.markdown": YAMLHandler, - "tests/hello-json.markdown": JSONHandler, - "tests/hello-toml.markdown": TOMLHandler, + "tests/yaml/hello-world.txt": YAMLHandler, + "tests/json/hello-json.md": JSONHandler, + "tests/toml/hello-toml.md": TOMLHandler, } def sanity_check(self, filename, handler_type): @@ -201,7 +196,7 @@ def test_sanity_all(self): def test_no_handler(self): "default to YAMLHandler when no handler is attached" - post = frontmatter.load("tests/hello-world.markdown") + post = frontmatter.load("tests/yaml/hello-world.txt") del post.handler text = frontmatter.dumps(post) @@ -242,14 +237,14 @@ def test_toml(self): "load toml frontmatter" if toml is None: return - post = frontmatter.load("tests/hello-toml.markdown") + post = frontmatter.load("tests/toml/hello-toml.md") metadata = {"author": "bob", "something": "else", "test": "tester"} for k, v in metadata.items(): self.assertEqual(post[k], v) def test_json(self): "load raw JSON frontmatter" - post = frontmatter.load("tests/hello-json.markdown") + post = frontmatter.load("tests/json/hello-json.md") metadata = {"author": "bob", "something": "else", "test": "tester"} for k, v in metadata.items(): self.assertEqual(post[k], v) @@ -266,7 +261,7 @@ def setUp(self): """ self.handler = None self.data = { - "filename": "tests/hello-world.markdown", + "filename": "tests/yaml/hello-world.txt", "content": """\ """, "metadata": {}, @@ -339,7 +334,7 @@ class YAMLHandlerTest(HandlerBaseTest, unittest.TestCase): def setUp(self): self.handler = YAMLHandler() self.data = { - "filename": "tests/hello-markdown.markdown", + "filename": "tests/yaml/hello-markdown.md", # TODO: YAMLHandler.split() is prepending '\n' to the content "content": """\ @@ -363,7 +358,7 @@ class JSONHandlerTest(HandlerBaseTest, unittest.TestCase): def setUp(self): self.handler = JSONHandler() self.data = { - "filename": "tests/hello-json.markdown", + "filename": "tests/json/hello-json.md", # TODO: JSONHandler.split() is prepending '\n' to the content "content": """\ @@ -389,7 +384,7 @@ class TOMLHandlerTest(HandlerBaseTest, unittest.TestCase): def setUp(self): self.handler = TOMLHandler() self.data = { - "filename": "tests/hello-toml.markdown", + "filename": "tests/toml/hello-toml.md", # TODO: TOMLHandler.split() is prepending '\n' to the content "content": """\ @@ -411,8 +406,8 @@ def setUp(self): if __name__ == "__main__": - doctest.testfile("README.md", extraglobs={"frontmatter": frontmatter}) - doctest.testmod( - frontmatter.default_handlers, extraglobs={"frontmatter": frontmatter} - ) + # doctest.testfile("README.md", extraglobs={"frontmatter": frontmatter}) + # doctest.testmod( + # frontmatter.default_handlers, extraglobs={"frontmatter": frontmatter} + # ) unittest.main() diff --git a/tests/test_docs.py b/tests/test_docs.py index 02d9c08..9d08afa 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -3,8 +3,15 @@ import frontmatter -def test_docs(): +def test_readme(): doctest.testfile("../README.md", extraglobs={"frontmatter": frontmatter}) + + +def test_api_docs(): + doctest.testmod(frontmatter, extraglobs={"frontmatter": frontmatter}) + + +def test_handler_docs(): doctest.testmod( frontmatter.default_handlers, extraglobs={"frontmatter": frontmatter} ) From 1d85a20c4bf88beec68cd4916a77e8121e9922d5 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 7 Mar 2021 14:59:10 -0500 Subject: [PATCH 13/27] Move old unit tests into tests/ directory, at least for now --- LICENSE | 2 +- docs/conf.py | 2 +- test.py => tests/unit_test.py | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) rename test.py => tests/unit_test.py (98%) diff --git a/LICENSE b/LICENSE index 3012155..92e6b96 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2019 Chris Amico, Glass Eye Media LLC +Copyright (c) 2021 Chris Amico Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/conf.py b/docs/conf.py index ef09deb..a329d67 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ # General information about the project. project = u"Python Frontmatter" -copyright = u"2017, Chris Amico" +copyright = u"2021, Chris Amico" author = u"Chris Amico" # The version info for the project you're documenting, acts as replacement for diff --git a/test.py b/tests/unit_test.py similarity index 98% rename from test.py rename to tests/unit_test.py index b3e0541..47f41f2 100644 --- a/test.py +++ b/tests/unit_test.py @@ -406,8 +406,4 @@ def setUp(self): if __name__ == "__main__": - # doctest.testfile("README.md", extraglobs={"frontmatter": frontmatter}) - # doctest.testmod( - # frontmatter.default_handlers, extraglobs={"frontmatter": frontmatter} - # ) unittest.main() From c524df39c89d485da58eed29fc9b30c7a448e0d4 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 7 Mar 2021 15:08:58 -0500 Subject: [PATCH 14/27] some edits --- README.md | 2 ++ docs/index.rst | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 30a797c..af3bdfd 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,5 @@ title: Hello, world! Well, hello there, world. ``` + +For more examples, see files in the `tests/` directory. Each sample file has a corresponding `.result.json` file showing the expected parsed output. diff --git a/docs/index.rst b/docs/index.rst index 2cc131b..611cd65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ Python Frontmatter useful way to add arbitrary, structured metadata to text documents, regardless of type. -This is a small package to load and parse files (or just text) with YAML +This is a package to load and parse files (or text strings) with YAML front matter. @@ -80,7 +80,7 @@ Metadata is a dictionary, with some handy proxies: >>> pprint(post.metadata) {'excerpt': 'tl;dr', 'layout': 'post', 'title': 'Hello, world!'} -If you don't need the whole post object, just parse: +If you don't need the whole post object, use `frontmatter.parse` to return metadata and content separately: :: From 1fd2ff618103d5d68c20b1d317b237e94ce80af0 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 7 Mar 2021 16:59:40 -0500 Subject: [PATCH 15/27] Make this branch 1.0 --- docs/conf.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a329d67..33d6797 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = u"0.5.0" +version = u"1.0.0" # The full version, including alpha/beta/rc tags. -release = u"0.5.0" +release = u"1.0.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index ac02e5b..8deecc4 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ readme = f.read() -VERSION = "0.5.0" +VERSION = "1.0.0" setup( From 43f95d118d70ab75e64f509ef0242bd6c266ac49 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 13 Mar 2021 20:52:03 -0500 Subject: [PATCH 16/27] Basic formatting api sketched out --- examples/__init__.py | 0 examples/content/reversed.txt | 28 +++++++++++++++++ examples/reversed.py | 55 +++++++++++++++++++++++++++++++++ frontmatter/default_handlers.py | 3 -- 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 examples/__init__.py create mode 100644 examples/content/reversed.txt create mode 100644 examples/reversed.py diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/content/reversed.txt b/examples/content/reversed.txt new file mode 100644 index 0000000..bbb952d --- /dev/null +++ b/examples/content/reversed.txt @@ -0,0 +1,28 @@ +This is txt format to prevent reformatting +============================================ + +Dextra tempore deus +------------------- + +Lorem markdownum est dicere pariter es dat si non, praesignis Styge, non +Maenalon magnae miserrimus. Corpora frustra committere insuetum et fecit +**Hippothousque arbore solio** inopem utraque concepit illa comantem me mortis +epulis protinus putares! Piceis *manibus*. Erinys et parum morsusque repugnat +ore corna sacris, pollice movet currus gestamina. + +Genitoris forti circumfuso videbit fertur vulnere simillima +----------------------------------------------------------- + +Audit enim, est illa nervis loco inque hoc, et rigido! Monstris vatibus laetos +contemptor Calydonia. Et visa capillo referens regia: usus: odiique nostro. +**Vim** sensit inpulit virginis metuens secum cogit, corpus. + +Humus ater Dromas est honorem, Titanida glandibus sinit, e terras capillos +cremet retinentibus male. Tertia et cedit eliso flectere haec, cute nihil +marmore armo. Mihi [Olympi](http://que.org/saepepoenas), iam sustinet addidit +humana similis. + +--- +title: Front matter, reversed +ref: https://github.com/eyeseast/python-frontmatter/issues/67 +--- diff --git a/examples/reversed.py b/examples/reversed.py new file mode 100644 index 0000000..6de012e --- /dev/null +++ b/examples/reversed.py @@ -0,0 +1,55 @@ +import frontmatter + +POST_TEMPLATE = """\ +{content} + +{start_delimiter} +{metadata} +{end_delimiter} +""" + + +class ReverseYAMLHandler(frontmatter.YAMLHandler): + """ + This is an example of using Handler.parse and Handler.format to move Frontmatter to the bottom + of a file, both for parsing and output. + + >>> with open("./content/reversed.txt") as f: + ... text = f.read() + >>> handler = ReverseYAMLHandler() + >>> post = frontmatter.loads(text, handler=handler) + >>> print(post['title']) + Front matter, reversed + >>> print(post['ref']) + https://github.com/eyeseast/python-frontmatter/issues/67 + >>> frontmatter.dumps(post, handler=handler) == text.strip() + True + """ + + # FM_BOUNDARY as a string, so we can rsplit + FM_BOUNDARY = "---" + + def split(self, text): + """ + Split text into frontmatter and content + """ + content, fm, _ = text.rsplit(self.FM_BOUNDARY, 2) + return fm, content + + def format(post, **kwargs): + start_delimiter = kwargs.pop("start_delimiter", handler.START_DELIMITER) + end_delimiter = kwargs.pop("end_delimiter", handler.END_DELIMITER) + metadata = self.export(post.metadata, **kwargs) + + return POST_TEMPLATE.format( + content=post.content, + metadata=metadata, + start_delimiter=start_delimiter, + end_delimiter=end_delimiter, + ).strip() + + +if __name__ == "__main__": + import doctest + + doctest.testmod() \ No newline at end of file diff --git a/frontmatter/default_handlers.py b/frontmatter/default_handlers.py index 46b40be..a69c045 100644 --- a/frontmatter/default_handlers.py +++ b/frontmatter/default_handlers.py @@ -97,15 +97,12 @@ # set YAML format when dumping, but the old handler attached >>> t1 = frontmatter.dumps(post, handler=YAMLHandler()) - # set a new handler, changing all future exports >>> post.handler = YAMLHandler() >>> t2 = frontmatter.dumps(post) - # remove handler, defaulting back to YAML >>> post.handler = None >>> t3 = frontmatter.dumps(post) - >>> t1 == t2 == t3 True From 2a399a689a6a7eabbb94ee6fad6873981f40d5e6 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 13 Mar 2021 21:36:58 -0500 Subject: [PATCH 17/27] Fix tests, with one expected fail --- frontmatter/conftest.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 frontmatter/conftest.py diff --git a/frontmatter/conftest.py b/frontmatter/conftest.py new file mode 100644 index 0000000..e804a6e --- /dev/null +++ b/frontmatter/conftest.py @@ -0,0 +1,8 @@ +import pytest + + +@pytest.fixture(autouse=True) +def add_globals(doctest_namespace): + import frontmatter + + doctest_namespace["frontmatter"] = frontmatter From 110ba96f86e10aaff36537419839748b7dd2db3e Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sat, 13 Mar 2021 21:37:10 -0500 Subject: [PATCH 18/27] Fix tests, with one expected fail --- .github/workflows/test.yml | 3 ++- README.md | 6 ++++-- examples/reversed.py | 8 +------- frontmatter/__init__.py | 28 ++++++++++++++-------------- frontmatter/default_handlers.py | 8 +++----- 5 files changed, 24 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f48607..bdb1c1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,4 +25,5 @@ jobs: run: | pip install -e '.[test]' - name: Run tests - run: pytest + run: | + pytest . --doctest-modules diff --git a/README.md b/README.md index af3bdfd..618870b 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,15 @@ This is a small package to load and parse files (or just text) with YAML front m ## Usage: ```python -import frontmatter +>>> import frontmatter + ``` Load a post from a filename: ```python -post = frontmatter.load('tests/yaml/hello-world.txt') +>>> post = frontmatter.load('tests/yaml/hello-world.txt') + ``` Or a file (or file-like object): diff --git a/examples/reversed.py b/examples/reversed.py index 6de012e..2ddfdbc 100644 --- a/examples/reversed.py +++ b/examples/reversed.py @@ -14,7 +14,7 @@ class ReverseYAMLHandler(frontmatter.YAMLHandler): This is an example of using Handler.parse and Handler.format to move Frontmatter to the bottom of a file, both for parsing and output. - >>> with open("./content/reversed.txt") as f: + >>> with open("examples/content/reversed.txt") as f: ... text = f.read() >>> handler = ReverseYAMLHandler() >>> post = frontmatter.loads(text, handler=handler) @@ -47,9 +47,3 @@ def format(post, **kwargs): start_delimiter=start_delimiter, end_delimiter=end_delimiter, ).strip() - - -if __name__ == "__main__": - import doctest - - doctest.testmod() \ No newline at end of file diff --git a/frontmatter/__init__.py b/frontmatter/__init__.py index b4b290c..98bb73c 100644 --- a/frontmatter/__init__.py +++ b/frontmatter/__init__.py @@ -57,11 +57,11 @@ def parse(text, encoding="utf-8", handler=None, **defaults): .. testsetup:: * - import frontmatter + >>> import frontmatter .. doctest:: - >>> with open('../tests/yaml/hello-world.txt') as f: + >>> with open('tests/yaml/hello-world.txt') as f: ... metadata, content = frontmatter.parse(f.read()) >>> print(metadata['title']) Hello, world! @@ -102,7 +102,7 @@ def check(fd, encoding="utf-8"): .. doctest:: - >>> frontmatter.check('../tests/yaml/hello-world.txt') + >>> frontmatter.check('tests/yaml/hello-world.txt') True """ @@ -125,7 +125,7 @@ def checks(text, encoding="utf-8"): .. doctest:: - >>> with open('../tests/yaml/hello-world.txt') as f: + >>> with open('tests/yaml/hello-world.txt') as f: ... frontmatter.checks(f.read()) True @@ -141,8 +141,8 @@ def load(fd, encoding="utf-8", handler=None, **defaults): .. doctest:: - >>> post = frontmatter.load('../tests/yaml/hello-world.txt') - >>> with open('../tests/yaml/hello-world.txt') as f: + >>> post = frontmatter.load('tests/yaml/hello-world.txt') + >>> with open('tests/yaml/hello-world.txt') as f: ... post = frontmatter.load(f) """ @@ -163,7 +163,7 @@ def loads(text, encoding="utf-8", handler=None, **defaults): .. doctest:: - >>> with open('../tests/yaml/hello-world.txt') as f: + >>> with open('tests/yaml/hello-world.txt') as f: ... post = frontmatter.loads(f.read()) """ @@ -181,7 +181,7 @@ def dump(post, fd, encoding="utf-8", handler=None, **kwargs): :: >>> from io import BytesIO - >>> post = frontmatter.load('../tests/yaml/hello-world.txt') + >>> post = frontmatter.load('tests/yaml/hello-world.txt') >>> f = BytesIO() >>> frontmatter.dump(post, f) >>> print(f.getvalue().decode('utf-8')) @@ -189,14 +189,14 @@ def dump(post, fd, encoding="utf-8", handler=None, **kwargs): layout: post title: Hello, world! --- - + Well, hello there, world. .. testcode:: from io import BytesIO - post = frontmatter.load('../tests/yaml/hello-world.txt') + post = frontmatter.load('tests/yaml/hello-world.txt') f = BytesIO() frontmatter.dump(post, f) print(f.getvalue().decode('utf-8')) @@ -207,7 +207,7 @@ def dump(post, fd, encoding="utf-8", handler=None, **kwargs): layout: post title: Hello, world! --- - + Well, hello there, world. """ @@ -232,18 +232,18 @@ def dumps(post, handler=None, **kwargs): :: - >>> post = frontmatter.load('../tests/yaml/hello-world.txt') + >>> post = frontmatter.load('tests/yaml/hello-world.txt') >>> print(frontmatter.dumps(post)) # doctest: +NORMALIZE_WHITESPACE --- layout: post title: Hello, world! --- - + Well, hello there, world. .. testcode:: - post = frontmatter.load('../tests/yaml/hello-world.txt') + post = frontmatter.load('tests/yaml/hello-world.txt') print(frontmatter.dumps(post)) .. testoutput:: diff --git a/frontmatter/default_handlers.py b/frontmatter/default_handlers.py index a69c045..a6a0ab1 100644 --- a/frontmatter/default_handlers.py +++ b/frontmatter/default_handlers.py @@ -40,7 +40,7 @@ >>> import frontmatter >>> from frontmatter.default_handlers import YAMLHandler, TOMLHandler - >>> post = frontmatter.load('tests/toml/hello-toml.markdown', handler=TOMLHandler()) + >>> post = frontmatter.load('tests/toml/hello-toml.md', handler=TOMLHandler()) >>> post.handler #doctest: +ELLIPSIS @@ -97,11 +97,9 @@ # set YAML format when dumping, but the old handler attached >>> t1 = frontmatter.dumps(post, handler=YAMLHandler()) - # set a new handler, changing all future exports - >>> post.handler = YAMLHandler() + >>> post.handler = YAMLHandler() # set a new handler, changing all future exports >>> t2 = frontmatter.dumps(post) - # remove handler, defaulting back to YAML - >>> post.handler = None + >>> post.handler = None # remove handler, defaulting back to YAML >>> t3 = frontmatter.dumps(post) >>> t1 == t2 == t3 True From 131b00a3b5a48aeb50356e0114c9ba4a9b54cabc Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 10:30:38 -0400 Subject: [PATCH 19/27] Add handler.format to customize string output --- .github/workflows/test.yml | 2 +- examples/reversed.py | 37 ++++++++++++++++++++++++++++----- frontmatter/__init__.py | 19 +---------------- frontmatter/default_handlers.py | 28 ++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bdb1c1b..dd2c033 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,4 +26,4 @@ jobs: pip install -e '.[test]' - name: Run tests run: | - pytest . --doctest-modules + pytest . --doctest-modules --doctest-glob "README.md" diff --git a/examples/reversed.py b/examples/reversed.py index 2ddfdbc..56715f1 100644 --- a/examples/reversed.py +++ b/examples/reversed.py @@ -22,8 +22,35 @@ class ReverseYAMLHandler(frontmatter.YAMLHandler): Front matter, reversed >>> print(post['ref']) https://github.com/eyeseast/python-frontmatter/issues/67 - >>> frontmatter.dumps(post, handler=handler) == text.strip() - True + >>> print(frontmatter.dumps(post, handler=handler)) + This is txt format to prevent reformatting + ============================================ + + Dextra tempore deus + ------------------- + + Lorem markdownum est dicere pariter es dat si non, praesignis Styge, non + Maenalon magnae miserrimus. Corpora frustra committere insuetum et fecit + **Hippothousque arbore solio** inopem utraque concepit illa comantem me mortis + epulis protinus putares! Piceis *manibus*. Erinys et parum morsusque repugnat + ore corna sacris, pollice movet currus gestamina. + + Genitoris forti circumfuso videbit fertur vulnere simillima + ----------------------------------------------------------- + + Audit enim, est illa nervis loco inque hoc, et rigido! Monstris vatibus laetos + contemptor Calydonia. Et visa capillo referens regia: usus: odiique nostro. + **Vim** sensit inpulit virginis metuens secum cogit, corpus. + + Humus ater Dromas est honorem, Titanida glandibus sinit, e terras capillos + cremet retinentibus male. Tertia et cedit eliso flectere haec, cute nihil + marmore armo. Mihi [Olympi](http://que.org/saepepoenas), iam sustinet addidit + humana similis. + + --- + ref: https://github.com/eyeseast/python-frontmatter/issues/67 + title: Front matter, reversed + --- """ # FM_BOUNDARY as a string, so we can rsplit @@ -36,9 +63,9 @@ def split(self, text): content, fm, _ = text.rsplit(self.FM_BOUNDARY, 2) return fm, content - def format(post, **kwargs): - start_delimiter = kwargs.pop("start_delimiter", handler.START_DELIMITER) - end_delimiter = kwargs.pop("end_delimiter", handler.END_DELIMITER) + def format(self, post, **kwargs): + start_delimiter = kwargs.pop("start_delimiter", self.START_DELIMITER) + end_delimiter = kwargs.pop("end_delimiter", self.END_DELIMITER) metadata = self.export(post.metadata, **kwargs) return POST_TEMPLATE.format( diff --git a/frontmatter/__init__.py b/frontmatter/__init__.py index 98bb73c..92b6323 100644 --- a/frontmatter/__init__.py +++ b/frontmatter/__init__.py @@ -13,13 +13,6 @@ __all__ = ["parse", "load", "loads", "dump", "dumps"] -POST_TEMPLATE = """\ -{start_delimiter} -{metadata} -{end_delimiter} - -{content} -""" # global handlers handlers = { @@ -259,17 +252,7 @@ def dumps(post, handler=None, **kwargs): if handler is None: handler = getattr(post, "handler", None) or YAMLHandler() - start_delimiter = kwargs.pop("start_delimiter", handler.START_DELIMITER) - end_delimiter = kwargs.pop("end_delimiter", handler.END_DELIMITER) - - metadata = handler.export(post.metadata, **kwargs) - - return POST_TEMPLATE.format( - metadata=metadata, - content=post.content, - start_delimiter=start_delimiter, - end_delimiter=end_delimiter, - ).strip() + return handler.format(post, **kwargs) class Post(object): diff --git a/frontmatter/default_handlers.py b/frontmatter/default_handlers.py index a6a0ab1..07ecbb5 100644 --- a/frontmatter/default_handlers.py +++ b/frontmatter/default_handlers.py @@ -109,6 +109,7 @@ - split metadata and content, based on a boundary pattern (``handler.split``) - parse plain text metadata into a Python dictionary (``handler.load``) - export a dictionary back into plain text (``handler.export``) +- format exported metadata and content into a single string (``handler.format``) """ @@ -138,7 +139,16 @@ __all__.append("TOMLHandler") -class BaseHandler(object): +DEFAULT_POST_TEMPLATE = """\ +{start_delimiter} +{metadata} +{end_delimiter} + +{content} +""" + + +class BaseHandler: """ BaseHandler lays out all the steps to detecting, splitting, parsing and exporting front matter metadata. @@ -194,6 +204,22 @@ def export(self, metadata, **kwargs): """ raise NotImplementedError + def format(self, post, **kwargs): + """ + Turn a post into a string, used in ``frontmatter.dumps`` + """ + start_delimiter = kwargs.pop("start_delimiter", self.START_DELIMITER) + end_delimiter = kwargs.pop("end_delimiter", self.END_DELIMITER) + + metadata = self.export(post.metadata, **kwargs) + + return DEFAULT_POST_TEMPLATE.format( + metadata=metadata, + content=post.content, + start_delimiter=start_delimiter, + end_delimiter=end_delimiter, + ).strip() + class YAMLHandler(BaseHandler): """ From 2a9a5952be6e01fe6015b684aa9accdc583d9257 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 10:41:44 -0400 Subject: [PATCH 20/27] fix a warning --- docs/handlers.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/handlers.rst b/docs/handlers.rst index c6dc6be..b82002d 100644 --- a/docs/handlers.rst +++ b/docs/handlers.rst @@ -1,8 +1,6 @@ Customizing input and output ============================ -.. module:: frontmatter - .. automodule:: frontmatter.default_handlers .. autoclass:: frontmatter.default_handlers.BaseHandler From a5ce9254322980605e37a5631bfffac0c9d9d179 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 14:46:52 -0400 Subject: [PATCH 21/27] Use handler.detect instead of global dict --- frontmatter/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontmatter/__init__.py b/frontmatter/__init__.py index 92b6323..b9efb5d 100644 --- a/frontmatter/__init__.py +++ b/frontmatter/__init__.py @@ -15,11 +15,11 @@ # global handlers -handlers = { - Handler.FM_BOUNDARY: Handler() +handlers = [ + Handler() for Handler in [YAMLHandler, JSONHandler, TOMLHandler] if Handler is not None -} +] def detect_format(text, handlers): @@ -32,8 +32,8 @@ def detect_format(text, handlers): ``handlers`` is a dictionary where keys are opening delimiters and values are handler instances. """ - for pattern, handler in handlers.items(): - if pattern.match(text): + for handler in handlers: + if handler.detect(text): return handler # nothing matched, give nothing back From 12f1f429841b5c33aee6731574c38d8c0de2261e Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 15:51:14 -0400 Subject: [PATCH 22/27] Add pandoc example --- examples/content/pandoc.txt | 11 +++++++++++ examples/pandoc.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 examples/content/pandoc.txt create mode 100644 examples/pandoc.py diff --git a/examples/content/pandoc.txt b/examples/content/pandoc.txt new file mode 100644 index 0000000..be71421 --- /dev/null +++ b/examples/content/pandoc.txt @@ -0,0 +1,11 @@ +--- +lipsum: https://www.lipsum.com/ +ref: https://github.com/eyeseast/python-frontmatter/issues/42 +title: Pandoc-flavored Front Matter +... + +Nulla pulvinar, turpis ullamcorper tempus posuere, sapien purus porttitor diam, id ullamcorper lorem neque id mauris. Sed facilisis, elit eget luctus posuere, quam nibh imperdiet magna, vel placerat arcu risus sit amet arcu. Morbi sit amet mollis leo. Mauris sit amet condimentum mi. Quisque lectus libero, varius scelerisque tempor ut, elementum ac diam. Morbi nisl sapien, ullamcorper ac elementum at, auctor quis magna. Curabitur nec neque purus. + +Proin tincidunt cursus turpis, mattis euismod sapien. Cras consectetur id felis non volutpat. Proin vulputate ante gravida quam euismod blandit. Nulla at varius nibh. Donec congue erat vel mattis volutpat. Sed fringilla lorem sit amet velit ornare gravida. Sed ac scelerisque nisi. + +Generated 2 paragraphs, 109 words, 730 bytes of Lorem Ipsum diff --git a/examples/pandoc.py b/examples/pandoc.py new file mode 100644 index 0000000..cd84796 --- /dev/null +++ b/examples/pandoc.py @@ -0,0 +1,35 @@ +import frontmatter + + +class PandocHandler(frontmatter.YAMLHandler): + """ + Pandoc format uses different start and end delimiters, which trips up parsing a bit. + To make it work, we need a custom handler that deals with each part separately. + + >>> with open('examples/content/pandoc.txt') as f: + ... text = f.read() + >>> post = frontmatter.loads(text, handler=PandocHandler()) + >>> post['title'] + 'Pandoc-flavored Front Matter' + >>> print(frontmatter.dumps(post)) + --- + lipsum: https://www.lipsum.com/ + ref: https://github.com/eyeseast/python-frontmatter/issues/42 + title: Pandoc-flavored Front Matter + ... + + Nulla pulvinar, turpis ullamcorper tempus posuere, sapien purus porttitor diam, id ullamcorper lorem neque id mauris. Sed facilisis, elit eget luctus posuere, quam nibh imperdiet magna, vel placerat arcu risus sit amet arcu. Morbi sit amet mollis leo. Mauris sit amet condimentum mi. Quisque lectus libero, varius scelerisque tempor ut, elementum ac diam. Morbi nisl sapien, ullamcorper ac elementum at, auctor quis magna. Curabitur nec neque purus. + + Proin tincidunt cursus turpis, mattis euismod sapien. Cras consectetur id felis non volutpat. Proin vulputate ante gravida quam euismod blandit. Nulla at varius nibh. Donec congue erat vel mattis volutpat. Sed fringilla lorem sit amet velit ornare gravida. Sed ac scelerisque nisi. + + Generated 2 paragraphs, 109 words, 730 bytes of Lorem Ipsum + """ + + START_DELIMITER = "---" + END_DELIMITER = "..." + + def split(self, text): + "It's going to take a few passes to split here" + _, start, text = text.partition(self.START_DELIMITER) + fm, end, content = text.partition(self.END_DELIMITER) + return fm, content From b340fb6c66c2fadfcfb787089c828c912e62d661 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 21:07:38 -0400 Subject: [PATCH 23/27] Add sorted example --- examples/content/sorted.txt | 12 ++++++++++++ examples/sorted.py | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 examples/content/sorted.txt create mode 100644 examples/sorted.py diff --git a/examples/content/sorted.txt b/examples/content/sorted.txt new file mode 100644 index 0000000..eb5d32f --- /dev/null +++ b/examples/content/sorted.txt @@ -0,0 +1,12 @@ +--- +title: Pandoc-flavored Front Matter +ref: https://github.com/eyeseast/python-frontmatter/issues/26 +lipsum: https://www.lipsum.com/ +alpha: false +--- + +Nulla pulvinar, turpis ullamcorper tempus posuere, sapien purus porttitor diam, id ullamcorper lorem neque id mauris. Sed facilisis, elit eget luctus posuere, quam nibh imperdiet magna, vel placerat arcu risus sit amet arcu. Morbi sit amet mollis leo. Mauris sit amet condimentum mi. Quisque lectus libero, varius scelerisque tempor ut, elementum ac diam. Morbi nisl sapien, ullamcorper ac elementum at, auctor quis magna. Curabitur nec neque purus. + +Proin tincidunt cursus turpis, mattis euismod sapien. Cras consectetur id felis non volutpat. Proin vulputate ante gravida quam euismod blandit. Nulla at varius nibh. Donec congue erat vel mattis volutpat. Sed fringilla lorem sit amet velit ornare gravida. Sed ac scelerisque nisi. + +Generated 2 paragraphs, 109 words, 730 bytes of Lorem Ipsum diff --git a/examples/sorted.py b/examples/sorted.py new file mode 100644 index 0000000..eab4b35 --- /dev/null +++ b/examples/sorted.py @@ -0,0 +1,21 @@ +""" +This is an example of setting ``sort_keys=False`` in ``frontmatter.dumps`` +to preserve the original sort order. +""" +import frontmatter +from pathlib import Path + + +def test_load_sorted(): + examples = Path(__file__).parent + text = (examples / "content" / "sorted.txt").read_text() + post = frontmatter.loads(text) + + assert frontmatter.dumps(post, sort_keys=False) == text.strip() + + +if __name__ == "__main__": + import sys + import pytest + + pytest.main([__file__] + sys.argv[1:]) From 3b6d85f478a5774f77e3c198cca75c86d786f091 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 21:13:20 -0400 Subject: [PATCH 24/27] readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 618870b..61ffa92 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ This is a small package to load and parse files (or just text) with YAML front m [![Build Status](https://travis-ci.org/eyeseast/python-frontmatter.svg?branch=master)](https://travis-ci.org/eyeseast/python-frontmatter) +**[Documentation](https://python-frontmatter.readthedocs.io/en/latest/)** + ## Install: pip install python-frontmatter @@ -112,4 +114,4 @@ Well, hello there, world. ``` -For more examples, see files in the `tests/` directory. Each sample file has a corresponding `.result.json` file showing the expected parsed output. +For more examples, see files in the `tests/` directory. Each sample file has a corresponding `.result.json` file showing the expected parsed output. See also the `examples/` directory, which covers more ways to customize input and output. From b16c0f89cdd694c0b7b25c36e6ca8df8385171b4 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 21:33:40 -0400 Subject: [PATCH 25/27] no 3.5 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 8deecc4..2f17565 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From a2aac3bcefb416575c9045625e10c4efe32cc191 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 21:35:15 -0400 Subject: [PATCH 26/27] Stable --- .travis.yml | 12 ------------ setup.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d1f8a23..0000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: python -dist: xenial -python: - - "3.6" - - "3.7" - - "3.8" - - "3.9" -# command to install dependencies -install: - - "pip install -e .[test]" -# command to run tests -script: pytest diff --git a/setup.py b/setup.py index 2f17565..7e78df2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ zip_safe=False, keywords="frontmatter", classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Natural Language :: English", From 9c13a029c0baa585439cd471d65626dccefa57de Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 14 Mar 2021 21:39:54 -0400 Subject: [PATCH 27/27] readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 61ffa92..f0d7e91 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ [Jekyll](http://jekyllrb.com/)-style YAML front matter offers a useful way to add arbitrary, structured metadata to text documents, regardless of type. -This is a small package to load and parse files (or just text) with YAML front matter. +This is a small package to load and parse files (or just text) with YAML (or JSON, TOML or other) front matter. -[![Build Status](https://travis-ci.org/eyeseast/python-frontmatter.svg?branch=master)](https://travis-ci.org/eyeseast/python-frontmatter) +[![Tests](https://github.com/eyeseast/feed-to-sqlite/workflows/Test/badge.svg)](https://github.com/eyeseast/feed-to-sqlite/actions?query=workflow%3ATest) +[![PyPI](https://img.shields.io/pypi/v/python-frontmatter.svg)](https://pypi.org/project/python-frontmatter/) **[Documentation](https://python-frontmatter.readthedocs.io/en/latest/)**