sqlite: add basic implementation

.github/workflows/tests.yml

@@ -20,10 +20,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
+        os: [ubuntu-22.04, windows-latest, macos-latest]
         pyv: ['3.8', '3.9', '3.10', '3.11']
-        include:
-        - {os: ubuntu-latest, pyv: 'pypy3.8'}
 
     steps:
     - name: Check out the repository

.gitignore

@@ -136,3 +136,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# vim
+*.swp

.pre-commit-config.yaml

@@ -53,3 +53,9 @@ repos:
       - id: bandit
         args: [-c, pyproject.toml]
         additional_dependencies: ["toml"]
+# NOTE: temporarily skipped
+#  - repo: https://github.com/sqlfluff/sqlfluff
+#    rev: 1.4.2
+#    hooks:
+#      - id: sqlfluff-fix
+#        args: [--FIX-EVEN-UNPARSABLE, --force]

MANIFEST.in

@@ -0,0 +1 @@
+global-include *.sql

README.rst

@@ -1,5 +1,5 @@
 SQLTrie
-=======
+========
 
 |PyPI| |Status| |Python Version| |License|
 

pyproject.toml

@@ -63,8 +63,16 @@ warn_redundant_casts = true
 warn_unreachable = true
 files = ["src", "tests"]
 
+[tool.pylint.master]
+load-plugins = ["pylint_pytest"]
+
 [tool.pylint.message_control]
 enable = ["c-extension-no-member", "no-else-return"]
+disable = [
+    "fixme",
+    "missing-function-docstring", "missing-module-docstring",
+    "missing-class-docstring",
+]
 
 [tool.pylint.variables]
 dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_"
@@ -76,3 +84,23 @@ ignore-words-list = " "
 [tool.bandit]
 exclude_dirs = ["tests"]
 skips = ["B101"]
+
+[tool.sqlfluff.core]
+dialect = "sqlite"
+exclude_rules = "L031"
+
+[tool.sqlfluff.rules]
+tab_space_size = 4
+max_line_length = 80
+indent_unit = "space"
+allow_scalar = true
+single_table_references = "consistent"
+unquoted_identifiers_policy = "all"
+
+[tool.sqlfluff.rules.L010]
+capitalisation_policy = "upper"
+
+[tool.sqlfluff.rules.L029]
+# these are not reserved in sqlite,
+# see https://www.sqlite.org/lang_keywords.html
+ignore_words = ["name", "value", "depth"]

setup.cfg

@@ -5,10 +5,18 @@ long_description = file: README.rst
 long_description_content_type = text/x-rst
 license = Apache-2.0
 license_file = LICENSE
-url = https://github.com/efiop/sqltrie
+url = https://github.com/iterative/sqltrie
 platforms=any
-authors = Ruslan Kuprieiev
-maintainer_email = [email protected]
+authors = DVC team
+maintainer_email = [email protected]
+keywords =
+    sqlite
+    sqlite3
+    sql
+    trie
+    prefix tree
+    data-science
+    diskcache
 classifiers =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.8
@@ -23,11 +31,13 @@ zip_safe = False
 package_dir=
     =src
 packages = find:
+include_package_data = True
 install_requires=
 
 [options.extras_require]
 tests =
     pytest==7.2.0
+    pytest-benchmark
     pytest-sugar==0.9.5
     pytest-cov==3.0.0
     pytest-mock==3.8.2

src/sqltrie/__init__.py

@@ -1,5 +1,17 @@
-"""SQLTrie."""
-
-from .trie import AbstractTrie, ShortKeyError
-from .sqlite import SQLiteTrie
-
+from .serialized import (  # noqa: F401, pylint: disable=unused-import
+    JSONTrie,
+    SerializedTrie,
+)
+from .sqlite import SQLiteTrie  # noqa: F401, pylint: disable=unused-import
+from .trie import (  # noqa: F401, pylint: disable=unused-import
+    ADD,
+    DELETE,
+    MODIFY,
+    RENAME,
+    UNCHANGED,
+    AbstractTrie,
+    Change,
+    ShortKeyError,
+    TrieKey,
+    TrieNode,
+)

src/sqltrie/serialized.py

@@ -0,0 +1,89 @@
+import json
+from abc import abstractmethod
+from typing import Any, Optional
+
+from .trie import AbstractTrie, Iterator, TrieKey
+
+
+class SerializedTrie(AbstractTrie):
+    @property
+    @abstractmethod
+    def _trie(self):
+        pass
+
+    @abstractmethod
+    def _load(self, key: TrieKey, value: Optional[bytes]) -> Optional[Any]:
+        pass
+
+    @abstractmethod
+    def _dump(self, key: TrieKey, value: Optional[Any]) -> Optional[bytes]:
+        pass
+
+    def __setitem__(self, key, value):
+        self._trie[key] = self._dump(key, value)
+
+    def __getitem__(self, key):
+        raw = self._trie[key]
+        return self._load(key, raw)
+
+    def __delitem__(self, key):
+        del self._trie[key]
+
+    def __len__(self):
+        return len(self._trie)
+
+    def view(self, key: Optional[TrieKey] = None) -> "SerializedTrie":
+        if not key:
+            return self
+
+        raw_trie = self._trie.view(key)
+        trie = type(self)()
+        # pylint: disable-next=protected-access
+        trie._trie = raw_trie  # type: ignore
+        return trie
+
+    def items(self, *args, **kwargs):
+        yield from (
+            (key, self._load(key, raw))
+            for key, raw in self._trie.items(*args, **kwargs)
+        )
+
+    def ls(self, key):
+        yield from self._trie.ls(key)
+
+    def traverse(self, node_factory, prefix=None):
+        def _node_factory_wrapper(path_conv, path, children, value):
+            return node_factory(
+                path_conv, path, children, self._load(path, value)
+            )
+
+        return self._trie.traverse(_node_factory_wrapper, prefix=prefix)
+
+    def diff(self, *args, **kwargs):
+        yield from self._trie.diff(*args, **kwargs)
+
+    def has_node(self, key):
+        return self._trie.has_node(key)
+
+    def shortest_prefix(self, key):
+        skey, raw = self._trie.shortest_prefix(key)
+        return key, self._load(skey, raw)
+
+    def longest_prefix(self, key):
+        lkey, raw = self._trie.longest_prefix(key)
+        return lkey, self._load(lkey, raw)
+
+    def __iter__(self) -> Iterator[TrieKey]:
+        yield from self._trie
+
+
+class JSONTrie(SerializedTrie):  # pylint: disable=abstract-method
+    def _load(self, key: TrieKey, value: Optional[bytes]) -> Optional[Any]:
+        if value is None:
+            return None
+        return json.loads(value.decode("utf-8"))
+
+    def _dump(self, key: TrieKey, value: Optional[Any]) -> Optional[bytes]:
+        if value is None:
+            return None
+        return json.dumps(value).encode("utf-8")

src/sqltrie/sqlite/__init__.py

@@ -0,0 +1 @@
+from .sqlite import SQLiteTrie  # noqa: F401, pylint: disable=unused-import