diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 72bc682..0000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,46 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install Protoc - uses: arduino/setup-protoc@v2 - with: - version: '23.2' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip install -e . - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da2243e..cda6800 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,88 +1,120 @@ name: test on: push: - branches: [ master ] pull_request: - branches: [ master ] + schedule: + - cron: "0 8 * * *" jobs: - buildv1: + py_39_proto_203: strategy: fail-fast: false matrix: - # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs - os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.8', '3.9', '3.10', '3.11'] - protoc: ['3.20.3'] + os: [ ubuntu-latest, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - # https://github.com/actions/setup-python - with: - python-version: ${{ matrix.python }} - architecture: x64 - cache: pip - - name: Install Protoc - uses: arduino/setup-protoc@v1 - # https://github.com/arduino/setup-protoc - with: - version: ${{ matrix.protoc }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: install dependencies - run: | - python -mpip install --progress-bar=off -r ci/requirements.txt - virtualenv --version - pip --version - tox --version - pip list --format=freeze - protoc --version - - name: test - env: - TOXPYTHON: ${{ matrix.python }} - run: > - tox -e ${{ matrix.python }} -v - buildv2: - # repetition due to lack of support in `uses:` of context/matrix variables - # https://github.com/orgs/community/discussions/9049 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + architecture: x64 + cache: pip + - name: Install Protoc + uses: arduino/setup-protoc@v1.1.2 + with: + version: 3.20.3 + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: install dependencies + run: | + pip install --upgrade pip + pip install --progress-bar=off -r requirements/ci.txt + pip --version + tox --version + protoc --version + pip freeze + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: | + tox -e check --skip-pkg-install + tox --skip-pkg-install + + py_31x_proto_252: strategy: fail-fast: false matrix: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.8', '3.9', '3.10', '3.11'] - protoc: ['21.12', '23.x'] + python: [ '3.10', '3.11'] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # https://github.com/actions/checkout with: fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 # https://github.com/actions/setup-python with: python-version: ${{ matrix.python }} architecture: x64 cache: pip - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v3 # https://github.com/arduino/setup-protoc with: - version: ${{ matrix.protoc }} + version: 25.2 + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: install dependencies + run: | + pip install --upgrade pip + pip install --progress-bar=off -r requirements/ci.txt + pip --version + tox --version + protoc --version + pip freeze + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: | + tox -e check --skip-pkg-install + tox --skip-pkg-install + + py_3x_proto_25x: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + architecture: x64 + cache: pip + - name: Install Protoc + uses: arduino/setup-protoc@v3 + with: + version: 25.x repo-token: ${{ secrets.GITHUB_TOKEN }} - name: install dependencies run: | - python -mpip install --progress-bar=off -r ci/requirements.txt - virtualenv --version + pip install --upgrade pip + pip install --progress-bar=off -r requirements/ci.txt pip --version tox --version - pip list --format=freeze protoc --version - - name: test - env: - TOXPYTHON: ${{ matrix.python }} - run: > - tox -e ${{ matrix.python }} -v + pip freeze + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: | + tox -e check --skip-pkg-install + tox --skip-pkg-install \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 96db5da..e09d022 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,16 +1,14 @@ graft src -graft ci graft tests include .bumpversion.cfg include .coveragerc include tox.ini -include .github/workflows/github-actions.yml +include .github/workflows/test.yml include .pre-commit-config.yaml include LICENSE include README.md -include requirements.txt -include requirements_test.txt +recursive-include requirements *.txt global-exclude *.py[cod] __pycache__/* *.so *.dylib diff --git a/README.md b/README.md index 74d5549..53987dd 100644 --- a/README.md +++ b/README.md @@ -15,35 +15,50 @@ It is useful for Python programs needing to parse protobuf messages without havi pip install proto-topy -## Usage example +## Example: address book + +Adaptation of the `protocolbuffers` [example](https://github.com/protocolbuffers/protobuf/tree/main/examples): ```python -import sys, os -from pathlib import Path -from distutils.spawn import find_executable +import requests +import sys +from shutil import which from proto_topy.entities import ProtoModule -from google.protobuf.timestamp_pb2 import Timestamp - -protoc_path = Path(find_executable("protoc") or os.environ.get('PROTOC')) +from pathlib import Path -source = """ +# Retrieve protobuf messages definitions +example_source = requests.get( + "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/" + "examples/addressbook.proto").text - syntax = "proto3"; - import "google/protobuf/timestamp.proto"; - - message Test5 { - google.protobuf.Timestamp created = 1; - } +example_path = Path( + "protocolbuffers/protobuf/blob/main/examples/addressbook.proto") -""" +# Compile and import +module = (ProtoModule(file_path=example_path, source=example_source) + .compiled(Path(which("protoc")))) +sys.modules["addressbook"] = module.py -proto = ProtoModule(file_path=Path("test5.proto"), source=source).compiled(protoc_path) -sys.modules["test5"] = proto.py +# Produce a serialized address book +address_book = module.py.AddressBook() +person = address_book.people.add() +person.id = 111 +person.name = "A Name" +person.email = "a.name@mail.com" +phone_number = person.phones.add() +phone_number.number = "+1234567" +phone_number.type = module.py.Person.MOBILE +with open("address_book.data", "wb") as o: + o.write(address_book.SerializeToString()) -assert isinstance(proto.py.Test5().created, Timestamp) +# Use a serialized address book +address_book = module.py.AddressBook() +with open("address_book.data", "rb") as i: + address_book.ParseFromString(i.read()) + for person in address_book.people: + print(person.id, person.name, person.email, phone_number.number) ``` -More examples in [test_proto_topy.py][tests]. [pypi]: https://pypi.org/project/proto-topy [test_badge]: https://github.com/decitre/python-proto-topy/actions/workflows/test.yml/badge.svg diff --git a/ci/bootstrap.py b/ci/bootstrap.py deleted file mode 100755 index 3ca06b7..0000000 --- a/ci/bootstrap.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -import os -import subprocess -import sys -from os.path import abspath -from os.path import dirname -from os.path import exists -from os.path import join -from os.path import relpath - -base_path = dirname(dirname(abspath(__file__))) -templates_path = join(base_path, "ci", "templates") - - -def check_call(args): - print("+", *args) - subprocess.check_call(args) - - -def exec_in_env(): - env_path = join(base_path, ".tox", "bootstrap") - if sys.platform == "win32": - bin_path = join(env_path, "Scripts") - else: - bin_path = join(env_path, "bin") - if not exists(env_path): - import subprocess - - print("Making bootstrap env in: {0} ...".format(env_path)) - try: - check_call([sys.executable, "-m", "venv", env_path]) - except subprocess.CalledProcessError: - try: - check_call([sys.executable, "-m", "virtualenv", env_path]) - except subprocess.CalledProcessError: - check_call(["virtualenv", env_path]) - print("Installing `jinja2` into bootstrap environment...") - check_call([join(bin_path, "pip"), "install", "jinja2", "tox"]) - python_executable = join(bin_path, "python") - if not os.path.exists(python_executable): - python_executable += '.exe' - - print("Re-executing with: {0}".format(python_executable)) - print("+ exec", python_executable, __file__, "--no-env") - os.execv(python_executable, [python_executable, __file__, "--no-env"]) - - -def main(): - import jinja2 - - print("Project path: {0}".format(base_path)) - - jinja = jinja2.Environment( - loader=jinja2.FileSystemLoader(templates_path), - trim_blocks=True, - lstrip_blocks=True, - keep_trailing_newline=True, - ) - - tox_environments = [ - line.strip() - # 'tox' need not be installed globally, but must be importable - # by the Python that is running this script. - # This uses sys.executable the same way that the call in - # cookiecutter-pylibrary/hooks/post_gen_project.py - # invokes this bootstrap.py itself. - for line in subprocess.check_output([sys.executable, '-m', 'tox', '--listenvs'], universal_newlines=True).splitlines() - ] - tox_environments = [line for line in tox_environments if line.startswith('py')] - - for root, _, files in os.walk(templates_path): - for name in files: - relative = relpath(root, templates_path) - with open(join(base_path, relative, name), "w") as fh: - fh.write(jinja.get_template(join(relative, name)).render(tox_environments=tox_environments)) - print("Wrote {}".format(name)) - print("DONE.") - - -if __name__ == "__main__": - args = sys.argv[1:] - if args == ["--no-env"]: - main() - elif not args: - exec_in_env() - else: - print("Unexpected arguments {0}".format(args), file=sys.stderr) - sys.exit(1) diff --git a/ci/templates/.github/workflows/github-actions.yml b/ci/templates/.github/workflows/github-actions.yml deleted file mode 100644 index 7ee6426..0000000 --- a/ci/templates/.github/workflows/github-actions.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: build -on: [push, pull_request] -jobs: - test: - name: {{ '${{ matrix.name }}' }} - runs-on: {{ '${{ matrix.os }}' }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - name: 'check' - python: '3.9' - toxpython: 'python3.9' - tox_env: 'check' - os: 'ubuntu-latest' - - name: 'docs' - python: '3.9' - toxpython: 'python3.9' - tox_env: 'docs' - os: 'ubuntu-latest' -{% for env in tox_environments %} -{% set prefix = env.split('-')[0] -%} -{% if prefix.startswith('pypy') %} -{% set python %}pypy-{{ prefix[4] }}.{{ prefix[5] }}{% endset %} -{% set cpython %}pp{{ prefix[4:5] }}{% endset %} -{% set toxpython %}pypy{{ prefix[4] }}.{{ prefix[5] }}{% endset %} -{% else %} -{% set python %}{{ prefix[2] }}.{{ prefix[3:] }}{% endset %} -{% set cpython %}cp{{ prefix[2:] }}{% endset %} -{% set toxpython %}python{{ prefix[2] }}.{{ prefix[3:] }}{% endset %} -{% endif %} -{% for os, python_arch in [ - ['ubuntu', 'x64'], - ['windows', 'x64'], - ['macos', 'x64'], -] %} - - name: '{{ env }} ({{ os }})' - python: '{{ python }}' - toxpython: '{{ toxpython }}' - python_arch: '{{ python_arch }}' - tox_env: '{{ env }}{% if 'cover' in env %},codecov{% endif %}' - os: '{{ os }}-latest' -{% endfor %} -{% endfor %} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: actions/setup-python@v2 - with: - python-version: {{ '${{ matrix.python }}' }} - architecture: {{ '${{ matrix.python_arch }}' }} - - name: install dependencies - run: | - python -mpip install --progress-bar=off -r ci/requirements.txt - virtualenv --version - pip --version - tox --version - pip list --format=freeze - - name: test - env: - TOXPYTHON: '{{ '${{ matrix.toxpython }}' }}' - run: > - tox -e {{ '${{ matrix.tox_env }}' }} -v diff --git a/pyproject.toml b/pyproject.toml index d5293e7..2cd017b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,11 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 88 -target-version = ['py310'] +target-version = ['py311'] skip-string-normalization = true [tool.bumpver] -current_version = "0.1.0" +current_version = "0.2.0" version_pattern = "MAJOR.MINOR.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 779a5a3..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -protobuf \ No newline at end of file diff --git a/requirements/build.txt b/requirements/build.txt new file mode 100644 index 0000000..9aa2177 --- /dev/null +++ b/requirements/build.txt @@ -0,0 +1 @@ +protobuf>=3.20.3 \ No newline at end of file diff --git a/ci/requirements.txt b/requirements/ci.txt similarity index 85% rename from ci/requirements.txt rename to requirements/ci.txt index a0ef106..30fdd1c 100644 --- a/ci/requirements.txt +++ b/requirements/ci.txt @@ -3,3 +3,4 @@ pip>=19.1.1 setuptools>=18.0.1 six>=1.14.0 tox +tox-gh>=1.2 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 0000000..225503b --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,7 @@ +flake8 +protobuf +pytest +pytest-recording +requests +urllib3<2 +bumpver \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt deleted file mode 100644 index fd5c0a9..0000000 --- a/requirements_test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest -protobuf \ No newline at end of file diff --git a/setup.py b/setup.py index 2ab61dd..c6eb09e 100755 --- a/setup.py +++ b/setup.py @@ -67,8 +67,8 @@ def read(*names, **kwargs): _packages = find_packages(where=_packages_path) version = get_property("__version__", _packages_path, _packages).pop() requirements = ["click"] - requirements.extend(get_requirements("requirements.txt", no_precise_version=True)) - requirements_test = get_requirements("requirements_test.txt") + requirements.extend(get_requirements("requirements/build.txt", no_precise_version=False)) + requirements_dev = get_requirements("requirements/dev.txt") setup( name=project_name, @@ -93,6 +93,7 @@ def read(*names, **kwargs): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", 'Programming Language :: Python :: Implementation :: CPython', "Operating System :: OS Independent", ], @@ -102,5 +103,5 @@ def read(*names, **kwargs): keywords=['protobuf'], python_requires='>=3.8', install_requires=requirements, - extras_require={"dev": requirements_test}, + extras_require={"dev": requirements_dev}, ) diff --git a/src/proto_topy/__init__.py b/src/proto_topy/__init__.py index 3dc1f76..d3ec452 100644 --- a/src/proto_topy/__init__.py +++ b/src/proto_topy/__init__.py @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/proto_topy/entities.py b/src/proto_topy/entities.py index 9da0cb4..0ab5559 100644 --- a/src/proto_topy/entities.py +++ b/src/proto_topy/entities.py @@ -1,14 +1,15 @@ +from google.protobuf import descriptor_pool +from google.protobuf.message import Message +from google.protobuf.message_factory import GetMessageClassesForFiles from google.protobuf.descriptor_pb2 import FileDescriptorSet -from google.protobuf.message_factory import GetMessages from google.protobuf.internal.encoder import _VarintBytes from google.protobuf.internal.decoder import _DecodeVarint32 -from google.protobuf.message import Message import os import importlib.util import sys import types -from distutils.spawn import find_executable +from shutil import which from pathlib import Path from tempfile import TemporaryDirectory from subprocess import PIPE, Popen @@ -69,12 +70,13 @@ def __init__(self, compiler_path: Path, *protos: ProtoModule): self.descriptor_data = None self.descriptor_set = None self.messages = {} + self.pool = descriptor_pool.DescriptorPool() if not self.compiler_path: if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): self.compiler_path = Path(os.environ['PROTOC']) else: - self.compiler_path or Path(find_executable('protoc')) + self.compiler_path or Path(which('protoc')) if not self.compiler_path.is_file(): raise FileNotFoundError() @@ -119,7 +121,9 @@ def compile(self, global_scope: dict = None) -> "ProtoCollection": with open(str(artifact_fds_path), mode="rb") as f: self.descriptor_data = f.read() self.descriptor_set = FileDescriptorSet.FromString(self.descriptor_data) - self.messages = GetMessages([file for file in self.descriptor_set.file]) + for file_descriptor_proto in self.descriptor_set.file: + self.pool.Add(file_descriptor_proto) + self.messages = GetMessageClassesForFiles([fdp.name for fdp in self.descriptor_set.file], self.pool) self._add_init_files(dir) @@ -132,7 +136,6 @@ def compile(self, global_scope: dict = None) -> "ProtoCollection": sys.path.pop() return self - def version(self) -> str: outs = ProtoCollection._do_compile( self.compiler_path, @@ -143,7 +146,6 @@ def version(self) -> str: if outs: return outs.split()[-1].decode() - @staticmethod def _do_compile( compiler_path: Path, diff --git a/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml b/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml new file mode 100644 index 0000000..d69dacb --- /dev/null +++ b/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml @@ -0,0 +1,84 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.31.0 + method: GET + uri: https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto + response: + body: + string: !!binary | + H4sIAAAAAAAAA22UXU/bMBSG7/Mrjno9ko1uN0RcgKg2tEEr6LSLCVVucpqYJrFnHzPQxH/fsZ2k + 6YZURY3j877P+bCzDO4R4W5xcXWzSNsSdsqA7PjZCpKqA9GVsHWyKXnVknGFX7VpkmX8g1tFeAb3 + 64u7ddi5uL0CEpUFYRCcRR8FhWpb7MgCKShxJzsEi1Fn2OO1yJEyUjQ2BVjX+BI0OkWghSFQO6Ca + A186Es+BcmUUqUI1cOl2OzQj01pBhcQ4rHtSoqYafotmT7VRrqqjkLSwkw0GZi9rsBHEJPgsWt2g + fceEeObVaiJtz7KsxCdslPY+lVJVgymnleme4WQbGbJSFTYbM0m8ws9YnhKLRphQ1IekT+McZkFh + PssTLYq9qHAsQ54kstWKU59Fw2jGRhnJFi0xaRqWONjb+NofmUzMH8WT2Bx9VDq0N3xoXUOS0974 + mlim4j5jfrRloGNizjvt65QOsBHEzo6DlCM0Gza1thNtCL4oS4PWXiq1Xw0hA/z/kJMMClsLo9/M + of/kLSxzBp/PsUervmTpom9sOgGYWL+lPjGv1JvGvDypSyWpdtujsRimYuzcMF9ZpcYp0dsJyL9O + EwjOzrKVfUj6f7Bibcb4kwDw0ZRdBX2ZP+S8JDuan4Is+f00B2Ch75385RCur6Bz7RZNOEbhMOig + lB6EsBWy4cg5jyEA8n5Y1arD9YvGYAhws7y8/rbgPe/z8P5lebMYvAF+LO++Bmd+e/UaI7SXuY3+ + UWhgj2ujwsGP/GOqZVBjOK9TMe3/++n9GJj7UzqUPl0PhwZ4HmnjdBkUzuFTnryGOi+dARHHA7Y8 + H/GK4Oo8OssXEF9b8RKymI4tmIxTyOZAFnujUXG/Y1KvY5cPrfwLfa1h8YAFAAA= + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - max-age=300 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Length: + - '674' + Content-Security-Policy: + - default-src 'none'; style-src 'unsafe-inline'; sandbox + Content-Type: + - text/plain; charset=utf-8 + Cross-Origin-Resource-Policy: + - cross-origin + Date: + - Sat, 03 Feb 2024 14:11:23 GMT + ETag: + - W/"3609494263025aad9f621723990edef51c0d69d5dbaa781f6a5867e4e66b4c94" + Expires: + - Sat, 03 Feb 2024 14:16:23 GMT + Source-Age: + - '0' + Strict-Transport-Security: + - max-age=31536000 + Vary: + - Authorization,Accept-Encoding,Origin + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Content-Type-Options: + - nosniff + X-Fastly-Request-ID: + - ed62ed6bb3831cc99bc14b1b39f32cd2859affe4 + X-Frame-Options: + - deny + X-GitHub-Request-Id: + - 4716:250E4C:2B13C31:2CCC87B:65BE4834 + X-Served-By: + - cache-fra-eddf8230121-FRA + X-Timer: + - S1706969484.625163,VS0,VE134 + X-XSS-Protection: + - 1; mode=block + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_proto_topy.py b/tests/test_proto_topy.py index 614297e..118ea86 100644 --- a/tests/test_proto_topy.py +++ b/tests/test_proto_topy.py @@ -1,19 +1,27 @@ +import pytest +import requests + from proto_topy.entities import ( ProtoCollection, ProtoModule, CompilationFailed, DelimitedMessageFactory, ) -import pytest + import os import sys from pathlib import Path -from distutils.spawn import find_executable +from shutil import which from array import array from io import BytesIO -protoc_path = Path(find_executable("protoc") or os.environ.get('PROTOC')) +protoc_path = Path(which("protoc") or os.environ.get('PROTOC')) + + +@pytest.fixture() +def address_book(): + return requests.get("https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto").text def test_compiler_version(): @@ -27,6 +35,7 @@ def unlink_proto(path_str: str) -> Path: proto.unlink() return proto + def test_add_proto(): test1_proto = unlink_proto("test1.proto") proto = ProtoModule(file_path=test1_proto, source="") @@ -71,6 +80,7 @@ def test_compile_invalid_source(): ProtoModule(file_path=test4_proto, source="foo").compiled(protoc_path) unlink_proto("test4.proto") + def test_compile_redundant_proto(): testr_proto = unlink_proto("testr.proto") proto_source = 'syntax = "proto3"; message TestR { int32 foo = 1; };' @@ -80,6 +90,7 @@ def test_compile_redundant_proto(): ProtoCollection(protoc_path, proto1, proto2).compile() unlink_proto("testr.proto") + def test_compile_minimal_proto(): from google.protobuf.timestamp_pb2 import Timestamp @@ -116,11 +127,10 @@ def test_compile_minimal_proto_in_a_package(): """, ).compiled(protoc_path) assert ( - "\n".join(proto.py_source.split("\n")[:4]) - == '''# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: p1/p2/p3/thing.proto -"""Generated protocol buffer code."""''' + proto.py_source.split("\n")[:3] + == ["# -*- coding: utf-8 -*-", + "# Generated by the protocol buffer compiler. DO NOT EDIT!", + "# source: p1/p2/p3/thing.proto"] ) sys.modules["thing"] = proto.py athing = proto.py.Thing() @@ -140,7 +150,6 @@ def test_compile_missing_dependency(): def test_compile_ununsed_dependency(): test_proto = unlink_proto("test.proto") - #test_proto.touch() proto_module = ProtoModule( file_path=test_proto, source=""" @@ -205,6 +214,7 @@ def test_compile_simple_dependency(): unlink_proto("p3/p4/test6.proto") unlink_proto("p1/p2/other2.proto") + def test_encode_message(): proto_source = 'syntax = "proto3"; message Test{n} {{ int32 foo = 1; }};' test7_proto = unlink_proto("test7.proto") @@ -269,3 +279,35 @@ def test_decode_messages_stream2(): foos.append(aTest11.foo) assert foos == [1, 12] unlink_proto("test11.proto") + + +@pytest.mark.vcr +def test_google_addressbook_example(address_book): + + adressbook_proto = unlink_proto("protocolbuffers/protobuf/blob/main/examples/addressbook.proto") + proto = ProtoModule( + file_path=adressbook_proto, + source=address_book, + ).compiled(protoc_path) + sys.modules["addressbook"] = proto.py + + # Produce serialized address book + address_book = proto.py.AddressBook() + person = address_book.people.add() + person.id = 111 + person.name = "A Name" + person.email = "a.name@mail.com" + phone_number = person.phones.add() + phone_number.number = "+1234567" + phone_number.type = proto.py.Person.MOBILE + address_book_data = address_book.SerializeToString() + + # Read address book + address_book = proto.py.AddressBook() + address_book.ParseFromString(address_book_data) + person = address_book.people[0] + assert person.id == 111 + assert person.name == "A Name" + assert person.email == "a.name@mail.com" + assert phone_number.number == "+1234567" + assert phone_number.type == proto.py.Person.MOBILE diff --git a/tests/tox_mac.sh b/tests/tox_mac.sh new file mode 100755 index 0000000..f68307e --- /dev/null +++ b/tests/tox_mac.sh @@ -0,0 +1,19 @@ +PROTOC_BOTTLES="protobuf@3 protobuf@21 protobuf" +YELLOW='\033[33m' +NC='\033[0m' + +brew install $PROTOC_BOTTLES +brew unlink $PROTOC_BOTTLES +PATHBAK=$PATH + +for bottle in $PROTOC_BOTTLES; do + printf "${YELLOW}Use $bottle${NC}\n" + brew link --overwrite --force $bottle + PATH="/usr/local/opt/$bottle/bin:$PATHBAK" + protoc --version + tox + [[ $? -ne 0 ]] && break + brew unlink $bottle +done + +brew link --overwrite protobuf diff --git a/tox.ini b/tox.ini index d2cd327..3a04dcb 100644 --- a/tox.ini +++ b/tox.ini @@ -13,16 +13,25 @@ passenv = envlist = clean, check, - {3.8,3.9,3.10,3.11}, + {3.8,3.9,3.10,3.11,3.12}, report ignore_basepython_conflict = true +[gh] +python = + 3.12 = py312 + 3.11 = py311 + 3.10 = py310 + 3.9 = py39 + 3.8 = py38 + [testenv] basepython = 3.8: {env:TOXPYTHON:python3.8} 3.9: {env:TOXPYTHON:python3.9} 3.10: {env:TOXPYTHON:python3.10} 3.11: {env:TOXPYTHON:python3.11} + 3.12: {env:TOXPYTHON:python3.12} {bootstrap,clean,check,report,codecov}: {env:TOXPYTHON:python3} setenv = PYTHONPATH={toxinidir}/tests @@ -33,6 +42,9 @@ usedevelop = false deps = pytest pytest-cov + requests + pytest-recording + urllib3<2 commands = {posargs:pytest --cov --cov-report=term-missing -vv tests}