From 0584fb8ec94111d19d19b90ceed849fa7fd2659c Mon Sep 17 00:00:00 2001 From: Emmanuel Decitre Date: Sat, 3 Feb 2024 15:56:47 +0100 Subject: [PATCH 01/13] replace the use of protobuf `GetMessages` with `descriptor_pool` and `GetMessageClassesForFiles` add support for Python 3.12 add protoc matrix to actions --- .github/workflows/python-package.yml | 2 +- .github/workflows/test.yml | 18 ++-- README.md | 4 +- ci/requirements.txt | 1 + requirements.txt | 2 +- requirements_test.txt | 5 +- setup.py | 2 +- src/proto_topy/entities.py | 16 ++-- .../test_google_addressbook_example.yaml | 84 +++++++++++++++++++ tests/test_proto_topy.py | 60 +++++++++++-- tests/tox_mac.sh | 19 +++++ tox.ini | 7 +- 12 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml create mode 100755 tests/tox_mac.sh diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 72bc682..30d6871 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da2243e..26da975 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,22 +12,22 @@ jobs: matrix: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.8', '3.9', '3.10', '3.11'] - protoc: ['3.20.3'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] + protoc: ['3.20.3', '3.21.12', '25.2'] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 # https://github.com/actions/setup-python with: python-version: ${{ matrix.python }} architecture: x64 cache: pip - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v3 # https://github.com/arduino/setup-protoc with: version: ${{ matrix.protoc }} @@ -53,22 +53,22 @@ jobs: matrix: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.8', '3.9', '3.10', '3.11'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] protoc: ['21.12', '23.x'] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 # https://github.com/actions/setup-python with: python-version: ${{ matrix.python }} architecture: x64 cache: pip - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v3 # https://github.com/arduino/setup-protoc with: version: ${{ matrix.protoc }} diff --git a/README.md b/README.md index 74d5549..e853151 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,11 @@ It is useful for Python programs needing to parse protobuf messages without havi ```python import sys, os from pathlib import Path -from distutils.spawn import find_executable +from shutil import which from proto_topy.entities import ProtoModule from google.protobuf.timestamp_pb2 import Timestamp -protoc_path = Path(find_executable("protoc") or os.environ.get('PROTOC')) +protoc_path = Path(which("protoc") or os.environ.get('PROTOC')) source = """ diff --git a/ci/requirements.txt b/ci/requirements.txt index a0ef106..7b876e8 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -3,3 +3,4 @@ pip>=19.1.1 setuptools>=18.0.1 six>=1.14.0 tox +virtualenv-pyenv \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 779a5a3..d25ebc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -protobuf \ No newline at end of file +protobuf>=3.20.0 \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt index fd5c0a9..f1233c2 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,2 +1,5 @@ pytest -protobuf \ No newline at end of file +protobuf +requests +pytest-recording +urllib3<2 \ No newline at end of file diff --git a/setup.py b/setup.py index 2ab61dd..24fb341 100755 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ def read(*names, **kwargs): _packages = find_packages(where=_packages_path) version = get_property("__version__", _packages_path, _packages).pop() requirements = ["click"] - requirements.extend(get_requirements("requirements.txt", no_precise_version=True)) + requirements.extend(get_requirements("requirements.txt", no_precise_version=False)) requirements_test = get_requirements("requirements_test.txt") setup( diff --git a/src/proto_topy/entities.py b/src/proto_topy/entities.py index 9da0cb4..0ab5559 100644 --- a/src/proto_topy/entities.py +++ b/src/proto_topy/entities.py @@ -1,14 +1,15 @@ +from google.protobuf import descriptor_pool +from google.protobuf.message import Message +from google.protobuf.message_factory import GetMessageClassesForFiles from google.protobuf.descriptor_pb2 import FileDescriptorSet -from google.protobuf.message_factory import GetMessages from google.protobuf.internal.encoder import _VarintBytes from google.protobuf.internal.decoder import _DecodeVarint32 -from google.protobuf.message import Message import os import importlib.util import sys import types -from distutils.spawn import find_executable +from shutil import which from pathlib import Path from tempfile import TemporaryDirectory from subprocess import PIPE, Popen @@ -69,12 +70,13 @@ def __init__(self, compiler_path: Path, *protos: ProtoModule): self.descriptor_data = None self.descriptor_set = None self.messages = {} + self.pool = descriptor_pool.DescriptorPool() if not self.compiler_path: if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): self.compiler_path = Path(os.environ['PROTOC']) else: - self.compiler_path or Path(find_executable('protoc')) + self.compiler_path or Path(which('protoc')) if not self.compiler_path.is_file(): raise FileNotFoundError() @@ -119,7 +121,9 @@ def compile(self, global_scope: dict = None) -> "ProtoCollection": with open(str(artifact_fds_path), mode="rb") as f: self.descriptor_data = f.read() self.descriptor_set = FileDescriptorSet.FromString(self.descriptor_data) - self.messages = GetMessages([file for file in self.descriptor_set.file]) + for file_descriptor_proto in self.descriptor_set.file: + self.pool.Add(file_descriptor_proto) + self.messages = GetMessageClassesForFiles([fdp.name for fdp in self.descriptor_set.file], self.pool) self._add_init_files(dir) @@ -132,7 +136,6 @@ def compile(self, global_scope: dict = None) -> "ProtoCollection": sys.path.pop() return self - def version(self) -> str: outs = ProtoCollection._do_compile( self.compiler_path, @@ -143,7 +146,6 @@ def version(self) -> str: if outs: return outs.split()[-1].decode() - @staticmethod def _do_compile( compiler_path: Path, diff --git a/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml b/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml new file mode 100644 index 0000000..d69dacb --- /dev/null +++ b/tests/cassettes/test_proto_topy/test_google_addressbook_example.yaml @@ -0,0 +1,84 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.31.0 + method: GET + uri: https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto + response: + body: + string: !!binary | + H4sIAAAAAAAAA22UXU/bMBSG7/Mrjno9ko1uN0RcgKg2tEEr6LSLCVVucpqYJrFnHzPQxH/fsZ2k + 6YZURY3j877P+bCzDO4R4W5xcXWzSNsSdsqA7PjZCpKqA9GVsHWyKXnVknGFX7VpkmX8g1tFeAb3 + 64u7ddi5uL0CEpUFYRCcRR8FhWpb7MgCKShxJzsEi1Fn2OO1yJEyUjQ2BVjX+BI0OkWghSFQO6Ca + A186Es+BcmUUqUI1cOl2OzQj01pBhcQ4rHtSoqYafotmT7VRrqqjkLSwkw0GZi9rsBHEJPgsWt2g + fceEeObVaiJtz7KsxCdslPY+lVJVgymnleme4WQbGbJSFTYbM0m8ws9YnhKLRphQ1IekT+McZkFh + PssTLYq9qHAsQ54kstWKU59Fw2jGRhnJFi0xaRqWONjb+NofmUzMH8WT2Bx9VDq0N3xoXUOS0974 + mlim4j5jfrRloGNizjvt65QOsBHEzo6DlCM0Gza1thNtCL4oS4PWXiq1Xw0hA/z/kJMMClsLo9/M + of/kLSxzBp/PsUervmTpom9sOgGYWL+lPjGv1JvGvDypSyWpdtujsRimYuzcMF9ZpcYp0dsJyL9O + EwjOzrKVfUj6f7Bibcb4kwDw0ZRdBX2ZP+S8JDuan4Is+f00B2Ch75385RCur6Bz7RZNOEbhMOig + lB6EsBWy4cg5jyEA8n5Y1arD9YvGYAhws7y8/rbgPe/z8P5lebMYvAF+LO++Bmd+e/UaI7SXuY3+ + UWhgj2ujwsGP/GOqZVBjOK9TMe3/++n9GJj7UzqUPl0PhwZ4HmnjdBkUzuFTnryGOi+dARHHA7Y8 + H/GK4Oo8OssXEF9b8RKymI4tmIxTyOZAFnujUXG/Y1KvY5cPrfwLfa1h8YAFAAA= + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - max-age=300 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Length: + - '674' + Content-Security-Policy: + - default-src 'none'; style-src 'unsafe-inline'; sandbox + Content-Type: + - text/plain; charset=utf-8 + Cross-Origin-Resource-Policy: + - cross-origin + Date: + - Sat, 03 Feb 2024 14:11:23 GMT + ETag: + - W/"3609494263025aad9f621723990edef51c0d69d5dbaa781f6a5867e4e66b4c94" + Expires: + - Sat, 03 Feb 2024 14:16:23 GMT + Source-Age: + - '0' + Strict-Transport-Security: + - max-age=31536000 + Vary: + - Authorization,Accept-Encoding,Origin + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Content-Type-Options: + - nosniff + X-Fastly-Request-ID: + - ed62ed6bb3831cc99bc14b1b39f32cd2859affe4 + X-Frame-Options: + - deny + X-GitHub-Request-Id: + - 4716:250E4C:2B13C31:2CCC87B:65BE4834 + X-Served-By: + - cache-fra-eddf8230121-FRA + X-Timer: + - S1706969484.625163,VS0,VE134 + X-XSS-Protection: + - 1; mode=block + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_proto_topy.py b/tests/test_proto_topy.py index 614297e..118ea86 100644 --- a/tests/test_proto_topy.py +++ b/tests/test_proto_topy.py @@ -1,19 +1,27 @@ +import pytest +import requests + from proto_topy.entities import ( ProtoCollection, ProtoModule, CompilationFailed, DelimitedMessageFactory, ) -import pytest + import os import sys from pathlib import Path -from distutils.spawn import find_executable +from shutil import which from array import array from io import BytesIO -protoc_path = Path(find_executable("protoc") or os.environ.get('PROTOC')) +protoc_path = Path(which("protoc") or os.environ.get('PROTOC')) + + +@pytest.fixture() +def address_book(): + return requests.get("https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto").text def test_compiler_version(): @@ -27,6 +35,7 @@ def unlink_proto(path_str: str) -> Path: proto.unlink() return proto + def test_add_proto(): test1_proto = unlink_proto("test1.proto") proto = ProtoModule(file_path=test1_proto, source="") @@ -71,6 +80,7 @@ def test_compile_invalid_source(): ProtoModule(file_path=test4_proto, source="foo").compiled(protoc_path) unlink_proto("test4.proto") + def test_compile_redundant_proto(): testr_proto = unlink_proto("testr.proto") proto_source = 'syntax = "proto3"; message TestR { int32 foo = 1; };' @@ -80,6 +90,7 @@ def test_compile_redundant_proto(): ProtoCollection(protoc_path, proto1, proto2).compile() unlink_proto("testr.proto") + def test_compile_minimal_proto(): from google.protobuf.timestamp_pb2 import Timestamp @@ -116,11 +127,10 @@ def test_compile_minimal_proto_in_a_package(): """, ).compiled(protoc_path) assert ( - "\n".join(proto.py_source.split("\n")[:4]) - == '''# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: p1/p2/p3/thing.proto -"""Generated protocol buffer code."""''' + proto.py_source.split("\n")[:3] + == ["# -*- coding: utf-8 -*-", + "# Generated by the protocol buffer compiler. DO NOT EDIT!", + "# source: p1/p2/p3/thing.proto"] ) sys.modules["thing"] = proto.py athing = proto.py.Thing() @@ -140,7 +150,6 @@ def test_compile_missing_dependency(): def test_compile_ununsed_dependency(): test_proto = unlink_proto("test.proto") - #test_proto.touch() proto_module = ProtoModule( file_path=test_proto, source=""" @@ -205,6 +214,7 @@ def test_compile_simple_dependency(): unlink_proto("p3/p4/test6.proto") unlink_proto("p1/p2/other2.proto") + def test_encode_message(): proto_source = 'syntax = "proto3"; message Test{n} {{ int32 foo = 1; }};' test7_proto = unlink_proto("test7.proto") @@ -269,3 +279,35 @@ def test_decode_messages_stream2(): foos.append(aTest11.foo) assert foos == [1, 12] unlink_proto("test11.proto") + + +@pytest.mark.vcr +def test_google_addressbook_example(address_book): + + adressbook_proto = unlink_proto("protocolbuffers/protobuf/blob/main/examples/addressbook.proto") + proto = ProtoModule( + file_path=adressbook_proto, + source=address_book, + ).compiled(protoc_path) + sys.modules["addressbook"] = proto.py + + # Produce serialized address book + address_book = proto.py.AddressBook() + person = address_book.people.add() + person.id = 111 + person.name = "A Name" + person.email = "a.name@mail.com" + phone_number = person.phones.add() + phone_number.number = "+1234567" + phone_number.type = proto.py.Person.MOBILE + address_book_data = address_book.SerializeToString() + + # Read address book + address_book = proto.py.AddressBook() + address_book.ParseFromString(address_book_data) + person = address_book.people[0] + assert person.id == 111 + assert person.name == "A Name" + assert person.email == "a.name@mail.com" + assert phone_number.number == "+1234567" + assert phone_number.type == proto.py.Person.MOBILE diff --git a/tests/tox_mac.sh b/tests/tox_mac.sh new file mode 100755 index 0000000..f68307e --- /dev/null +++ b/tests/tox_mac.sh @@ -0,0 +1,19 @@ +PROTOC_BOTTLES="protobuf@3 protobuf@21 protobuf" +YELLOW='\033[33m' +NC='\033[0m' + +brew install $PROTOC_BOTTLES +brew unlink $PROTOC_BOTTLES +PATHBAK=$PATH + +for bottle in $PROTOC_BOTTLES; do + printf "${YELLOW}Use $bottle${NC}\n" + brew link --overwrite --force $bottle + PATH="/usr/local/opt/$bottle/bin:$PATHBAK" + protoc --version + tox + [[ $? -ne 0 ]] && break + brew unlink $bottle +done + +brew link --overwrite protobuf diff --git a/tox.ini b/tox.ini index d2cd327..3ba0590 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ passenv = envlist = clean, check, - {3.8,3.9,3.10,3.11}, + {3.8,3.9,3.10,3.11,3.12}, report ignore_basepython_conflict = true @@ -23,16 +23,21 @@ basepython = 3.9: {env:TOXPYTHON:python3.9} 3.10: {env:TOXPYTHON:python3.10} 3.11: {env:TOXPYTHON:python3.11} + 3.12: {env:TOXPYTHON:python3.12} {bootstrap,clean,check,report,codecov}: {env:TOXPYTHON:python3} setenv = PYTHONPATH={toxinidir}/tests PYTHONUNBUFFERED=yes + VIRTUALENV_DISCOVERY = pyenv passenv = * usedevelop = false deps = pytest pytest-cov + requests + pytest-recording + urllib3<2 commands = {posargs:pytest --cov --cov-report=term-missing -vv tests} From 8dcd34a500ad5079f5e2111ed7fbdb2c8b8c5cb1 Mon Sep 17 00:00:00 2001 From: Emmanuel Decitre Date: Sat, 3 Feb 2024 16:15:25 +0100 Subject: [PATCH 02/13] Iterrate over python-package.yml --- .github/workflows/python-package.yml | 4 +--- requirements_test.txt | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 30d6871..c6e1236 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,9 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip install -e . + pip install -r requirements_test.txt -r requirements.txt -e . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/requirements_test.txt b/requirements_test.txt index f1233c2..039b521 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,6 @@ -pytest +flake8 protobuf -requests +pytest pytest-recording +requests urllib3<2 \ No newline at end of file From 09758c699c9a95cfdd424c10f7bbdac2d726eff0 Mon Sep 17 00:00:00 2001 From: Emmanuel Decitre Date: Sat, 3 Feb 2024 16:31:28 +0100 Subject: [PATCH 03/13] Iterate over python-packages.yml, test.yml (remove pyenv dependency) --- .github/workflows/python-package.yml | 8 ++--- .github/workflows/test.yml | 45 ++-------------------------- ci/requirements.txt | 2 +- 3 files changed, 8 insertions(+), 47 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c6e1236..66a4c59 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -19,15 +19,15 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v3 with: - version: '23.2' + version: '25.2' repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26da975..0d4ae66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,13 +6,14 @@ on: branches: [ master ] jobs: - buildv1: + build: strategy: fail-fast: false matrix: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs os: [ubuntu-latest, macos-latest, windows-latest] python: ['3.8', '3.9', '3.10', '3.11', '3.12'] + # ['21.12', '23.x'] ?? protoc: ['3.20.3', '3.21.12', '25.2'] runs-on: ${{ matrix.os }} timeout-minutes: 30 @@ -45,44 +46,4 @@ jobs: TOXPYTHON: ${{ matrix.python }} run: > tox -e ${{ matrix.python }} -v - buildv2: - # repetition due to lack of support in `uses:` of context/matrix variables - # https://github.com/orgs/community/discussions/9049 - strategy: - fail-fast: false - matrix: - # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs - os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.8', '3.9', '3.10', '3.11', '3.12'] - protoc: ['21.12', '23.x'] - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@v5 - # https://github.com/actions/setup-python - with: - python-version: ${{ matrix.python }} - architecture: x64 - cache: pip - - name: Install Protoc - uses: arduino/setup-protoc@v3 - # https://github.com/arduino/setup-protoc - with: - version: ${{ matrix.protoc }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: install dependencies - run: | - python -mpip install --progress-bar=off -r ci/requirements.txt - virtualenv --version - pip --version - tox --version - pip list --format=freeze - protoc --version - - name: test - env: - TOXPYTHON: ${{ matrix.python }} - run: > - tox -e ${{ matrix.python }} -v + diff --git a/ci/requirements.txt b/ci/requirements.txt index 7b876e8..30fdd1c 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -3,4 +3,4 @@ pip>=19.1.1 setuptools>=18.0.1 six>=1.14.0 tox -virtualenv-pyenv \ No newline at end of file +tox-gh>=1.2 \ No newline at end of file From 3e8ccb97d253a5cad41bc819452bf785929ac852 Mon Sep 17 00:00:00 2001 From: Emmanuel Decitre Date: Sat, 3 Feb 2024 16:39:21 +0100 Subject: [PATCH 04/13] iterate over test.yml --- .github/workflows/test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d4ae66..0644441 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] python: ['3.8', '3.9', '3.10', '3.11', '3.12'] # ['21.12', '23.x'] ?? - protoc: ['3.20.3', '3.21.12', '25.2'] + protoc: ['25.x'] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: @@ -42,8 +42,6 @@ jobs: pip list --format=freeze protoc --version - name: test - env: - TOXPYTHON: ${{ matrix.python }} run: > - tox -e ${{ matrix.python }} -v + tox -v From 41d29334cca8407a2d2bf5a6cd8c33fb027f8d04 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:25:51 +0100 Subject: [PATCH 05/13] Iterate over python-packages.yml, test.yml (reduce matrix size) --- .github/workflows/python-package.yml | 5 +++-- .github/workflows/test.yml | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 66a4c59..2cbfe51 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -10,13 +10,14 @@ on: branches: [ master ] jobs: - build: + build_25_2: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + #, "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0644441..e657955 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,10 @@ jobs: fail-fast: false matrix: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs - os: [ubuntu-latest, macos-latest, windows-latest] - python: ['3.8', '3.9', '3.10', '3.11', '3.12'] + # , macos-latest, windows-latest] + os: [ubuntu-latest] + # , '3.9', '3.10', '3.11', '3.12'] + python: ['3.8'] # ['21.12', '23.x'] ?? protoc: ['25.x'] runs-on: ${{ matrix.os }} From 04d0c675fc260d8f0ede7cbfa4cf0ac3906fc472 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:48:29 +0100 Subject: [PATCH 06/13] Iterate over python-packages.yml, test.yml and tox.ini to get tox 4 and tox-gh to run --- .github/workflows/python-package.yml | 9 +++++---- .github/workflows/test.yml | 8 ++++---- tox.ini | 9 ++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2cbfe51..22ac501 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,10 +4,11 @@ name: Python package on: - push: - branches: [ master ] - pull_request: - branches: [ master ] + #push: + # branches: [ master ] + #pull_request: + # branches: [ master ] + workflow_dispatch: jobs: build_25_2: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e657955..22b442e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,12 +38,12 @@ jobs: - name: install dependencies run: | python -mpip install --progress-bar=off -r ci/requirements.txt - virtualenv --version pip --version tox --version pip list --format=freeze protoc --version - - name: test - run: > - tox -v + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: tox --skip-pkg-install diff --git a/tox.ini b/tox.ini index 3ba0590..8cc4b7e 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,14 @@ envlist = report ignore_basepython_conflict = true +[gh] +python = + 3.12 = py312 + 3.11 = py311, type + 3.10 = py310 + 3.9 = py39 + 3.8 = py38 + [testenv] basepython = 3.8: {env:TOXPYTHON:python3.8} @@ -28,7 +36,6 @@ basepython = setenv = PYTHONPATH={toxinidir}/tests PYTHONUNBUFFERED=yes - VIRTUALENV_DISCOVERY = pyenv passenv = * usedevelop = false From 4aa2d2b98b2886a02b03f6c07c425cd2c3422b47 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 14:00:20 +0100 Subject: [PATCH 07/13] Swap to 3.11 and ad a tox -e check --- .github/workflows/test.yml | 12 +++++++----- tox.ini | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22b442e..c9622d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,9 @@ name: test on: push: - branches: [ master ] pull_request: - branches: [ master ] + schedule: + - cron: "0 8 * * *" jobs: build: @@ -13,8 +13,8 @@ jobs: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs # , macos-latest, windows-latest] os: [ubuntu-latest] - # , '3.9', '3.10', '3.11', '3.12'] - python: ['3.8'] + # , '3.9', '3.10', '3.8', '3.12'] + python: ['3.11'] # ['21.12', '23.x'] ?? protoc: ['25.x'] runs-on: ${{ matrix.os }} @@ -45,5 +45,7 @@ jobs: - name: Setup test suite run: tox -vv --notest - name: Run test suite - run: tox --skip-pkg-install + run: | + tox -e check + tox --skip-pkg-install diff --git a/tox.ini b/tox.ini index 8cc4b7e..3a04dcb 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ ignore_basepython_conflict = true [gh] python = 3.12 = py312 - 3.11 = py311, type + 3.11 = py311 3.10 = py310 3.9 = py39 3.8 = py38 From 5ded0e82c3eafa34940d93e88f885a67e3d228e0 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 14:33:10 +0100 Subject: [PATCH 08/13] remove python-package.yml (to keep a tox driven test workflow) reduce test configs to two sets of matrices --- .github/workflows/python-package.yml | 46 ------------ .github/workflows/test.yml | 105 ++++++++++++++++++--------- 2 files changed, 69 insertions(+), 82 deletions(-) delete mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 22ac501..0000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,46 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: - #push: - # branches: [ master ] - #pull_request: - # branches: [ master ] - workflow_dispatch: - -jobs: - build_25_2: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - #, "3.9", "3.10", "3.11", "3.12"] - python-version: ["3.8"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - version: '25.2' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements_test.txt -r requirements.txt -e . - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9622d6..be69077 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,46 +6,79 @@ on: - cron: "0 8 * * *" jobs: - build: + build_py39_proto203: strategy: fail-fast: false matrix: - # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs - # , macos-latest, windows-latest] - os: [ubuntu-latest] - # , '3.9', '3.10', '3.8', '3.12'] - python: ['3.11'] - # ['21.12', '23.x'] ?? - protoc: ['25.x'] + os: [ ubuntu-latest, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-python@v5 - # https://github.com/actions/setup-python - with: - python-version: ${{ matrix.python }} - architecture: x64 - cache: pip - - name: Install Protoc - uses: arduino/setup-protoc@v3 - # https://github.com/arduino/setup-protoc - with: - version: ${{ matrix.protoc }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: install dependencies - run: | - python -mpip install --progress-bar=off -r ci/requirements.txt - pip --version - tox --version - pip list --format=freeze - protoc --version - - name: Setup test suite - run: tox -vv --notest - - name: Run test suite - run: | - tox -e check - tox --skip-pkg-install + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + architecture: x64 + cache: pip + - name: Install Protoc + uses: arduino/setup-protoc@v3 + with: + version: 20.3 + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: install dependencies + run: | + pip install --upgrade pip + pip install --progress-bar=off -r ci/requirements.txt + pip --version + tox --version + protoc --version + pip freeze + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: | + tox -e check + tox --skip-pkg-install + build_py31x_proto25x: + strategy: + fail-fast: false + matrix: + # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs + os: [ubuntu-latest, macos-latest, windows-latest] + python: [ '3.10', '3.11', '3.12' ] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + # https://github.com/actions/checkout + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + # https://github.com/actions/setup-python + with: + python-version: ${{ matrix.python }} + architecture: x64 + cache: pip + - name: Install Protoc + uses: arduino/setup-protoc@v3 + # https://github.com/arduino/setup-protoc + with: + version: 25.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: install dependencies + run: | + pip install --upgrade pip + pip install --progress-bar=off -r ci/requirements.txt + pip --version + tox --version + protoc --version + pip freeze + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: | + tox -e check --skip-pkg-install + tox --skip-pkg-install From 27bb1d873efb0a4a9b7afe94d81e8ba6eebe1eb3 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 15:06:30 +0100 Subject: [PATCH 09/13] Fix a protoc version (3.20.3) Add a "latest" build Add py 3.12 support --- .github/workflows/test.yml | 48 +++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- setup.py | 1 + tox.ini | 2 +- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be69077..ba0df2b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: - cron: "0 8 * * *" jobs: - build_py39_proto203: + old_config: strategy: fail-fast: false matrix: @@ -25,7 +25,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v3 with: - version: 20.3 + version: 3.20.3 repo-token: ${{ secrets.GITHUB_TOKEN }} - name: install dependencies run: | @@ -39,16 +39,16 @@ jobs: run: tox -vv --notest - name: Run test suite run: | - tox -e check + tox -e check --skip-pkg-install tox --skip-pkg-install - build_py31x_proto25x: + newer: strategy: fail-fast: false matrix: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs os: [ubuntu-latest, macos-latest, windows-latest] - python: [ '3.10', '3.11', '3.12' ] + python: [ '3.10', '3.11'] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: @@ -66,7 +66,7 @@ jobs: uses: arduino/setup-protoc@v3 # https://github.com/arduino/setup-protoc with: - version: 25.x + version: 25.2 repo-token: ${{ secrets.GITHUB_TOKEN }} - name: install dependencies run: | @@ -82,3 +82,39 @@ jobs: run: | tox -e check --skip-pkg-install tox --skip-pkg-install + + latest: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + architecture: x64 + cache: pip + - name: Install Protoc + uses: arduino/setup-protoc@v3 + with: + version: 25.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: install dependencies + run: | + pip install --upgrade pip + pip install --progress-bar=off -r ci/requirements.txt + pip --version + tox --version + protoc --version + pip freeze + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: | + tox -e check --skip-pkg-install + tox --skip-pkg-install \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d5293e7..1611528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 88 -target-version = ['py310'] +target-version = ['py311'] skip-string-normalization = true [tool.bumpver] diff --git a/setup.py b/setup.py index 24fb341..01ff82e 100755 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ def read(*names, **kwargs): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", 'Programming Language :: Python :: Implementation :: CPython', "Operating System :: OS Independent", ], diff --git a/tox.ini b/tox.ini index 3a04dcb..1a8ddac 100644 --- a/tox.ini +++ b/tox.ini @@ -46,7 +46,7 @@ deps = pytest-recording urllib3<2 commands = - {posargs:pytest --cov --cov-report=term-missing -vv tests} + {posargs:pytest --cov=src --cov-report=xml --cov-report=term-missing -vv tests} [testenv:check] deps = From 547f840c5f42dd17e19c806f04667c04bb7cf599 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 15:15:53 +0100 Subject: [PATCH 10/13] Use setup_protoc@v1.1.2 to use proto 3.20.3 --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba0df2b..ed2157d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: - cron: "0 8 * * *" jobs: - old_config: + py_39_proto_203: strategy: fail-fast: false matrix: @@ -23,7 +23,7 @@ jobs: architecture: x64 cache: pip - name: Install Protoc - uses: arduino/setup-protoc@v3 + uses: arduino/setup-protoc@v1.1.2 with: version: 3.20.3 repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -42,7 +42,7 @@ jobs: tox -e check --skip-pkg-install tox --skip-pkg-install - newer: + py_31x_proto_252: strategy: fail-fast: false matrix: @@ -83,7 +83,7 @@ jobs: tox -e check --skip-pkg-install tox --skip-pkg-install - latest: + py_3x_proto_25x: strategy: fail-fast: false matrix: From fe2007a57a8ea760ee0705aba58a1f50707f14d2 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 16:44:56 +0100 Subject: [PATCH 11/13] Remove ci directory Centralize requirements in a dedicated directory Add google address book example to README --- .github/workflows/test.yml | 6 +- MANIFEST.in | 6 +- README.md | 50 ++++++---- ci/bootstrap.py | 93 ------------------- .../.github/workflows/github-actions.yml | 65 ------------- requirements.txt | 1 - requirements/build.txt | 1 + ci/requirements.txt => requirements/ci.txt | 0 requirements_test.txt => requirements/dev.txt | 3 +- setup.py | 6 +- tox.ini | 2 +- 11 files changed, 43 insertions(+), 190 deletions(-) delete mode 100755 ci/bootstrap.py delete mode 100644 ci/templates/.github/workflows/github-actions.yml delete mode 100644 requirements.txt create mode 100644 requirements/build.txt rename ci/requirements.txt => requirements/ci.txt (100%) rename requirements_test.txt => requirements/dev.txt (74%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed2157d..cda6800 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: - name: install dependencies run: | pip install --upgrade pip - pip install --progress-bar=off -r ci/requirements.txt + pip install --progress-bar=off -r requirements/ci.txt pip --version tox --version protoc --version @@ -71,7 +71,7 @@ jobs: - name: install dependencies run: | pip install --upgrade pip - pip install --progress-bar=off -r ci/requirements.txt + pip install --progress-bar=off -r requirements/ci.txt pip --version tox --version protoc --version @@ -107,7 +107,7 @@ jobs: - name: install dependencies run: | pip install --upgrade pip - pip install --progress-bar=off -r ci/requirements.txt + pip install --progress-bar=off -r requirements/ci.txt pip --version tox --version protoc --version diff --git a/MANIFEST.in b/MANIFEST.in index 96db5da..e09d022 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,16 +1,14 @@ graft src -graft ci graft tests include .bumpversion.cfg include .coveragerc include tox.ini -include .github/workflows/github-actions.yml +include .github/workflows/test.yml include .pre-commit-config.yaml include LICENSE include README.md -include requirements.txt -include requirements_test.txt +recursive-include requirements *.txt global-exclude *.py[cod] __pycache__/* *.so *.dylib diff --git a/README.md b/README.md index e853151..0adae3c 100644 --- a/README.md +++ b/README.md @@ -15,35 +15,47 @@ It is useful for Python programs needing to parse protobuf messages without havi pip install proto-topy -## Usage example +## Google address book + +Adaptation of the [example](https://github.com/protocolbuffers/protobuf/tree/main/examples): ```python -import sys, os -from pathlib import Path +import requests +import sys from shutil import which from proto_topy.entities import ProtoModule -from google.protobuf.timestamp_pb2 import Timestamp - -protoc_path = Path(which("protoc") or os.environ.get('PROTOC')) - -source = """ +from pathlib import Path - syntax = "proto3"; - import "google/protobuf/timestamp.proto"; - - message Test5 { - google.protobuf.Timestamp created = 1; - } +# Retrieve protobuf messages definitions +example_source = requests.get( + "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto").text +example_path = Path( + "protocolbuffers/protobuf/blob/main/examples/addressbook.proto") -""" +# Compile and import +module = ProtoModule(file_path=example_path, source=example_source).compiled(Path(which("protoc"))) +sys.modules["addressbook"] = module.py -proto = ProtoModule(file_path=Path("test5.proto"), source=source).compiled(protoc_path) -sys.modules["test5"] = proto.py +# Produce a serialized address book +address_book = module.py.AddressBook() +person = address_book.people.add() +person.id = 111 +person.name = "A Name" +person.email = "a.name@mail.com" +phone_number = person.phones.add() +phone_number.number = "+1234567" +phone_number.type = module.py.Person.MOBILE +with open("address_book.data", "wb") as o: + o.write(address_book.SerializeToString()) -assert isinstance(proto.py.Test5().created, Timestamp) +# Use a serialized address book +address_book = module.py.AddressBook() +with open("address_book.data", "rb") as i: + address_book.ParseFromString(i.read()) + for person in address_book.people: + print(person.id, person.name, person.email, phone_number.number) ``` -More examples in [test_proto_topy.py][tests]. [pypi]: https://pypi.org/project/proto-topy [test_badge]: https://github.com/decitre/python-proto-topy/actions/workflows/test.yml/badge.svg diff --git a/ci/bootstrap.py b/ci/bootstrap.py deleted file mode 100755 index 3ca06b7..0000000 --- a/ci/bootstrap.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -import os -import subprocess -import sys -from os.path import abspath -from os.path import dirname -from os.path import exists -from os.path import join -from os.path import relpath - -base_path = dirname(dirname(abspath(__file__))) -templates_path = join(base_path, "ci", "templates") - - -def check_call(args): - print("+", *args) - subprocess.check_call(args) - - -def exec_in_env(): - env_path = join(base_path, ".tox", "bootstrap") - if sys.platform == "win32": - bin_path = join(env_path, "Scripts") - else: - bin_path = join(env_path, "bin") - if not exists(env_path): - import subprocess - - print("Making bootstrap env in: {0} ...".format(env_path)) - try: - check_call([sys.executable, "-m", "venv", env_path]) - except subprocess.CalledProcessError: - try: - check_call([sys.executable, "-m", "virtualenv", env_path]) - except subprocess.CalledProcessError: - check_call(["virtualenv", env_path]) - print("Installing `jinja2` into bootstrap environment...") - check_call([join(bin_path, "pip"), "install", "jinja2", "tox"]) - python_executable = join(bin_path, "python") - if not os.path.exists(python_executable): - python_executable += '.exe' - - print("Re-executing with: {0}".format(python_executable)) - print("+ exec", python_executable, __file__, "--no-env") - os.execv(python_executable, [python_executable, __file__, "--no-env"]) - - -def main(): - import jinja2 - - print("Project path: {0}".format(base_path)) - - jinja = jinja2.Environment( - loader=jinja2.FileSystemLoader(templates_path), - trim_blocks=True, - lstrip_blocks=True, - keep_trailing_newline=True, - ) - - tox_environments = [ - line.strip() - # 'tox' need not be installed globally, but must be importable - # by the Python that is running this script. - # This uses sys.executable the same way that the call in - # cookiecutter-pylibrary/hooks/post_gen_project.py - # invokes this bootstrap.py itself. - for line in subprocess.check_output([sys.executable, '-m', 'tox', '--listenvs'], universal_newlines=True).splitlines() - ] - tox_environments = [line for line in tox_environments if line.startswith('py')] - - for root, _, files in os.walk(templates_path): - for name in files: - relative = relpath(root, templates_path) - with open(join(base_path, relative, name), "w") as fh: - fh.write(jinja.get_template(join(relative, name)).render(tox_environments=tox_environments)) - print("Wrote {}".format(name)) - print("DONE.") - - -if __name__ == "__main__": - args = sys.argv[1:] - if args == ["--no-env"]: - main() - elif not args: - exec_in_env() - else: - print("Unexpected arguments {0}".format(args), file=sys.stderr) - sys.exit(1) diff --git a/ci/templates/.github/workflows/github-actions.yml b/ci/templates/.github/workflows/github-actions.yml deleted file mode 100644 index 7ee6426..0000000 --- a/ci/templates/.github/workflows/github-actions.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: build -on: [push, pull_request] -jobs: - test: - name: {{ '${{ matrix.name }}' }} - runs-on: {{ '${{ matrix.os }}' }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - name: 'check' - python: '3.9' - toxpython: 'python3.9' - tox_env: 'check' - os: 'ubuntu-latest' - - name: 'docs' - python: '3.9' - toxpython: 'python3.9' - tox_env: 'docs' - os: 'ubuntu-latest' -{% for env in tox_environments %} -{% set prefix = env.split('-')[0] -%} -{% if prefix.startswith('pypy') %} -{% set python %}pypy-{{ prefix[4] }}.{{ prefix[5] }}{% endset %} -{% set cpython %}pp{{ prefix[4:5] }}{% endset %} -{% set toxpython %}pypy{{ prefix[4] }}.{{ prefix[5] }}{% endset %} -{% else %} -{% set python %}{{ prefix[2] }}.{{ prefix[3:] }}{% endset %} -{% set cpython %}cp{{ prefix[2:] }}{% endset %} -{% set toxpython %}python{{ prefix[2] }}.{{ prefix[3:] }}{% endset %} -{% endif %} -{% for os, python_arch in [ - ['ubuntu', 'x64'], - ['windows', 'x64'], - ['macos', 'x64'], -] %} - - name: '{{ env }} ({{ os }})' - python: '{{ python }}' - toxpython: '{{ toxpython }}' - python_arch: '{{ python_arch }}' - tox_env: '{{ env }}{% if 'cover' in env %},codecov{% endif %}' - os: '{{ os }}-latest' -{% endfor %} -{% endfor %} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: actions/setup-python@v2 - with: - python-version: {{ '${{ matrix.python }}' }} - architecture: {{ '${{ matrix.python_arch }}' }} - - name: install dependencies - run: | - python -mpip install --progress-bar=off -r ci/requirements.txt - virtualenv --version - pip --version - tox --version - pip list --format=freeze - - name: test - env: - TOXPYTHON: '{{ '${{ matrix.toxpython }}' }}' - run: > - tox -e {{ '${{ matrix.tox_env }}' }} -v diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d25ebc1..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -protobuf>=3.20.0 \ No newline at end of file diff --git a/requirements/build.txt b/requirements/build.txt new file mode 100644 index 0000000..9aa2177 --- /dev/null +++ b/requirements/build.txt @@ -0,0 +1 @@ +protobuf>=3.20.3 \ No newline at end of file diff --git a/ci/requirements.txt b/requirements/ci.txt similarity index 100% rename from ci/requirements.txt rename to requirements/ci.txt diff --git a/requirements_test.txt b/requirements/dev.txt similarity index 74% rename from requirements_test.txt rename to requirements/dev.txt index 039b521..225503b 100644 --- a/requirements_test.txt +++ b/requirements/dev.txt @@ -3,4 +3,5 @@ protobuf pytest pytest-recording requests -urllib3<2 \ No newline at end of file +urllib3<2 +bumpver \ No newline at end of file diff --git a/setup.py b/setup.py index 01ff82e..c6eb09e 100755 --- a/setup.py +++ b/setup.py @@ -67,8 +67,8 @@ def read(*names, **kwargs): _packages = find_packages(where=_packages_path) version = get_property("__version__", _packages_path, _packages).pop() requirements = ["click"] - requirements.extend(get_requirements("requirements.txt", no_precise_version=False)) - requirements_test = get_requirements("requirements_test.txt") + requirements.extend(get_requirements("requirements/build.txt", no_precise_version=False)) + requirements_dev = get_requirements("requirements/dev.txt") setup( name=project_name, @@ -103,5 +103,5 @@ def read(*names, **kwargs): keywords=['protobuf'], python_requires='>=3.8', install_requires=requirements, - extras_require={"dev": requirements_test}, + extras_require={"dev": requirements_dev}, ) diff --git a/tox.ini b/tox.ini index 1a8ddac..3a04dcb 100644 --- a/tox.ini +++ b/tox.ini @@ -46,7 +46,7 @@ deps = pytest-recording urllib3<2 commands = - {posargs:pytest --cov=src --cov-report=xml --cov-report=term-missing -vv tests} + {posargs:pytest --cov --cov-report=term-missing -vv tests} [testenv:check] deps = From 36ae987b9850917c7d95f741076934d3c8c8a689 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 16:44:56 +0100 Subject: [PATCH 12/13] Remove ci directory Centralize requirements in a dedicated directory Add google address book example to README --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0adae3c..53987dd 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ It is useful for Python programs needing to parse protobuf messages without havi pip install proto-topy -## Google address book +## Example: address book -Adaptation of the [example](https://github.com/protocolbuffers/protobuf/tree/main/examples): +Adaptation of the `protocolbuffers` [example](https://github.com/protocolbuffers/protobuf/tree/main/examples): ```python import requests @@ -28,12 +28,15 @@ from pathlib import Path # Retrieve protobuf messages definitions example_source = requests.get( - "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/examples/addressbook.proto").text + "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/" + "examples/addressbook.proto").text + example_path = Path( "protocolbuffers/protobuf/blob/main/examples/addressbook.proto") # Compile and import -module = ProtoModule(file_path=example_path, source=example_source).compiled(Path(which("protoc"))) +module = (ProtoModule(file_path=example_path, source=example_source) + .compiled(Path(which("protoc")))) sys.modules["addressbook"] = module.py # Produce a serialized address book From 561fb8d1b92952beec30e539e9a55177c4e7b0a2 Mon Sep 17 00:00:00 2001 From: decitre <590094+decitre@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:56:35 +0100 Subject: [PATCH 13/13] bump version 0.1.0 -> 0.2.0 --- pyproject.toml | 2 +- src/proto_topy/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1611528..2cd017b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ target-version = ['py311'] skip-string-normalization = true [tool.bumpver] -current_version = "0.1.0" +current_version = "0.2.0" version_pattern = "MAJOR.MINOR.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/proto_topy/__init__.py b/src/proto_topy/__init__.py index 3dc1f76..d3ec452 100644 --- a/src/proto_topy/__init__.py +++ b/src/proto_topy/__init__.py @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.2.0"