replace the use of protobuf GetMessages with descriptor_pool and … (

#3) * replace the use of protobuf `GetMessages` with `descriptor_pool` and `GetMessageClassesForFiles` add support for Python 3.12 add protoc matrix to actions * remove python-package.yml (to keep a tox driven test workflow) reduce test configs to three sets of matrices * Use [email protected] to use proto 3.20.3 * Remove ci directory Centralize requirements in a dedicated directory Add google address book example to README * bump version 0.1.0 -> 0.2.0 --------- Co-authored-by: Emmanuel Decitre <[email protected]>
decitre · Feb 4, 2024 · 53d03a5 · 53d03a5
1 parent c752a05
commit 53d03a5
Show file tree

Hide file tree

Showing 19 changed files with 312 additions and 305 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,88 +1,120 @@
 name: test
 on:
   push:
-    branches: [ master ]
   pull_request:
-    branches: [ master ]
+  schedule:
+    - cron: "0 8 * * *"
 
 jobs:
-  buildv1:
+  py_39_proto_203:
     strategy:
       fail-fast: false
       matrix:
-        # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python: ['3.8', '3.9', '3.10', '3.11']
-        protoc: ['3.20.3']
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@v4
-      # https://github.com/actions/setup-python
-      with:
-        python-version: ${{ matrix.python }}
-        architecture: x64
-        cache: pip
-    - name: Install Protoc
-      uses: arduino/setup-protoc@v1
-      # https://github.com/arduino/setup-protoc
-      with:
-        version: ${{ matrix.protoc }}
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: install dependencies
-      run: |
-        python -mpip install --progress-bar=off -r ci/requirements.txt
-        virtualenv --version
-        pip --version
-        tox --version
-        pip list --format=freeze
-        protoc --version
-    - name: test
-      env:
-        TOXPYTHON: ${{ matrix.python }}
-      run: >
-        tox -e ${{ matrix.python }} -v
-  buildv2:
-    # repetition due to lack of support in `uses:` of context/matrix variables
-    # https://github.com/orgs/community/discussions/9049
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+          architecture: x64
+          cache: pip
+      - name: Install Protoc
+        uses: arduino/[email protected]
+        with:
+          version: 3.20.3
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --progress-bar=off -r requirements/ci.txt
+          pip --version
+          tox --version
+          protoc --version
+          pip freeze
+      - name: Setup test suite
+        run: tox -vv --notest
+      - name: Run test suite
+        run: |
+          tox -e check --skip-pkg-install
+          tox --skip-pkg-install
+
+  py_31x_proto_252:
     strategy:
       fail-fast: false
       matrix:
         # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: ['3.8', '3.9', '3.10', '3.11']
-        protoc: ['21.12', '23.x']
+        python: [ '3.10', '3.11']
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        # https://github.com/actions/checkout
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         # https://github.com/actions/setup-python
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
           cache: pip
       - name: Install Protoc
-        uses: arduino/setup-protoc@v2
+        uses: arduino/setup-protoc@v3
         # https://github.com/arduino/setup-protoc
         with:
-          version: ${{ matrix.protoc }}
+          version: 25.2
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --progress-bar=off -r requirements/ci.txt
+          pip --version
+          tox --version
+          protoc --version
+          pip freeze
+      - name: Setup test suite
+        run: tox -vv --notest
+      - name: Run test suite
+        run: |
+          tox -e check --skip-pkg-install
+          tox --skip-pkg-install
+
+  py_3x_proto_25x:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+          architecture: x64
+          cache: pip
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: 25.x
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: install dependencies
         run: |
-          python -mpip install --progress-bar=off -r ci/requirements.txt
-          virtualenv --version
+          pip install --upgrade pip
+          pip install --progress-bar=off -r requirements/ci.txt
           pip --version
           tox --version
-          pip list --format=freeze
           protoc --version
-      - name: test
-        env:
-          TOXPYTHON: ${{ matrix.python }}
-        run: >
-          tox -e ${{ matrix.python }} -v
+          pip freeze
+      - name: Setup test suite
+        run: tox -vv --notest
+      - name: Run test suite
+        run: |
+          tox -e check --skip-pkg-install
+          tox --skip-pkg-install
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,16 +1,14 @@
 graft src
-graft ci
 graft tests
 
 include .bumpversion.cfg
 include .coveragerc
 include tox.ini
-include .github/workflows/github-actions.yml
+include .github/workflows/test.yml
 include .pre-commit-config.yaml
 include LICENSE
 include README.md
-include requirements.txt
-include requirements_test.txt
+recursive-include requirements *.txt
 
 
 global-exclude *.py[cod] __pycache__/* *.so *.dylib
diff --git a/README.md b/README.md
@@ -15,35 +15,50 @@ It is useful for Python programs needing to parse protobuf messages without havi
 
     pip install proto-topy
 
-## Usage example
+## Example: address book
+
+Adaptation of the `protocolbuffers` [example](https://github.com/protocolbuffers/protobuf/tree/main/examples):
 
 ```python
-import sys, os
-from pathlib import Path
-from distutils.spawn import find_executable
+import requests
+import sys
+from shutil import which
 from proto_topy.entities import ProtoModule
-from google.protobuf.timestamp_pb2 import Timestamp
-
-protoc_path = Path(find_executable("protoc") or os.environ.get('PROTOC'))
+from pathlib import Path
 
-source = """
+# Retrieve protobuf messages definitions
+example_source = requests.get(
+    "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/"
+    "examples/addressbook.proto").text
 
-    syntax = "proto3";
-    import "google/protobuf/timestamp.proto";
-    
-    message Test5 {
-        google.protobuf.Timestamp created = 1;
-    }
+example_path = Path(
+    "protocolbuffers/protobuf/blob/main/examples/addressbook.proto")
 
-"""
+# Compile and import
+module = (ProtoModule(file_path=example_path, source=example_source)
+          .compiled(Path(which("protoc"))))
+sys.modules["addressbook"] = module.py
 
-proto = ProtoModule(file_path=Path("test5.proto"), source=source).compiled(protoc_path)
-sys.modules["test5"] = proto.py
+# Produce a serialized address book
+address_book = module.py.AddressBook()
+person = address_book.people.add()
+person.id = 111
+person.name = "A Name"
+person.email = "[email protected]"
+phone_number = person.phones.add()
+phone_number.number = "+1234567"
+phone_number.type = module.py.Person.MOBILE
+with open("address_book.data", "wb") as o:
+    o.write(address_book.SerializeToString())
 
-assert isinstance(proto.py.Test5().created, Timestamp)
+# Use a serialized address book
+address_book = module.py.AddressBook()
+with open("address_book.data", "rb") as i:
+    address_book.ParseFromString(i.read())
+    for person in address_book.people:
+        print(person.id, person.name, person.email, phone_number.number)
 ```
 
-More examples in [test_proto_topy.py][tests].
 
 [pypi]: https://pypi.org/project/proto-topy
 [test_badge]: https://github.com/decitre/python-proto-topy/actions/workflows/test.yml/badge.svg