test publish process

apps/odk_publish/admin.py

@@ -11,9 +11,6 @@
     AppUserFormVersion,
     TemplateVariable,
 )
-from django.contrib import messages
-from django.db.models import QuerySet
-from django.utils.translation import ngettext
 
 
 logger = structlog.getLogger(__name__)
@@ -48,35 +45,6 @@ class FormTemplateAdmin(admin.ModelAdmin):
     list_filter = ("created_at", "modified_at")
     ordering = ("form_id_base",)
 
-    actions = ("create_next_version",)
-
-    @admin.action(description="Create next version")
-    def create_next_version(self, request, queryset: QuerySet[FormTemplate]):
-        """Attempt to create the next version of the selected form templates."""
-        versions_created = 0
-        for form_template in queryset:
-            try:
-                form_template.create_next_version(user=request.user)
-                versions_created += 1
-            except Exception as e:
-                logger.exception("Error creating next version", form_template=form_template)
-                self.message_user(
-                    request,
-                    f"Error creating next version for {form_template}: {e}",
-                    messages.ERROR,
-                )
-        if versions_created:
-            self.message_user(
-                request,
-                ngettext(
-                    "%d form template version was created.",
-                    "%d form template versions were created.",
-                    queryset.count(),
-                )
-                % queryset.count(),
-                messages.SUCCESS,
-            )
-
 
 @admin.register(FormTemplateVersion)
 class FormTemplateVersionAdmin(admin.ModelAdmin):

apps/odk_publish/consumers.py

@@ -1,35 +1,17 @@
 import json
-from requests import Response
-import traceback
 import pprint
+import traceback
 
 import structlog
 from channels.generic.websocket import WebsocketConsumer
 from django.template.loader import render_to_string
-from django.db import transaction
-from pydantic import BaseModel, field_validator
+from requests import Response
 
-from .etl.odk.client import ODKPublishClient
-from .models import FormTemplate, FormTemplateVersion
+from .etl.load import PublishTemplateEvent, publish_form_template
 
 logger = structlog.getLogger(__name__)
 
 
-class PublishTemplateEvent(BaseModel):
-    """Model to parse and validate the publish WebSocket message payload."""
-
-    form_template: int
-    app_users: list[str]
-
-    @field_validator("app_users", mode="before")
-    @classmethod
-    def split_comma_separated_app_users(cls, v):
-        """Split comma-separated app users into a list."""
-        if isinstance(v, str):
-            return v.split(",")
-        return v
-
-
 class PublishTemplateConsumer(WebsocketConsumer):
     """Websocket consumer for publishing form templates to ODK Central."""
 
@@ -69,42 +51,9 @@ def receive(self, text_data):
 
     def publish_form_template(self, event_data: dict):
         """Publish a form template to ODK Central and stream progress to the browser."""
-        user = self.scope["user"]
         # Parse the event data and raise an error if it's invalid
         publish_event = PublishTemplateEvent(**event_data)
-        self.send_message(f"New {repr(publish_event)}")
-        # Get the form template
-        form_template = FormTemplate.objects.select_related().get(id=publish_event.form_template)
-        self.send_message(f"Publishing next version of {repr(form_template)}")
-        # Get the next version by querying ODK Central
-        client = ODKPublishClient(
-            base_url=form_template.project.central_server.base_url,
-            project_id=form_template.project.central_id,
-        )
-        version = client.odk_publish.get_unique_version_by_form_id(
-            xml_form_id_base=form_template.form_id_base
+        # Hand off to the ETL process to publish the form template
+        publish_form_template(
+            event=publish_event, user=self.scope["user"], send_message=self.send_message
         )
-        self.send_message(f"Generated version: {version}")
-        # Download the template from Google Sheets
-        file = form_template.download_google_sheet(
-            user=user, name=f"{form_template.form_id_base}-{version}.xlsx"
-        )
-        self.send_message(f"Downloaded template: {file}")
-        with transaction.atomic():
-            # Create the next version
-            template_version = FormTemplateVersion.objects.create(
-                form_template=form_template, user=user, file=file, version=version
-            )
-            # Create a version for each app user
-            app_users = form_template.project.app_users.filter(name__in=publish_event.app_users)
-            app_user_versions = template_version.create_app_user_versions(
-                app_users=app_users, send_message=self.send_message
-            )
-            # Publish each app user version to ODK Central
-            for app_user_version in app_user_versions:
-                form = client.odk_publish.create_or_update_form(
-                    xml_form_id=app_user_version.app_user_form_template.xml_form_id,
-                    definition=app_user_version.file.read(),
-                )
-                self.send_message(f"Published form: {form.xmlFormId}")
-        self.send_message(f"Successfully published {version}", complete=True)

apps/odk_publish/etl/load.py

@@ -1,38 +1,86 @@
+from typing import Callable
+
 import structlog
 from django.core.files.uploadedfile import SimpleUploadedFile
+from django.db import transaction
 from django.db.models import QuerySet
+from pydantic import BaseModel, field_validator
+
+from apps.users.models import User
 
-from ..models import AppUser, AppUserFormTemplate, CentralServer, FormTemplate, Project
+from ..models import (
+    AppUser,
+    CentralServer,
+    FormTemplate,
+    FormTemplateVersion,
+    Project,
+)
 from .odk.client import ODKPublishClient
 from .odk.qrcode import create_app_user_qrcode
 
 logger = structlog.getLogger(__name__)
 
 
-def create_or_update_app_users(form_template: FormTemplate):
-    """Create or update app users for the form template, effectively syncing
-    ODK Publish to ODK Central.
+class PublishTemplateEvent(BaseModel):
+    """Model to parse and validate the publish WebSocket message payload."""
+
+    form_template: int
+    app_users: list[str]
+
+    @field_validator("app_users", mode="before")
+    @classmethod
+    def split_comma_separated_app_users(cls, v):
+        """Split comma-separated app users into a list."""
+        if isinstance(v, str):
+            return v.split(",")
+        return v
+
+
+def publish_form_template(event: PublishTemplateEvent, user: User, send_message: Callable):
+    """The main function for publishing a form template to ODK Central.
 
     Steps include:
+    * Download the form template from Google Sheets
+    * Create the next version of the form template and app user versions
     * Get or create app users in ODK Central
-    * Create or update form assignments in ODK Central
+    * Publish each app user version to ODK Central
     """
-    app_user_forms: QuerySet[AppUserFormTemplate] = form_template.app_user_forms.select_related()
-
-    with ODKPublishClient.new_client(
-        base_url=form_template.project.central_server.base_url
-    ) as client:
-        app_users = client.odk_publish.get_or_create_app_users(
-            display_names=[app_user_form.app_user.name for app_user_form in app_user_forms],
-            project_id=form_template.project.central_id,
+    send_message(f"New {repr(event)}")
+    # Get the form template
+    form_template = FormTemplate.objects.select_related().get(id=event.form_template)
+    send_message(f"Publishing next version of {repr(form_template)}")
+    # Get the next version by querying ODK Central
+    client = ODKPublishClient(
+        base_url=form_template.project.central_server.base_url,
+        project_id=form_template.project.central_id,
+    )
+    version = client.odk_publish.get_unique_version_by_form_id(
+        xml_form_id_base=form_template.form_id_base
+    )
+    send_message(f"Generated version: {version}")
+    # Download the template from Google Sheets
+    file = form_template.download_user_google_sheet(
+        user=user, name=f"{form_template.form_id_base}-{version}.xlsx"
+    )
+    send_message(f"Downloaded template: {file}")
+    with transaction.atomic():
+        # Create the next version
+        template_version = FormTemplateVersion.objects.create(
+            form_template=form_template, user=user, file=file, version=version
         )
-        # Link form assignments to app users locally
-        for app_user_form in app_user_forms:
-            app_users[app_user_form.app_user.name].xml_form_ids.append(app_user_form.xml_form_id)
-        # Create or update the form assignments on the server
-        client.odk_publish.assign_forms(
-            app_users=app_users.values(), project_id=form_template.project.central_id
+        # Create a version for each app user
+        app_users = form_template.project.app_users.filter(name__in=event.app_users)
+        app_user_versions = template_version.create_app_user_versions(
+            app_users=app_users, send_message=send_message
         )
+        # Publish each app user version to ODK Central
+        for app_user_version in app_user_versions:
+            form = client.odk_publish.create_or_update_form(
+                xml_form_id=app_user_version.app_user_form_template.xml_form_id,
+                definition=app_user_version.file.read(),
+            )
+            send_message(f"Published form: {form.xmlFormId}")
+    send_message(f"Successfully published {version}", complete=True)
 
 
 def generate_and_save_app_user_collect_qrcodes(project: Project):

apps/odk_publish/models.py

@@ -10,7 +10,6 @@
 
 from .etl import template
 from .etl.google import download_user_google_sheet
-from .etl.odk.client import ODKPublishClient
 
 logger = structlog.getLogger(__name__)
 
@@ -91,7 +90,7 @@ class FormTemplate(AbstractBaseModel):
     def __str__(self):
         return f"{self.form_id_base} ({self.id})"
 
-    def download_google_sheet(self, user: User, name: str) -> SimpleUploadedFile:
+    def download_user_google_sheet(self, user: User, name: str) -> SimpleUploadedFile:
         """Download the Google Sheet Excel file for this form template."""
         social_token = user.get_google_social_token()
         if social_token is None:
@@ -103,35 +102,6 @@ def download_google_sheet(self, user: User, name: str) -> SimpleUploadedFile:
             name=name,
         )
 
-    def create_next_version(self, user: User) -> "FormTemplateVersion":
-        """Create the next version of this form template.
-
-        Steps to create the next version:
-
-        1. Query the ODK Central server for this `form_id_base` and increment
-           the version number with today's date.
-        2. Download the Google Sheet Excel file for this form template.
-        3. Create a new FormTemplateVersion instance with the downloaded file.
-        """
-        with ODKPublishClient(
-            base_url=self.project.central_server.base_url, project_id=self.project.central_id
-        ) as client:
-            version = client.odk_publish.get_unique_version_by_form_id(
-                xml_form_id_base=self.form_id_base
-            )
-            name = f"{self.form_id_base}-{version}.xlsx"
-            file = self.download_google_sheet(user=user, name=name)
-            version = FormTemplateVersion.objects.create(
-                form_template=self, user=user, file=file, version=version
-            )
-            app_user_versions = version.create_app_user_versions()
-            for app_user_version in app_user_versions:
-                client.odk_publish.create_or_update_form(
-                    xml_form_id=app_user_version.app_user_form_template.xml_form_id,
-                    definition=app_user_version.file.read(),
-                )
-            return version
-
 
 class FormTemplateVersion(AbstractBaseModel):
     """A version (like v5) of a form template."""
@@ -175,6 +145,8 @@ def create_app_user_versions(
 
 
 class AppUserTemplateVariable(AbstractBaseModel):
+    """A template variable value for an app user."""
+
     app_user = models.ForeignKey(
         "AppUser", on_delete=models.CASCADE, related_name="app_user_template_variables"
     )
@@ -195,6 +167,8 @@ def __str__(self):
 
 
 class AppUser(AbstractBaseModel):
+    """An app user in ODK Central."""
+
     name = models.CharField(max_length=255)
     central_id = models.PositiveIntegerField(
         verbose_name="app user ID", help_text="The ID of this app user in ODK Central."
@@ -264,6 +238,8 @@ def create_next_version(self, form_template_version: FormTemplateVersion):
 
 
 class AppUserFormVersion(AbstractBaseModel):
+    """A version of an app user's form template that is published to ODK Central."""
+
     app_user_form_template = models.ForeignKey(
         AppUserFormTemplate,
         on_delete=models.CASCADE,

pyproject.toml

@@ -42,7 +42,7 @@ profile = "django"
 custom_blocks="switch,partialdef"
 
 [tool.pytest.ini_options]
-addopts = "--ds=config.settings.test --cov=apps --cov-report=html"
+addopts = "--ds=config.settings.test --cov=apps --cov-report=html --reuse-db --no-migrations"
 testpaths = [ "tests" ]
 pythonpath = [ "." ]
 

tests/odk_publish/etl/extract/test_google.py

@@ -1,8 +1,12 @@
+import pytest
+
 from apps.odk_publish.etl.google import (
-    gspread_client,
-    export_sheet_by_url,
     download_user_google_sheet,
+    export_sheet_by_url,
+    gspread_client,
 )
+from tests.odk_publish.factories import FormTemplateFactory
+from tests.users.factories import SocialTokenFactory, UserFactory
 
 
 class TestDownloadUserGoogleSheet:
@@ -38,3 +42,25 @@ def test_download_user_google_sheet(self, mocker):
         )
         assert downloaded_file.name == "mysheet.xlsx"
         assert downloaded_file.read() == b"file content"
+
+    def test_form_template_download_user_google_sheet(self, mocker):
+        """Test the download of a user Google Sheet from a FormTemplate."""
+        user = UserFactory.build()
+        token = SocialTokenFactory.build(token="token", token_secret="token_secret")
+        form_template = FormTemplateFactory.build(form_id_base="staff_registration")
+        mocker.patch.object(user, "get_google_social_token", return_value=token)
+        mock_gspread_client = mocker.patch("apps.odk_publish.etl.google.gspread_client")
+        mock_gspread_client.return_value.open_by_url.return_value.export.return_value = (
+            b"file content"
+        )
+        downloaded_file = form_template.download_user_google_sheet(user=user, name="mysheet.xlsx")
+        assert downloaded_file.name == "mysheet.xlsx"
+        assert downloaded_file.read() == b"file content"
+
+    def test_form_template_download_user_google_sheet_no_token(self, mocker):
+        """Test error raised if user has no Google token."""
+        user = UserFactory.build()
+        form_template = FormTemplateFactory.build(form_id_base="staff_registration")
+        mocker.patch.object(user, "get_google_social_token", return_value=None)
+        with pytest.raises(ValueError):
+            form_template.download_user_google_sheet(user=user, name="mysheet.xlsx")

tests/odk_publish/etl/load/test_prepare_publish.py

@@ -2,6 +2,7 @@
 
 
 from apps.odk_publish.etl.odk.client import ODKPublishClient
+from apps.odk_publish.etl.load import PublishTemplateEvent
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 
@@ -28,6 +29,18 @@ def odk_client(project):
 # app_user__project=factory.SelfAttribute("..form_template.project"),
 
 
+class TestPublishEvent:
+    def test_split_comma_separated_app_users(self):
+        """Test that the app users are split into a list."""
+        event = PublishTemplateEvent(form_template=1, app_users="user1,user2")
+        assert event.app_users == ["user1", "user2"]
+
+    def test_split_comma_separated_app_users_single(self):
+        """Test that a single app user is split into a list."""
+        event = PublishTemplateEvent(form_template=1, app_users="user1")
+        assert event.app_users == ["user1"]
+
+
 class TestCreateVersions:
     """Test the creation of form template versions before publishing to ODK Central."""