publish over websockets

Co-authored-by: Tobias McNulty <[email protected]>

apps/odk_publish/consumers.py

@@ -0,0 +1,81 @@
+import json
+import traceback
+
+import structlog
+from channels.generic.websocket import WebsocketConsumer
+from django.template.loader import render_to_string
+from django.db import transaction
+from pydantic import BaseModel, field_validator
+
+from .etl.odk.client import ODKPublishClient
+from .models import FormTemplate, FormTemplateVersion
+
+logger = structlog.getLogger(__name__)
+
+
+class PublishTemplateEvent(BaseModel):
+    form_template: int
+    app_users: list[str]
+
+    @field_validator("app_users", mode="before")
+    @classmethod
+    def split_comma_separated_app_users(cls, v):
+        if isinstance(v, str):
+            return v.split(",")
+        return v
+
+
+class PublishTemplateConsumer(WebsocketConsumer):
+    def connect(self):
+        logger.debug(f"New connection: {self.channel_layer}")
+        super().connect()
+
+    def send_message(self, message: str, error: bool = False):
+        if not error:
+            logger.debug(f"Sending message: {message}")
+        message = render_to_string(
+            "odk_publish/ws/message.html", {"message_text": message, "error": error}
+        )
+        self.send(text_data=message)
+
+    def receive(self, text_data):
+        logger.debug("Received message", text_data=f"{text_data[:50]}...")
+        event_data = json.loads(text_data)
+        try:
+            self.publish_form_template(event_data=event_data)
+        except Exception as e:
+            logger.exception(f"Error publishing form: {e}")
+            self.send_message(traceback.format_exc(), error=True)
+
+    def publish_form_template(self, event_data: dict):
+        publish_event = PublishTemplateEvent(**event_data)
+        self.send_message(f"Received event: {publish_event}")
+        form_template = FormTemplate.objects.select_related().get(id=publish_event.form_template)
+        app_users = form_template.project.app_users.filter(name__in=publish_event.app_users)
+        self.send_message(f'Publishing form template "{form_template.title_base}"')
+        user = self.scope["user"]
+        client = ODKPublishClient(
+            base_url=form_template.project.central_server.base_url,
+            project_id=form_template.project.central_id,
+        )
+        version = client.odk_publish.get_unique_version_by_form_id(
+            xml_form_id_base=form_template.form_id_base
+        )
+        self.send_message(f"Generated version: {version}")
+        name = f"{form_template.form_id_base}-{version}.xlsx"
+        file = form_template.download_google_sheet(user=user, name=name)
+        self.send_message(f"Downloaded template: {file}")
+        with transaction.atomic():
+            version = FormTemplateVersion.objects.create(
+                form_template=form_template, user=user, file=file, version=version
+            )
+            app_user_versions = version.create_app_user_versions(
+                app_users=app_users, send_message=self.send_message
+            )
+            for app_user_version in app_user_versions:
+                form = client.odk_publish.create_or_update_form(
+                    xml_form_id=app_user_version.app_user_form_template.xml_form_id,
+                    definition=app_user_version.file.read(),
+                )
+                self.send_message(f"Published form: {form.xmlFormId}")
+        self.send_message("Publish complete!")

apps/odk_publish/etl/excel.py

@@ -45,5 +45,5 @@ def find_cells_containing_value(sheet: Worksheet, value: str) -> Generator[Cell,
     # skip header row
     for row in sheet.iter_rows(min_row=2):
         for cell in row:
-            if cell.value is not None and value in cell.value:
+            if cell.value is not None and value in str(cell.value):
                 yield cell

apps/odk_publish/etl/template.py

@@ -85,7 +85,7 @@ def discover_entity_lists(workbook: Workbook) -> list[str]:
         for cell in row:
             if cell.value:
                 for pattern in patterns:
-                    match = pattern.search(cell.value)
+                    match = pattern.search(str(cell.value))
                     if match:
                         entity_lists.add(match.group(1))
                         logger.debug(
@@ -99,7 +99,7 @@ def discover_entity_lists(workbook: Workbook) -> list[str]:
         for row in workbook["entities"].iter_rows(min_row=2, min_col=1, max_col=1):
             for cell in row:
                 if cell.value:
-                    entity_lists.add(cell.value)
+                    entity_lists.add(str(cell.value))
                     logger.debug(
                         "Discovered entity list",
                         entity_list=cell.value,

apps/odk_publish/forms.py

@@ -7,6 +7,7 @@
 
 from .etl.odk.client import ODKPublishClient
 from .http import HttpRequest
+from .models import FormTemplate
 
 
 class ProjectSyncForm(PlatformFormMixin, forms.Form):
@@ -59,15 +60,17 @@ def set_project_choices(self, base_url: str):
 class PublishTemplateForm(PlatformFormMixin, forms.Form):
     """Form for publishing a form template to ODK Central."""
 
+    form_template = forms.IntegerField(widget=forms.HiddenInput())
     app_users = forms.CharField(
         required=False,
         label="Limit App Users",
         help_text="Publish to a limited set of app users by entering a comma-separated list.",
         widget=TextInput(attrs={"placeholder": "e.g., 10001, 10002, 10003"}),
     )
 
-    def __init__(self, request: HttpRequest, *args, **kwargs):
+    def __init__(self, request: HttpRequest, form_template: FormTemplate, *args, **kwargs):
         self.request = request
+        kwargs["initial"] = {"form_template": form_template.id}
         super().__init__(*args, **kwargs)
 
     def clean_app_users(self):

apps/odk_publish/models.py

@@ -143,11 +143,21 @@ class Meta:
     def __str__(self):
         return self.file.name
 
-    def create_app_user_versions(self) -> list["AppUserFormVersion"]:
+    def create_app_user_versions(
+        self, app_users: models.QuerySet["AppUser"] = None, send_message=None
+    ) -> list["AppUserFormVersion"]:
         app_user_versions = []
-        for app_user_form in AppUserFormTemplate.objects.filter(form_template=self.form_template):
+        q = models.Q(form_template=self.form_template)
+        if app_users:
+            q &= models.Q(app_user__in=app_users)
+        for app_user_form in AppUserFormTemplate.objects.filter(q):
             logger.info("Creating next AppUserFormVersion", app_user_form=app_user_form)
-            app_user_versions.append(app_user_form.create_next_version(form_template_version=self))
+            app_user_version = app_user_form.create_next_version(form_template_version=self)
+            xml_form_id = app_user_version.app_user_form_template.xml_form_id
+            version = app_user_version.form_template_version.version
+            if send_message:
+                send_message(f"Created FormTemplateVersion({xml_form_id=}, {version=})")
+            app_user_versions.append(app_user_version)
         return app_user_versions
 
 

apps/odk_publish/routing.py

@@ -0,0 +1,11 @@
+from django.urls import path
+
+from . import consumers
+
+websocket_urlpatterns = [
+    path(
+        "ws/publish-template/",
+        consumers.PublishTemplateConsumer.as_asgi(),
+        name="publish_template",
+    ),
+]

apps/odk_publish/views.py

@@ -120,7 +120,9 @@ def form_template_publish(request: HttpRequest, odk_project_pk: int, form_templa
     form_template: FormTemplate = get_object_or_404(
         request.odk_project.form_templates, pk=form_template_id
     )
-    form = PublishTemplateForm(request=request, data=request.POST or None)
+    form = PublishTemplateForm(
+        request=request, form_template=form_template, data=request.POST or None
+    )
     if request.method == "POST" and form.is_valid():
         pass
     context = {

config/asgi.py

@@ -9,15 +9,24 @@
 
 import os
 
-from channels.routing import ProtocolTypeRouter
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+from channels.security.websocket import AllowedHostsOriginValidator
 from django.core.asgi import get_asgi_application
 
-# from apps.chat.routing import websocket_urlpatterns
+from apps.odk_publish.routing import websocket_urlpatterns
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 
 application = ProtocolTypeRouter(
     {
         "http": get_asgi_application(),
+        "websocket": AllowedHostsOriginValidator(
+            AuthMiddlewareStack(
+                URLRouter(
+                    websocket_urlpatterns,  # add this alongside other routes.
+                )
+            )
+        ),
     }
 )

config/templates/odk_publish/form_template_publish.html

@@ -4,14 +4,12 @@
     Publish next version of {{ form_template.title_base }}
 {% endblock page_header %}
 {% block content %}
-    <!-- <p class="mb-4 dark:text-gray-400">
-    Use this form to sync a Project from an ODK Central server to ODK Publish. No changes are pushed to ODK Central.
-</p> -->
     <form method="post"
           class="max-w-2xl"
           x-data="{ submitting: false }"
           x-on:submit="submitting = true">
         {% csrf_token %}
+        {{ form.form_template.as_hidden }}
         <div class="grid gap-4 sm:grid-cols-1 sm:gap-6">
             <div id="id_form_container">
                 {{ form.app_users.label_tag }}
@@ -52,4 +50,18 @@
             </div>
         </div>
     </form>
+    {% if form.is_valid %}
+        <h3 class="text-l mt-5 mb-3 font-bold dark:text-white">Publishing...</h3>
+        <div id="message-list-wrapper"
+             hx-ext="ws"
+             ws-connect="/ws/publish-template/">
+            <form hx-ws="send" hx-trigger="load delay:1ms">
+                {{ form.form_template.as_hidden }}
+                {{ form.app_users.as_hidden }}
+            </form>
+            <div id="message-list"
+                 class="h-[50rem] overflow-y-auto"
+                 hx-swap-oob="beforeend"></div>
+        </div>
+    {% endif %}
 {% endblock content %}

config/templates/odk_publish/ws/message.html

@@ -0,0 +1,6 @@
+<div id="message-list" hx-swap-oob="beforeend">
+    <div class="p-4 mb-4 text-sm rounded-lg {% if not error %}text-primary-700 bg-primary-100 dark:bg-gray-800 dark:text-primary-300{% else %} font-mono text-red-700 bg-red-50 dark:bg-gray-800 dark:text-red-400{% endif %}">
+        {% if error %}<pre>{% endif %}{{ message_text|safe }}{% if error %}</pre>
+        {% endif %}
+    </div>
+</div>