Prototype of the remote boefje runner

boefjes/Makefile

@@ -42,6 +42,7 @@ images:  # Build the images for the containerized boefjes
 	docker build -f ./boefjes/plugins/kat_nmap_tcp/boefje.Dockerfile -t openkat/nmap  .
 
 
+
 ##
 ##|------------------------------------------------------------------------|
 ##			Migrations

boefjes/boefjes/api.py

@@ -159,7 +159,7 @@ def create_boefje_meta(task, local_repository):
     arguments = {"oci_arguments": boefje_resource.oci_arguments}
 
     if input_ooi:
-        reference = Reference.from_str(input_ooi)
+        reference = Reference.from_str(input_ooi)  # TODO SOUF why is this here? Just giving `input_ooi` works too
         try:
             ooi = get_octopoes_api_connector(organization).get(reference, valid_time=datetime.now(timezone.utc))
         except ObjectNotFoundException as e:
@@ -170,7 +170,7 @@ def create_boefje_meta(task, local_repository):
     boefje_meta = BoefjeMeta(
         id=task.id,
         boefje=boefje,
-        input_ooi=input_ooi,
+        input_ooi=input_ooi,  # `input_ooi` is of type str, and we just give it an instance here?
         arguments=arguments,
         organization=organization,
         environment=environment,

boefjes/boefjes/app.py

@@ -99,10 +99,8 @@ def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
         all_queues_empty = True
 
         for queue_type in queues:
-            logger.debug("Popping from queue %s", queue_type.id)
-
             try:
-                p_item = self.scheduler_client.pop_item(queue_type.id)
+                p_item = self.scheduler_client.pop_non_remote_item(queue_type.id)
             except (HTTPError, ValidationError):
                 logger.exception("Popping task from scheduler failed, sleeping 10 seconds")
                 time.sleep(10)

boefjes/boefjes/clients/scheduler_client.py

@@ -2,6 +2,7 @@
 import logging
 import uuid
 from enum import Enum
+from typing import Literal
 
 from httpx import Client, HTTPTransport, Response
 from pydantic import BaseModel, TypeAdapter
@@ -26,6 +27,7 @@ class QueuePrioritizedItem(BaseModel):
     priority: int
     hash: str | None = None
     data: BoefjeMeta | NormalizerMeta
+    remote: bool
 
 
 class TaskStatus(Enum):
@@ -47,13 +49,26 @@ class Task(BaseModel):
     status: TaskStatus
     created_at: datetime.datetime
     modified_at: datetime.datetime
+    remote: bool
+
+
+# TODO: SOUF ask where to put this
+class Filter(BaseModel):
+    column: str
+    field: str | None = None
+    operator: Literal["=="] = "=="
+    value: bool
+
+
+class QueuePopModel(BaseModel):
+    filters: list[Filter]
 
 
 class SchedulerClientInterface:
     def get_queues(self) -> list[Queue]:
         raise NotImplementedError()
 
-    def pop_item(self, queue: str) -> QueuePrioritizedItem | None:
+    def pop_non_remote_item(self, queue: str) -> QueuePrioritizedItem | None:
         raise NotImplementedError()
 
     def patch_task(self, task_id: uuid.UUID, status: TaskStatus) -> None:
@@ -80,8 +95,9 @@ def get_queues(self) -> list[Queue]:
 
         return TypeAdapter(list[Queue]).validate_json(response.content)
 
-    def pop_item(self, queue: str) -> QueuePrioritizedItem | None:
-        response = self._session.post(f"/queues/{queue}/pop")
+    def pop_non_remote_item(self, queue: str) -> QueuePrioritizedItem | None:
+        non_remote_filter = QueuePopModel(filters=[Filter(column="remote", operator="==", value=False)])
+        response = self._session.post(f"/queues/{queue}/pop", json=non_remote_filter.model_dump())
         self._verify_response(response)
 
         return TypeAdapter(QueuePrioritizedItem | None).validate_json(response.content)

boefjes/boefjes/job_handler.py

@@ -15,6 +15,7 @@
 from boefjes.job_models import BoefjeMeta, NormalizerMeta, SerializedOOI, SerializedOOIValue
 from boefjes.katalogus.local_repository import LocalPluginRepository
 from boefjes.plugins.models import _default_mime_types
+from boefjes.remote_boefjes_runner import RemoteBoefjesRunner
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
 from octopoes.api.models import Affirmation, Declaration, Observation
 from octopoes.connector.octopoes import OctopoesAPIConnector
@@ -102,9 +103,23 @@ def __init__(
 
     def handle(self, boefje_meta: BoefjeMeta) -> None:
         logger.info("Handling boefje %s[task_id=%s]", boefje_meta.boefje.id, str(boefje_meta.id))
+        boefje_resource = self.local_repository.by_id(boefje_meta.boefje.id)
+
+        env_keys = boefje_resource.environment_keys
+        boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {}
+
+        # Check if the user has provided the boefje with a `remote_url`, if so, use the `RemoteBoefjesRunner`
+        if boefje_meta.environment and boefje_meta.environment.get("REMOTE_URL", ""):
+            logger.info(
+                "Forwarding boefje %s[task_id=%s] to a remote container",
+                boefje_meta.boefje.id,
+                str(boefje_meta.id),
+            )
+            remote_runner = RemoteBoefjesRunner(boefje_resource, boefje_meta)
+
+            return remote_runner.run()
 
         # Check if this boefje is container-native, if so, continue using the Docker boefjes runner
-        boefje_resource = self.local_repository.by_id(boefje_meta.boefje.id)
         if boefje_resource.oci_image:
             logger.info(
                 "Delegating boefje %s[task_id=%s] to Docker runner with OCI image [%s]",
@@ -113,7 +128,7 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
                 boefje_resource.oci_image,
             )
             docker_runner = DockerBoefjesRunner(boefje_resource, boefje_meta)
-            return docker_runner.run()
+            return docker_runner.run()  # TODO: Make docker use the now-provided environment variables
 
         if boefje_meta.input_ooi:
             reference = Reference.from_str(boefje_meta.input_ooi)
@@ -126,10 +141,7 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
 
             boefje_meta.arguments["input"] = serialize_ooi(ooi)
 
-        env_keys = boefje_resource.environment_keys
-
         boefje_meta.runnable_hash = boefje_resource.runnable_hash
-        boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {}
 
         mime_types = _default_mime_types(boefje_meta.boefje)
 

boefjes/boefjes/job_models.py

@@ -19,6 +19,7 @@ class Job(BaseModel):
     id: UUID
     started_at: AwareDatetime | None = Field(default=None)
     ended_at: AwareDatetime | None = Field(default=None)
+    remote: bool = Field(default=False)
 
     @property
     def runtime(self) -> timedelta | None:
@@ -33,6 +34,7 @@ class Boefje(BaseModel):
 
     id: Annotated[str, StringConstraints(min_length=1)]
     version: str | None = Field(default=None)
+    remote: bool = Field(default=False)
 
 
 class Normalizer(BaseModel):

boefjes/boefjes/katalogus/models.py

@@ -20,6 +20,7 @@ class Plugin(BaseModel):
     environment_keys: list[str] = Field(default_factory=list)
     related: list[str] | None = None
     enabled: bool = False
+    remote: bool = False
 
     def __str__(self):
         return f"{self.id}:{self.version}"

boefjes/boefjes/local.py

@@ -18,6 +18,7 @@
 from octopoes.models import OOI, DeclaredScanProfile
 
 logger = logging.getLogger(__name__)
+# TODO: SOUF change filename to `local_boefjes_runner` for consistency
 
 
 class TemporaryEnvironment:

boefjes/boefjes/plugins/kat_remote_scanner/boefje.Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+RUN apt-get update && apt-get install -y nmap && pip install httpx
+
+ARG BOEFJE_PATH=./boefjes/plugins/kat_remote_scanner
+ENV PYTHONPATH=/app:$BOEFJE_PATH
+
+COPY ./images/oci_adapter.py ./
+COPY $BOEFJE_PATH $BOEFJE_PATH
+
+ENTRYPOINT ["/usr/local/bin/python", "-m", "oci_adapter"]

boefjes/boefjes/plugins/kat_remote_scanner/boefje.json

@@ -0,0 +1,17 @@
+{
+  "id": "remote-scanner",
+  "name": "Remote scanner",
+  "description": "Scans from the remote",
+  "consumes": [
+    "HTTPResource",
+    "IPAddressV4",
+    "IPAddressV6"
+  ],
+  "environment_keys": [
+    "REMOTE_URL",
+    "COMMAND"
+  ],
+  "scan_level": 4,
+  "oci_image": "openkat/remote-scanner",
+  "remote": true
+}

boefjes/boefjes/plugins/kat_remote_scanner/description.md

@@ -0,0 +1,3 @@
+# REMOTE SCANNER
+
+WHERE DO I GET DISPLAYED

boefjes/boefjes/plugins/kat_remote_scanner/main.py

@@ -0,0 +1,16 @@
+import os
+
+
+def run(boefje_meta: dict):
+    cmd = os.getenv("COMMAND", "echo no command given")
+    return [(set(), f"{cmd} has been ran!")]
+
+
+"""echo $PWD
+from ipaddress import ip_address
+import socket
+hostname = socket.gethostname()
+IPAddr = socket.gethostbyname(hostname)
+ip = ip_address(boefje_meta["arguments"]["website"]["ip_service"]["ip_port"]["address"]["address"])
+return [(set(), f"{hostname};;{IPAddr};;{ip.exploded}")]
+"""

boefjes/boefjes/plugins/kat_remote_scanner/schema.json

@@ -0,0 +1,17 @@
+{
+  "title": "Arguments",
+  "type": "object",
+  "properties": {
+    "COMMAND": {
+      "title": "COMMAND",
+      "type": "string",
+      "description": "The command to execute remotely"
+    },
+    "REMOTE_URL": {
+      "title": "REMOTE_URL",
+      "type": "string",
+      "description": "The command location to execute the command"
+    }
+  },
+  "required": []
+}

boefjes/boefjes/remote_boefjes_runner.py

@@ -0,0 +1,87 @@
+import logging
+from datetime import datetime, timezone
+
+import httpx
+from httpx import Timeout
+from pydantic import BaseModel
+
+from boefjes.clients.bytes_client import BytesAPIClient
+from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
+from boefjes.config import settings
+from boefjes.job_models import BoefjeMeta
+from boefjes.katalogus.models import Boefje
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteBoefjesRunner:
+    def __init__(self, boefje_resource: Boefje, boefje_meta: BoefjeMeta):
+        self.boefje_resource = boefje_resource
+        self.boefje_meta = boefje_meta
+        self.scheduler_client = SchedulerAPIClient(str(settings.scheduler_api))
+        self.bytes_api_client = BytesAPIClient(
+            str(settings.bytes_api),
+            username=settings.bytes_username,
+            password=settings.bytes_password,
+        )
+
+    def run(self) -> None:
+        REMOTE_URL = self.boefje_meta.environment.get("REMOTE_URL", "")
+        if not REMOTE_URL:
+            raise RuntimeError("Boefje does not have a URL")
+
+        # local import to prevent circular dependency
+        import boefjes.plugins.models  # TODO: ask why this is needed since boefjes.plugins.models gets imported on top
+
+        stderr_mime_types = boefjes.plugins.models._default_mime_types(self.boefje_meta.boefje)
+
+        task_id = self.boefje_meta.id
+        self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING)
+        self.boefje_meta.started_at = datetime.now(timezone.utc)
+
+        try:
+            task_url = str(settings.api).rstrip("/") + f"/api/v0/tasks/{task_id}"
+
+            request = RemoteBoefjeRequest(
+                name="kat_boefje_" + str(task_id),
+                task_url=task_url,
+                boefje_resource=self.boefje_resource,
+                boefje_meta=self.boefje_meta,
+            )
+
+            logger.info(request.model_dump_json())
+            response = httpx.post(url=REMOTE_URL, data=request.model_dump_json(), timeout=Timeout(timeout=10))
+
+            if response.is_error:
+                logger.exception(response)
+
+                self.boefje_meta.ended_at = datetime.now(timezone.utc)
+                self.bytes_api_client.save_boefje_meta(self.boefje_meta)  # The task didn't create a boefje_meta object
+                self.bytes_api_client.save_raw(task_id, response, stderr_mime_types.union({"error/boefje"}))
+                self.scheduler_client.patch_task(task_id, TaskStatus.FAILED)
+
+                # have to raise exception to prevent _start_working function from setting status to completed
+                raise RuntimeError("Boefje did not call output API endpoint")
+
+        except httpx.NetworkError as e:
+            logger.exception("Container error")
+            logger.exception(request.model_dump_json())
+            # save container log (stderr) to bytes
+            self.bytes_api_client.login()
+            self.boefje_meta.ended_at = datetime.now(timezone.utc)
+            try:
+                # this boefje_meta might be incomplete, it comes from the scheduler instead of the Boefje I/O API
+                self.bytes_api_client.save_boefje_meta(self.boefje_meta)
+            except httpx.HTTPError:
+                logger.error("Failed to save boefje meta to bytes, continuing anyway")
+            self.bytes_api_client.save_raw(task_id, "TODO", stderr_mime_types)  # TODO
+            self.scheduler_client.patch_task(task_id, TaskStatus.FAILED)
+            # have to raise exception to prevent _start_working function from setting status to completed
+            raise e
+
+
+class RemoteBoefjeRequest(BaseModel):
+    name: str
+    task_url: str
+    boefje_resource: Boefje
+    boefje_meta: BoefjeMeta

boefjes/tests/conftest.py

@@ -45,11 +45,12 @@ def get_queues(self) -> list[Queue]:
         time.sleep(self.sleep_time)
         return TypeAdapter(list[Queue]).validate_json(self.queue_response)
 
-    def pop_item(self, queue: str) -> QueuePrioritizedItem | None:
+    def pop_non_remote_item(self, queue: str) -> QueuePrioritizedItem | None:
         time.sleep(self.sleep_time)
 
         try:
             if WorkerManager.Queue.BOEFJES.value in queue:
+                print(self.boefje_responses[0].decode())
                 p_item = TypeAdapter(QueuePrioritizedItem).validate_json(self.boefje_responses.pop(0))
                 self._popped_items[str(p_item.id)] = p_item
                 self._tasks[str(p_item.id)] = self._task_from_id(p_item.id)
@@ -86,6 +87,7 @@ def _task_from_id(self, task_id: UUID):
             status=TaskStatus.DISPATCHED,
             created_at=datetime.now(timezone.utc),
             modified_at=datetime.now(timezone.utc),
+            remote=False,
         )
 
     def push_item(self, queue_id: str, p_item: QueuePrioritizedItem) -> None:

boefjes/tests/examples/scheduler/pop_response_boefje.json

@@ -6,14 +6,17 @@
     "id": "70da7d4f-f41f-4940-901b-d98a92e9014b",
     "boefje": {
       "id": "dns-records",
-      "version": null
+      "version": null,
+      "remote": false
     },
     "input_ooi": "Hostname|internet|test.test",
     "organization": "_dev",
     "arguments": {},
     "started_at": null,
     "runnable_hash": null,
     "environment": null,
-    "ended_at": null
-  }
+    "ended_at": null,
+    "remote": false
+  },
+  "remote": false
 }