Generate prometheus metrics.

This will allow monitoring the execution of the code.

.gitignore

@@ -8,3 +8,4 @@ __pycache__/
 /index.html
 /details.html
 /ipv6-in-real-life.json
+/metrics.json

ipv6_in_real_life/data_input.py

@@ -7,13 +7,21 @@
 import json
 from typing import IO, Any, Dict, Iterable, Iterator, Sequence
 
-from . import data_structures
+from . import data_structures, observability
 
 
 def _source_from_json(json_data: Iterable[Dict[str, Any]]) -> data_structures.Source:
-    source = data_structures.Source()
-    source.extend_from_json(json_data)
-    return source
+    try:
+        source = data_structures.Source()
+        source.extend_from_json(json_data)
+    except Exception as e:
+        observability.Metrics.get().set_source_loaded(observability.LoadStatus.FAILED)
+        raise e
+    else:
+        observability.Metrics.get().set_source_loaded(
+            observability.LoadStatus.COMPLETED
+        )
+        return source
 
 
 def load_input_data(input_files: Sequence[IO[str]]) -> data_structures.Source:

ipv6_in_real_life/data_structures.py

@@ -12,6 +12,8 @@
 import aiodns
 import pycountry
 
+from . import observability
+
 _LOGGER = logging.getLogger(__name__)
 
 HostJson = Dict[str, Union[bool, str, None]]
@@ -29,14 +31,19 @@ async def resolve(self, resolver: aiodns.DNSResolver) -> None:
             self.has_ipv4_address = True
         except aiodns.error.DNSError:
             _LOGGER.warning(f"{self.name} IPv4 DNS record not found either")
+            observability.Metrics.get().count_ipv4_resolution_failure()
             self.has_ipv4_address = False
+        else:
+            observability.Metrics.get().count_ipv4_resolution_success()
 
         try:
             all_results = await resolver.query(self.name, "AAAA")
             _LOGGER.debug(f"{self.name} resolved to {all_results!r}")
         except aiodns.error.DNSError:
             self.has_ipv6_address = False
+            observability.Metrics.get().count_ipv6_resolution_failure()
         else:
+            observability.Metrics.get().count_ipv6_resolution_success()
             valid_ipv6 = [
                 result.host
                 for result in all_results

ipv6_in_real_life/observability.py

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: 2023 Diego Elio Pettenò
+#
+# SPDX-License-Identifier: 0BSD
+
+import enum
+import pathlib
+import time
+from typing import Optional
+
+import prometheus_client
+
+
+class LoadStatus(enum.Enum):
+    NOT_STARTED = "not_started"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class Metrics:
+    _SINGLETON: Optional["Metrics"] = None
+
+    @classmethod
+    def get(cls) -> "Metrics":
+        if cls._SINGLETON is None:
+            cls._SINGLETON = Metrics()
+
+        return cls._SINGLETON
+
+    def __init__(self):
+        self._registry = prometheus_client.CollectorRegistry()
+
+        self._run_timestamp = prometheus_client.Gauge(
+            "last_run_timestamp",
+            "Unix Timestamp of the last generation.",
+            unit="seconds",
+            registry=self._registry,
+        )
+        self._run_timestamp.set(time.time())
+
+        self._source_loaded = prometheus_client.Enum(
+            "source_load_status",
+            "Status of the loading of data source.",
+            registry=self._registry,
+            states=[e.value for e in LoadStatus],
+        )
+        self.set_source_loaded(LoadStatus.NOT_STARTED)
+
+        self._ipv4_resolution_successes = prometheus_client.Counter(
+            "ipv4_resolution_successes",
+            "Number of resolution successes when resolving A records",
+            registry=self._registry,
+        )
+
+        self._ipv4_resolution_failures = prometheus_client.Counter(
+            "ipv4_resolution_failures",
+            "Number of resolution failures when resolving A records",
+            registry=self._registry,
+        )
+
+        self._ipv6_resolution_successes = prometheus_client.Counter(
+            "ipv6_resolution_successes",
+            "Number of resolution successes when resolving AAAA records",
+            registry=self._registry,
+        )
+
+        self._ipv6_resolution_failures = prometheus_client.Counter(
+            "ipv6_resolution_failures",
+            "Number of resolution failures when resolving AAAA records",
+            registry=self._registry,
+        )
+
+    def set_source_loaded(self, state: LoadStatus) -> None:
+        self._source_loaded.state(state.value)
+
+    def count_ipv4_resolution_success(self) -> None:
+        self._ipv4_resolution_successes.inc()
+
+    def count_ipv4_resolution_failure(self) -> None:
+        self._ipv4_resolution_failures.inc()
+
+    def count_ipv6_resolution_success(self) -> None:
+        self._ipv6_resolution_successes.inc()
+
+    def count_ipv6_resolution_failure(self) -> None:
+        self._ipv6_resolution_failures.inc()
+
+    def write_out(self, output: pathlib.Path) -> None:
+        prometheus_client.write_to_textfile(str(output), self._registry)

manual_generate.py

@@ -9,25 +9,31 @@
 import click
 import click_log
 
-from ipv6_in_real_life import data_input, render
+from ipv6_in_real_life import data_input, observability, render
 
 click_log.basic_config()
 
 
 async def amain(
     input_files: Sequence[IO[str]], output_directory: pathlib.Path, json_only: bool
 ) -> None:
+    # Initialize the start timestamp.
+    observability.Metrics.get()
+
     if input_files:
         source = data_input.load_input_data(input_files)
     else:
         source = data_input.load_packaged_data()
 
     await source.resolve_all()
 
-    (output_directory / "ipv6-in-real-life.json").write_text(source.as_json())
-    if not json_only:
-        (output_directory / "index.html").write_text(render.index(source))
-        (output_directory / "details.html").write_text(render.details(source))
+    try:
+        (output_directory / "ipv6-in-real-life.json").write_text(source.as_json())
+        if not json_only:
+            (output_directory / "index.html").write_text(render.index(source))
+            (output_directory / "details.html").write_text(render.details(source))
+    finally:
+        observability.Metrics.get().write_out(output_directory / "metrics.json")
 
 
 @click.command()

requirements.txt

@@ -8,3 +8,4 @@ click_log
 cmarkgfm
 jinja2
 pycountry
+prometheus-client