working example

.circleci/config.yml

@@ -2,7 +2,8 @@ version: 2.1
 
 aliases:
   docker-image: &image
-    - image: mambaorg/micromamba
+    image: mambaorg/micromamba
+
   filter-pr-only: &PR-only
     branches:
       ignore:
@@ -216,13 +217,28 @@ workflows:
 
 jobs:
   lint:
-    docker: *image
+    docker:
+      - *image
     steps:
       - make-env:
           use_specific_requirements_file: requirements.dev.txt
       - lint-project
   build:
-    docker: *image
+    docker:
+      - *image
+      - image: circleci/postgres:12-alpine
+        environment:
+            POSTGRES_DB: postgres
+            POSTGRES_USER: postgres
+            POSTGRES_PASSWORD: testing_db
+            POSTGRES_HOST: localhost
+            POSTGRES_PORT: 5432
+            PYTEST_MONITOR_DB_NAME: postgres
+            PYTEST_MONITOR_DB_USER: postgres
+            PYTEST_MONITOR_DB_PASSWORD: testing_db
+            PYTEST_MONITOR_DB_HOST: localhost
+            PYTEST_MONITOR_DB_PORT: 5432
+
     parameters:
         python:
             type: string
@@ -235,7 +251,8 @@ jobs:
           pytest: << parameters.pytest >>
       - test-project
   publish:
-    docker: *image
+    docker:
+      - *image
     steps:
       - make-env:
           extra_deps: twine setuptools build

.gitlab-ci.yml

@@ -1,5 +1,20 @@
 image: continuumio/miniconda
 
+variables:
+   - POSTGRES_DB: postgres 
+   - POSTGRES_USER: postgres 
+   - POSTGRES_PASSWORD: testing_db
+   - POSTGRES_HOST: localhost
+   - POSTGRES_PORT: 5432
+   - PYTEST_MONITOR_DB_NAME: postgres
+   - PYTEST_MONITOR_DB_USER: postgres
+   - PYTEST_MONITOR_DB_PASSWORD: testing_db
+   - PYTEST_MONITOR_DB_HOST: localhost
+   - PYTEST_MONITOR_DB_PORT: 5432
+
+services:
+   - name: postgres:16
+
 stages:
    - test
    - deploy

AUTHORS

@@ -3,4 +3,5 @@ Project developed and lead by Jean-Sébastine Dieu.
 Contributors include:
  - Raymond Gauthier (jraygauthier) added Python 3.5 support.
  - Kyle Altendorf (altendky) fixed bugs on session teardown
- - Hannes Engelhardt (veritogen) added Bitbucket CI support.
+ - Hannes Engelhardt (veritogen) added Bitbucket CI support and Postgres DB Handler.
+ - Lucas Haupt (lhpt2) added Postgres DB Handler.

docker-compose.yml

@@ -0,0 +1,29 @@
+# Use postgres/example user/password credentials
+services:
+
+  db:
+    image: postgres
+    restart: always
+    # set shared memory limit when using docker-compose
+    shm_size: 128mb
+    # or set shared memory limit when deploy via swarm stack
+    #volumes:
+    #  - type: tmpfs
+    #    target: /dev/shm
+    #    tmpfs:
+    #      size: 134217728 # 128*2^20 bytes = 128Mb
+    environment:
+      POSTGRES_PASSWORD: testing_db
+    ports:
+      - 5432:5432
+    command: [ "postgres", "-c", "log_statement=all" ]
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "50m"
+
+  adminer:
+    image: adminer
+    restart: always
+    ports:
+      - 8080:8080

docs/sources/changelog.rst

@@ -3,8 +3,8 @@ Changelog
 =========
 
 * :release:`to be discussed`
+* :feature: `#77` Add a PostgreSQL backend implementation to optionally use a PostgreSQL Database for test metric logging.
 * :feature:`#75` Automatically gather CI build information for Bitbucket CI.
-
 * :release:`1.6.6 <2023-05-06>`
 * :bug:`#64` Prepare version 1.7.0 of pytest-monitor. Last version to support Python <= 3.7 and all pytest <= 5.*
 * :bug:`#0` Improve and fix some CI issues, notably one that may cause python to not be the requested one but a more recent one.

docs/sources/introduction.rst

@@ -28,4 +28,4 @@ Extending your application with new features, or fixing its bugs, might have an
 Usage
 -----
 
-Simply run pytest as usual: pytest-monitor is active by default as soon as it is installed. After running your first session, a .pymon sqlite database will be accessible in the directory where pytest was run.
+Simply run pytest as usual: pytest-monitor is active by default as soon as it is installed. After running your first session, a .pymon sqlite database (or optionally another database implementation like PostgreSQL) will be accessible in the directory where pytest was run.

docs/sources/operating.rst

@@ -19,6 +19,26 @@ You are free to override the name of this database by setting the `--db` option:
     pytest --db /path/to/your/monitor/database
 
 
+There is also a PostgreSQL implementation that talks to a PostgreSQL database. 
+The `--use-postgres` option is set for using PostgreSQL as database.
+
+.. code-block:: shell
+
+    pytest --use-postgres
+
+The connection parameters are set by the following environment variables:
+
+PYTEST_MONITOR_DB_HOST
+     The hostname of the instance running the PostgreSQL server
+PYTEST_MONITOR_DB_PORT
+     The port the PostgreSQL server listens on.
+PYTEST_MONITOR_DB_NAME
+     The name of the database to connect to.
+PYTEST_MONITOR_DB_USER
+     The name of the user to log into the database as.
+PYTEST_MONITOR_DB_PASSWORD
+     The password to log into the database.
+
 You can also sends your tests result to a monitor server (under development at that time) in order to centralize
 your Metrics and Execution Context (see below):
 

pyproject.toml

@@ -65,6 +65,12 @@ dev = [
     "flake8-pyproject==1.2.3",
     "pre-commit==3.3.3"
 ]
+psycopg = [
+    "psycopg"
+]
+psycopg2 = [
+    "psycopg2"
+]
 
 [tool.flake8]
 max-line-length = 120

pytest_monitor/handler.py

@@ -1,12 +1,24 @@
+import os
 import sqlite3
 
+try:
+    import psycopg
+except ImportError:
+    import psycopg2 as psycopg
 
-class DBHandler:
+
+class SqliteDBHandler:
     def __init__(self, db_path):
         self.__db = db_path
         self.__cnx = sqlite3.connect(self.__db) if db_path else None
         self.prepare()
 
+    def close(self):
+        self.__cnx.close()
+
+    def __del__(self):
+        self.__cnx.close()
+
     def query(self, what, bind_to, many=False):
         cursor = self.__cnx.cursor()
         cursor.execute(what, bind_to)
@@ -15,7 +27,8 @@ def query(self, what, bind_to, many=False):
     def insert_session(self, h, run_date, scm_id, description):
         with self.__cnx:
             self.__cnx.execute(
-                "insert into TEST_SESSIONS(SESSION_H, RUN_DATE, SCM_ID, RUN_DESCRIPTION)" " values (?,?,?,?)",
+                "insert into TEST_SESSIONS(SESSION_H, RUN_DATE, SCM_ID, RUN_DESCRIPTION)"
+                " values (?,?,?,?)",
                 (h, run_date, scm_id, description),
             )
 
@@ -131,3 +144,186 @@ def prepare(self):
 """
         )
         self.__cnx.commit()
+
+    def get_env_id(self, env_hash):
+        query_result = self.query(
+            "SELECT ENV_H FROM EXECUTION_CONTEXTS WHERE ENV_H= ?", (env_hash,)
+        )
+        return query_result[0] if query_result else None
+
+
+class PostgresDBHandler:
+    def __init__(self):
+        self.__db = os.getenv("PYTEST_MONITOR_DB_NAME")
+        if not self.__db:
+            raise Exception(
+                "Please provide the postgres db name using the PYTEST_MONITOR_DB_NAME environment variable."
+            )
+        self.__user = os.getenv("PYTEST_MONITOR_DB_USER")
+        if not self.__user:
+            raise Exception(
+                "Please provide the postgres user name using the PYTEST_MONITOR_DB_USER environment variable."
+            )
+        self.__password = os.getenv("PYTEST_MONITOR_DB_PASSWORD")
+        if not self.__password:
+            raise Exception(
+                "Please provide the postgres user password using the PYTEST_MONITOR_DB_PASSWORD environment variable."
+            )
+        self.__host = os.getenv("PYTEST_MONITOR_DB_HOST")
+        if not self.__host:
+            raise Exception(
+                "Please provide the postgres hostname using the PYTEST_MONITOR_DB_HOST environment variable."
+            )
+        self.__port = os.getenv("PYTEST_MONITOR_DB_PORT")
+        if not self.__port:
+            raise Exception(
+                "Please provide the postgres port using the PYTEST_MONITOR_DB_PORT environment variable."
+            )
+        self.__cnx = self.connect()
+        self.prepare()
+
+    def __del__(self):
+        self.__cnx.close()
+
+    def close(self):
+        self.__cnx.close()
+
+    def connect(self):
+        connection_string = (
+            f"dbname='{self.__db}' user='{self.__user}' password='{self.__password}' "
+            + f"host='{self.__host}' port='{self.__port}'"
+        )
+        return psycopg.connect(connection_string)
+
+    def query(self, what, bind_to, many=False):
+        cursor = self.__cnx.cursor()
+        cursor.execute(what, bind_to)
+        return cursor.fetchall() if many else cursor.fetchone()
+
+    def insert_session(self, h, run_date, scm_id, description):
+        self.__cnx.cursor().execute(
+            "insert into TEST_SESSIONS(SESSION_H, RUN_DATE, SCM_ID, RUN_DESCRIPTION)"
+            " values (%s,%s,%s,%s)",
+            (h, run_date, scm_id, description),
+        )
+        self.__cnx.commit()
+
+    def insert_metric(
+        self,
+        session_id,
+        env_id,
+        item_start_date,
+        item,
+        item_path,
+        item_variant,
+        item_loc,
+        kind,
+        component,
+        total_time,
+        user_time,
+        kernel_time,
+        cpu_usage,
+        mem_usage,
+    ):
+        self.__cnx.cursor().execute(
+            "insert into TEST_METRICS(SESSION_H,ENV_H,ITEM_START_TIME,ITEM,"
+            "ITEM_PATH,ITEM_VARIANT,ITEM_FS_LOC,KIND,COMPONENT,TOTAL_TIME,"
+            "USER_TIME,KERNEL_TIME,CPU_USAGE,MEM_USAGE) "
+            "values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+            (
+                session_id,
+                env_id,
+                item_start_date,
+                item,
+                item_path,
+                item_variant,
+                item_loc,
+                kind,
+                component,
+                total_time,
+                user_time,
+                kernel_time,
+                cpu_usage,
+                mem_usage,
+            ),
+        )
+        self.__cnx.commit()
+
+    def insert_execution_context(self, exc_context):
+        self.__cnx.cursor().execute(
+            "insert into EXECUTION_CONTEXTS(CPU_COUNT,CPU_FREQUENCY_MHZ,CPU_TYPE,CPU_VENDOR,"
+            "RAM_TOTAL_MB,MACHINE_NODE,MACHINE_TYPE,MACHINE_ARCH,SYSTEM_INFO,"
+            "PYTHON_INFO,ENV_H) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+            (
+                exc_context.cpu_count,
+                exc_context.cpu_frequency,
+                exc_context.cpu_type,
+                exc_context.cpu_vendor,
+                exc_context.ram_total,
+                exc_context.fqdn,
+                exc_context.machine,
+                exc_context.architecture,
+                exc_context.system_info,
+                exc_context.python_info,
+                exc_context.compute_hash(),
+            ),
+        )
+        self.__cnx.commit()
+
+    def prepare(self):
+        cursor = self.__cnx.cursor()
+        cursor.execute(
+            """
+CREATE TABLE IF NOT EXISTS TEST_SESSIONS(
+    SESSION_H varchar(64) primary key not null unique, -- Session identifier
+    RUN_DATE varchar(64), -- Date of test run
+    SCM_ID varchar(128), -- SCM change id
+    RUN_DESCRIPTION json
+);"""
+        )
+        cursor.execute(
+            """
+CREATE TABLE IF NOT EXISTS EXECUTION_CONTEXTS (
+   ENV_H varchar(64) primary key not null unique,
+   CPU_COUNT integer,
+   CPU_FREQUENCY_MHZ integer,
+   CPU_TYPE varchar(64),
+   CPU_VENDOR varchar(256),
+   RAM_TOTAL_MB integer,
+   MACHINE_NODE varchar(512),
+   MACHINE_TYPE varchar(32),
+   MACHINE_ARCH varchar(16),
+   SYSTEM_INFO varchar(256),
+   PYTHON_INFO varchar(512)
+);
+"""
+        )
+        cursor.execute(
+            """
+CREATE TABLE IF NOT EXISTS TEST_METRICS (
+    SESSION_H varchar(64), -- Session identifier
+    ENV_H varchar(64), -- Environment description identifier
+    ITEM_START_TIME varchar(64), -- Effective start time of the test
+    ITEM_PATH varchar(4096), -- Path of the item, following Python import specification
+    ITEM varchar(2048), -- Name of the item
+    ITEM_VARIANT varchar(2048), -- Optional parametrization of an item.
+    ITEM_FS_LOC varchar(2048), -- Relative path from pytest invocation directory to the item's module.
+    KIND varchar(64), -- Package, Module or function
+    COMPONENT varchar(512) NULL, -- Tested component if any
+    TOTAL_TIME float, -- Total time spent running the item
+    USER_TIME float, -- time spent in user space
+    KERNEL_TIME float, -- time spent in kernel space
+    CPU_USAGE float, -- cpu usage
+    MEM_USAGE float, -- Max resident memory used.
+    FOREIGN KEY (ENV_H) REFERENCES EXECUTION_CONTEXTS(ENV_H),
+    FOREIGN KEY (SESSION_H) REFERENCES TEST_SESSIONS(SESSION_H)
+);"""
+        )
+
+        self.__cnx.commit()
+
+    def get_env_id(self, env_hash):
+        query_result = self.query(
+            "select ENV_H from EXECUTION_CONTEXTS where ENV_H = %s", (env_hash,)
+        )
+        return query_result[0] if query_result else None