Merge pull request #814 from mediacloud/postgres_walg

Replace PgBackRest with WAL-G

apps/docker-compose.dist.yml

@@ -1346,44 +1346,36 @@ services:
             - default
         environment:
 
-            # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest
-            # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown
-            MC_PGBACKREST_ENABLE: "1"
+            # (optional) Set to "1" (string "1", not integer 1!) to enable WAL-G
+            # backups to S3; for more information, refer to doc/postgresql_walg.markdown
+            MC_WALG_ENABLE: "1"
 
-            # (required if PgBackRest is enabled) Full backup retention count/time:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full
-            MC_PGBACKREST_RETENTION_FULL: "2"
+            # (required if WAL-G is enabled) S3 Access Key ID:
+            MC_WALG_S3_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
 
-            # (required if PgBackRest is enabled) S3 repository endpoint:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint
-            MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com"
+            # (required if WAL-G is enabled) S3 Secret Access Key
+            MC_WALG_S3_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY"
 
-            # (required if PgBackRest is enabled) S3 repository bucket:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket
-            MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example"
+            # (required if WAL-G is enabled) S3 region
+            MC_WALG_S3_REGION: "us-east-1"
 
-            # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"):
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls
-            MC_PGBACKREST_S3_VERIFY_TLS: "y"
+            # (required) S3 URI bucket name and prefix (no slash at the end!)
+            MC_WALG_S3_BUCKET_PREFIX: "s3://mediacloud-postgresql-wal-backups-test/postgresql-server-test"
 
-            # (required if PgBackRest is enabled) S3 repository access key (Access Key ID):
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key
-            MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE"
+            # (optional) S3 or S3-compatible endpoint
+            # MC_WALG_S3_ENDPOINT: "https://s3.amazonaws.com"
 
-            # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key):
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret
-            MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY"
+            # (optional) S3 storage class
+            # MC_WALG_S3_STORAGE_CLASS: "STANDARD"
 
-            # (required if PgBackRest is enabled) S3 repository region:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region
-            MC_PGBACKREST_S3_REGION: "us-east-1"
+            # (optional) Whether to enable S3 path-style addressing ("true" or "false")
+            # MC_WALG_S3_FORCE_PATH_STYLE: "false"
 
-            # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-path
-            #
-            # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST!
-            #
-            MC_PGBACKREST_S3_PATH: "/postgresql-server"
+            # (optional) Use ListObjects instead of ListObjectsV2 ("true" or "false")
+            # MC_WALG_S3_USE_LIST_OBJECTS_V1: "false"
+
+            # (optional) Base64-encoded TLS certificate
+            # MC_WALG_S3_CA_CERT_BASE64: ""
 
         expose:
             - 5432
@@ -1884,44 +1876,36 @@ services:
             - default
         environment:
 
-            # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest
-            # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown
-            MC_PGBACKREST_ENABLE: "1"
+            # (optional) Set to "1" (string "1", not integer 1!) to enable WAL-G
+            # backups to S3; for more information, refer to doc/postgresql_walg.markdown
+            MC_WALG_ENABLE: "1"
 
-            # (required if PgBackRest is enabled) Full backup retention count/time:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full
-            MC_PGBACKREST_RETENTION_FULL: "2"
+            # (required if WAL-G is enabled) S3 Access Key ID:
+            MC_WALG_S3_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
 
-            # (required if PgBackRest is enabled) S3 repository endpoint:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint
-            MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com"
+            # (required if WAL-G is enabled) S3 Secret Access Key
+            MC_WALG_S3_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY"
 
-            # (required if PgBackRest is enabled) S3 repository bucket:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket
-            MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example"
+            # (required if WAL-G is enabled) S3 region
+            MC_WALG_S3_REGION: "us-east-1"
 
-            # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"):
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls
-            MC_PGBACKREST_S3_VERIFY_TLS: "y"
+            # (required) S3 URI bucket name and prefix (no slash at the end!)
+            MC_WALG_S3_BUCKET_PREFIX: "s3://mediacloud-postgresql-wal-backups-test/temporal-postgresql-test"
 
-            # (required if PgBackRest is enabled) S3 repository access key (Access Key ID):
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key
-            MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE"
+            # (optional) S3 or S3-compatible endpoint
+            # MC_WALG_S3_ENDPOINT: "https://s3.amazonaws.com"
 
-            # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key):
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret
-            MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY"
+            # (optional) S3 storage class
+            # MC_WALG_S3_STORAGE_CLASS: "STANDARD"
 
-            # (required if PgBackRest is enabled) S3 repository region:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region
-            MC_PGBACKREST_S3_REGION: "us-east-1"
+            # (optional) Whether to enable S3 path-style addressing ("true" or "false")
+            # MC_WALG_S3_FORCE_PATH_STYLE: "false"
 
-            # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash:
-            # https://pgbackrest.org/configuration.html#section-repository/option-repo-path
-            #
-            # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST!
-            #
-            MC_PGBACKREST_S3_PATH: "/temporal-postgresql"
+            # (optional) Use ListObjects instead of ListObjectsV2 ("true" or "false")
+            # MC_WALG_S3_USE_LIST_OBJECTS_V1: "false"
+
+            # (optional) Base64-encoded TLS certificate
+            # MC_WALG_S3_CA_CERT_BASE64: ""
 
         expose:
             - 5432

apps/postgresql-base/Dockerfile

@@ -41,12 +41,20 @@ RUN \
     fi; \
     true
 
-# Install pgBackRest for backing up PostgreSQL
+# Install WAL-G for backing up PostgreSQL
 RUN \
-    apt-get -y --no-install-recommends install pgbackrest && \
-    # Remove default configuration
-    rm -rf /etc/pgbackrest.conf /etc/pgbackrest/ && \
+    /dl_to_stdout.sh https://github.com/wal-g/wal-g/releases/download/v1.1/wal-g-pg-ubuntu-20.04-amd64.tar.gz > /var/tmp/wal-g.tar.gz && \
+    cd /var/tmp/ && \
     #
+    # Verify SHA1 so that we're sure about what we're installing
+    echo "f7cc6bf4d3f8e36cf05ae7fdd03bd3a0906359a3 wal-g.tar.gz" > /var/tmp/wal-g.tar.gz.sha1 && \
+    sha1sum -c wal-g.tar.gz.sha1 && \
+    tar -zxf wal-g.tar.gz && \
+    rm /var/tmp/wal-g.tar.gz* && \
+    #
+    # Users are expected to use wal-g.sh wrapper instead of "wal-g" binary directly
+    mv wal-g-pg-ubuntu-20.04-amd64 /usr/bin/_wal-g && \
+    chmod +x /usr/bin/_wal-g && \
     true
 
 # Make some run directories
@@ -58,7 +66,6 @@ RUN \
 # Write our own configuration
 RUN rm -rf /etc/postgresql/13/main/
 COPY etc/postgresql/13/main/ /etc/postgresql/13/main/
-COPY etc/pgbackrest/ /etc/pgbackrest/
 
 RUN \
     #
@@ -67,30 +74,30 @@ RUN \
     touch /var/run/postgresql/postgresql-memory.conf && \
     chown postgres:postgres /var/run/postgresql/postgresql-memory.conf && \
     #
-    # This is where "generate_runtime_config.sh" script will write PgBackRest-related
+    # This is where "generate_runtime_config.sh" script will write WAL-G-related
     # configuration
-    touch /var/run/postgresql/postgresql-pgbackrest.conf && \
-    chown postgres:postgres /var/run/postgresql/postgresql-pgbackrest.conf && \
-    #
-    # We'll write runtime S3 credentials there
-    chown postgres:postgres /etc/pgbackrest/conf.d/ && \
+    touch /var/run/postgresql/postgresql-walg.conf && \
+    chown postgres:postgres /var/run/postgresql/postgresql-walg.conf && \
     #
-    # Get rid of /var/lib/pgbackrest/ as we won't be using it
-    rm -rf /var/lib/pgbackrest/ && \
-    #
-    # (Re)-create /var/spool/pgbackrest/
-    rm -rf "/var/spool/pgbackrest/" && \
-    mkdir -p "/var/spool/pgbackrest/" && \
-    chown postgres:postgres "/var/spool/pgbackrest/" && \
-    chmod 750 "/var/spool/pgbackrest/" && \
+    # This is where "generate_runtime_config.sh" script will write WAL-G
+    # configuration to later be "source"'d in by wal-g.sh wrapper script
+    touch /var/run/postgresql/walg.env && \
+    chown postgres:postgres /var/run/postgresql/walg.env && \
+    chmod 600 /var/run/postgresql/walg.env && \
     #
     true
 
 # Copy helper scripts
 RUN mkdir -p /opt/postgresql-base/
 COPY bin/* /opt/postgresql-base/bin/
 
-ENV PATH="/opt/postgresql-base/bin:${PATH}"
+ENV \
+    PATH="/opt/postgresql-base/bin:${PATH}" \
+    #
+    # Make sure that we can connect via "psql" without sudoing into "postgres" user
+    # (PGUSER, PGPASSWORD and PGDATABASE will be set by sub-images of this image)
+    PGHOST=localhost \
+    PGPORT=5432
 
 USER postgres
 

apps/postgresql-base/bin/generate_runtime_config.sh

@@ -25,63 +25,108 @@ EOF
 
 
 #
-# Update PgBackRest configuration
+# Update WAL-G configuration
 #
 
-MC_POSTGRESQL_PGBACKREST_CONF_PATH="/var/run/postgresql/postgresql-pgbackrest.conf"
-MC_BACKREST_CONF_D_S3_CONF_PATH="/etc/pgbackrest/conf.d/s3.conf"
+MC_POSTGRESQL_WALG_CONF_PATH="/var/run/postgresql/postgresql-walg.conf"
 
+# Keep in sync with wal-g.sh
+MC_POSTGRESQL_WALG_ENV_PATH="/var/run/postgresql/walg.env"
 
-if [ -z ${MC_PGBACKREST_ENABLE+x} ]; then
+if [ ! -f "${MC_POSTGRESQL_WALG_CONF_PATH}" ]; then
+    echo "PostgreSQL WAL-G configuration file does not exist in ${MC_POSTGRESQL_WALG_CONF_PATH}"
+    exit 1
+fi
+if [ ! -f "${MC_POSTGRESQL_WALG_ENV_PATH}" ]; then
+    echo "PostgreSQL WAL-G environment file does not exist in ${MC_POSTGRESQL_WALG_ENV_PATH}"
+    exit 1
+fi
+
+if [ -z ${MC_WALG_ENABLE+x} ]; then
 
-    echo "PgBackRest is disabled."
+    echo "WAL-G is disabled."
 
-    cat > "${MC_POSTGRESQL_PGBACKREST_CONF_PATH}" << EOF
+    cat > "${MC_POSTGRESQL_WALG_CONF_PATH}" << EOF
 #
 # Auto-generated, please don't edit!
 #
 
 archive_mode = off
 EOF
 
-    cat > "${MC_BACKREST_CONF_D_S3_CONF_PATH}" << EOF
+    cat > "${MC_POSTGRESQL_WALG_ENV_PATH}" << EOF
 #
 # Auto-generated, please don't edit!
 #
 
-# S3 archiving disabled
+# WAL-G is disabled.
 EOF
 
 else
 
-    echo "PgBackRest is enabled."
+    echo "WAL-G is enabled."
 
-    cat > "${MC_POSTGRESQL_PGBACKREST_CONF_PATH}" << EOF
+    cat > "${MC_POSTGRESQL_WALG_CONF_PATH}" << EOF
 #
 # Auto-generated, please don't edit!
 #
 
-# Back up with PgBackRest
-# (stanzas of all users of postgresql-base are called "main")
+# Back up with WAL-G
 archive_mode = on
-archive_command = 'pgbackrest --stanza=main archive-push %p'
+archive_command = '/opt/postgresql-base/bin/wal-g.sh wal-push %p'
 EOF
 
-    cat > "${MC_BACKREST_CONF_D_S3_CONF_PATH}" << EOF
+    if [[ ! "${MC_WALG_S3_BUCKET_PREFIX}" == "s3://"* ]]; then
+        echo "S3 bucket + prefix must start with 's3://': ${MC_WALG_S3_BUCKET_PREFIX}"
+        exit 1
+    fi
+
+    if [ "${MC_WALG_S3_BUCKET_PREFIX: -1}" == "/" ]; then
+        echo "S3 bucket + prefix can't end with a slash: ${MC_WALG_S3_BUCKET_PREFIX}"
+        exit 1
+    fi
+
+    if [ -z ${MC_WALG_S3_ENDPOINT+x} ]; then
+        MC_WALG_S3_ENDPOINT="https://s3.amazonaws.com"
+    fi
+
+    if [[ ! "${MC_WALG_S3_ENDPOINT}" == "http"* ]]; then
+        echo "S3 endpoint must be 'https://' or 'http://': ${MC_WALG_S3_ENDPOINT}"
+        exit 1
+    fi
+
+    if [ -z ${MC_WALG_S3_STORAGE_CLASS+x} ]; then
+        MC_WALG_S3_STORAGE_CLASS="STANDARD"
+    fi
+    if [ -z ${MC_WALG_S3_FORCE_PATH_STYLE+x} ]; then
+        MC_WALG_S3_FORCE_PATH_STYLE="false"
+    fi
+    if [ -z ${MC_WALG_S3_USE_LIST_OBJECTS_V1+x} ]; then
+        MC_WALG_S3_USE_LIST_OBJECTS_V1="false"
+    fi
+
+    cat > "${MC_POSTGRESQL_WALG_ENV_PATH}" << EOF
 #
 # Auto-generated, please don't edit!
 #
 
-# S3 credentials
-[global]
-repo1-retention-full=${MC_PGBACKREST_RETENTION_FULL}
-repo1-s3-endpoint=${MC_PGBACKREST_S3_ENDPOINT}
-repo1-s3-bucket=${MC_PGBACKREST_S3_BUCKET}
-repo1-storage-verify-tls=${MC_PGBACKREST_S3_VERIFY_TLS}
-repo1-s3-key=${MC_PGBACKREST_S3_KEY}
-repo1-s3-key-secret=${MC_PGBACKREST_S3_KEY_SECRET}
-repo1-s3-region=${MC_PGBACKREST_S3_REGION}
-repo1-path=${MC_PGBACKREST_S3_PATH}
+# Keep up to 6 delta backups
+export WALG_DELTA_MAX_STEPS=6
+
+export AWS_ACCESS_KEY_ID=${MC_WALG_S3_ACCESS_KEY_ID}
+export AWS_SECRET_ACCESS_KEY=${MC_WALG_S3_SECRET_ACCESS_KEY}
+export AWS_REGION=${MC_WALG_S3_REGION}
+export AWS_ENDPOINT=${MC_WALG_S3_ENDPOINT}
+export WALG_S3_PREFIX=${MC_WALG_S3_BUCKET_PREFIX}
+export WALG_S3_STORAGE_CLASS=${MC_WALG_S3_STORAGE_CLASS}
+export AWS_S3_FORCE_PATH_STYLE=${MC_WALG_S3_FORCE_PATH_STYLE}
+export S3_USE_LIST_OBJECTS_V1=${MC_WALG_S3_USE_LIST_OBJECTS_V1}
 EOF
 
+    if [ ! -z ${MC_WALG_S3_CA_CERT_BASE64+x} ]; then
+        MC_WALG_S3_CA_CERT_FILE=/var/run/postgresql/walg.cert
+        echo "${MC_WALG_S3_CA_CERT_BASE64}" | base64 -d > "${MC_WALG_S3_CA_CERT_FILE}"
+        echo "export WALG_S3_CA_CERT_FILE=${MC_WALG_S3_CA_CERT_FILE}" >> \
+            "${MC_POSTGRESQL_WALG_ENV_PATH}"
+    fi
 fi

apps/postgresql-base/bin/wal-g.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Wrapper around "wal-g" binary which reads pre-configured credentials
+#
+
+set -u
+set -e
+
+# Keep in sync with generate_runtime_config.sh
+MC_POSTGRESQL_WALG_ENV_PATH="/var/run/postgresql/walg.env"
+
+if [ ! -f "${MC_POSTGRESQL_WALG_ENV_PATH}" ]; then
+    echo "WAL-G environment file ${MC_POSTGRESQL_WALG_ENV_PATH} does not exist;"
+    echo "maybe you haven't run PostgreSQL yet?"
+    exit 1
+fi
+
+source /var/run/postgresql/walg.env
+
+exec /usr/bin/_wal-g "$@"