diff --git a/apps/docker-compose.dist.yml b/apps/docker-compose.dist.yml index 599685ac06..7a63729317 100644 --- a/apps/docker-compose.dist.yml +++ b/apps/docker-compose.dist.yml @@ -1346,44 +1346,36 @@ services: - default environment: - # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest - # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown - MC_PGBACKREST_ENABLE: "1" + # (optional) Set to "1" (string "1", not integer 1!) to enable WAL-G + # backups to S3; for more information, refer to doc/postgresql_walg.markdown + MC_WALG_ENABLE: "1" - # (required if PgBackRest is enabled) Full backup retention count/time: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full - MC_PGBACKREST_RETENTION_FULL: "2" + # (required if WAL-G is enabled) S3 Access Key ID: + MC_WALG_S3_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - # (required if PgBackRest is enabled) S3 repository endpoint: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint - MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com" + # (required if WAL-G is enabled) S3 Secret Access Key + MC_WALG_S3_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY" - # (required if PgBackRest is enabled) S3 repository bucket: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket - MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example" + # (required if WAL-G is enabled) S3 region + MC_WALG_S3_REGION: "us-east-1" - # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"): - # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls - MC_PGBACKREST_S3_VERIFY_TLS: "y" + # (required) S3 URI bucket name and prefix (no slash at the end!) + MC_WALG_S3_BUCKET_PREFIX: "s3://mediacloud-postgresql-wal-backups-test/postgresql-server-test" - # (required if PgBackRest is enabled) S3 repository access key (Access Key ID): - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key - MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE" + # (optional) S3 or S3-compatible endpoint + # MC_WALG_S3_ENDPOINT: "https://s3.amazonaws.com" - # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key): - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret - MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY" + # (optional) S3 storage class + # MC_WALG_S3_STORAGE_CLASS: "STANDARD" - # (required if PgBackRest is enabled) S3 repository region: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region - MC_PGBACKREST_S3_REGION: "us-east-1" + # (optional) Whether to enable S3 path-style addressing ("true" or "false") + # MC_WALG_S3_FORCE_PATH_STYLE: "false" - # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-path - # - # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST! - # - MC_PGBACKREST_S3_PATH: "/postgresql-server" + # (optional) Use ListObjects instead of ListObjectsV2 ("true" or "false") + # MC_WALG_S3_USE_LIST_OBJECTS_V1: "false" + + # (optional) Base64-encoded TLS certificate + # MC_WALG_S3_CA_CERT_BASE64: "" expose: - 5432 @@ -1884,44 +1876,36 @@ services: - default environment: - # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest - # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown - MC_PGBACKREST_ENABLE: "1" + # (optional) Set to "1" (string "1", not integer 1!) to enable WAL-G + # backups to S3; for more information, refer to doc/postgresql_walg.markdown + MC_WALG_ENABLE: "1" - # (required if PgBackRest is enabled) Full backup retention count/time: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full - MC_PGBACKREST_RETENTION_FULL: "2" + # (required if WAL-G is enabled) S3 Access Key ID: + MC_WALG_S3_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - # (required if PgBackRest is enabled) S3 repository endpoint: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint - MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com" + # (required if WAL-G is enabled) S3 Secret Access Key + MC_WALG_S3_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY" - # (required if PgBackRest is enabled) S3 repository bucket: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket - MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example" + # (required if WAL-G is enabled) S3 region + MC_WALG_S3_REGION: "us-east-1" - # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"): - # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls - MC_PGBACKREST_S3_VERIFY_TLS: "y" + # (required) S3 URI bucket name and prefix (no slash at the end!) + MC_WALG_S3_BUCKET_PREFIX: "s3://mediacloud-postgresql-wal-backups-test/temporal-postgresql-test" - # (required if PgBackRest is enabled) S3 repository access key (Access Key ID): - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key - MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE" + # (optional) S3 or S3-compatible endpoint + # MC_WALG_S3_ENDPOINT: "https://s3.amazonaws.com" - # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key): - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret - MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY" + # (optional) S3 storage class + # MC_WALG_S3_STORAGE_CLASS: "STANDARD" - # (required if PgBackRest is enabled) S3 repository region: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region - MC_PGBACKREST_S3_REGION: "us-east-1" + # (optional) Whether to enable S3 path-style addressing ("true" or "false") + # MC_WALG_S3_FORCE_PATH_STYLE: "false" - # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash: - # https://pgbackrest.org/configuration.html#section-repository/option-repo-path - # - # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST! - # - MC_PGBACKREST_S3_PATH: "/temporal-postgresql" + # (optional) Use ListObjects instead of ListObjectsV2 ("true" or "false") + # MC_WALG_S3_USE_LIST_OBJECTS_V1: "false" + + # (optional) Base64-encoded TLS certificate + # MC_WALG_S3_CA_CERT_BASE64: "" expose: - 5432 diff --git a/apps/postgresql-base/Dockerfile b/apps/postgresql-base/Dockerfile index 6401567c48..5636eb5646 100644 --- a/apps/postgresql-base/Dockerfile +++ b/apps/postgresql-base/Dockerfile @@ -41,12 +41,20 @@ RUN \ fi; \ true -# Install pgBackRest for backing up PostgreSQL +# Install WAL-G for backing up PostgreSQL RUN \ - apt-get -y --no-install-recommends install pgbackrest && \ - # Remove default configuration - rm -rf /etc/pgbackrest.conf /etc/pgbackrest/ && \ + /dl_to_stdout.sh https://github.com/wal-g/wal-g/releases/download/v1.1/wal-g-pg-ubuntu-20.04-amd64.tar.gz > /var/tmp/wal-g.tar.gz && \ + cd /var/tmp/ && \ # + # Verify SHA1 so that we're sure about what we're installing + echo "f7cc6bf4d3f8e36cf05ae7fdd03bd3a0906359a3 wal-g.tar.gz" > /var/tmp/wal-g.tar.gz.sha1 && \ + sha1sum -c wal-g.tar.gz.sha1 && \ + tar -zxf wal-g.tar.gz && \ + rm /var/tmp/wal-g.tar.gz* && \ + # + # Users are expected to use wal-g.sh wrapper instead of "wal-g" binary directly + mv wal-g-pg-ubuntu-20.04-amd64 /usr/bin/_wal-g && \ + chmod +x /usr/bin/_wal-g && \ true # Make some run directories @@ -58,7 +66,6 @@ RUN \ # Write our own configuration RUN rm -rf /etc/postgresql/13/main/ COPY etc/postgresql/13/main/ /etc/postgresql/13/main/ -COPY etc/pgbackrest/ /etc/pgbackrest/ RUN \ # @@ -67,22 +74,16 @@ RUN \ touch /var/run/postgresql/postgresql-memory.conf && \ chown postgres:postgres /var/run/postgresql/postgresql-memory.conf && \ # - # This is where "generate_runtime_config.sh" script will write PgBackRest-related + # This is where "generate_runtime_config.sh" script will write WAL-G-related # configuration - touch /var/run/postgresql/postgresql-pgbackrest.conf && \ - chown postgres:postgres /var/run/postgresql/postgresql-pgbackrest.conf && \ - # - # We'll write runtime S3 credentials there - chown postgres:postgres /etc/pgbackrest/conf.d/ && \ + touch /var/run/postgresql/postgresql-walg.conf && \ + chown postgres:postgres /var/run/postgresql/postgresql-walg.conf && \ # - # Get rid of /var/lib/pgbackrest/ as we won't be using it - rm -rf /var/lib/pgbackrest/ && \ - # - # (Re)-create /var/spool/pgbackrest/ - rm -rf "/var/spool/pgbackrest/" && \ - mkdir -p "/var/spool/pgbackrest/" && \ - chown postgres:postgres "/var/spool/pgbackrest/" && \ - chmod 750 "/var/spool/pgbackrest/" && \ + # This is where "generate_runtime_config.sh" script will write WAL-G + # configuration to later be "source"'d in by wal-g.sh wrapper script + touch /var/run/postgresql/walg.env && \ + chown postgres:postgres /var/run/postgresql/walg.env && \ + chmod 600 /var/run/postgresql/walg.env && \ # true @@ -90,7 +91,13 @@ RUN \ RUN mkdir -p /opt/postgresql-base/ COPY bin/* /opt/postgresql-base/bin/ -ENV PATH="/opt/postgresql-base/bin:${PATH}" +ENV \ + PATH="/opt/postgresql-base/bin:${PATH}" \ + # + # Make sure that we can connect via "psql" without sudoing into "postgres" user + # (PGUSER, PGPASSWORD and PGDATABASE will be set by sub-images of this image) + PGHOST=localhost \ + PGPORT=5432 USER postgres diff --git a/apps/postgresql-base/bin/generate_runtime_config.sh b/apps/postgresql-base/bin/generate_runtime_config.sh index 3653f1fb3d..d9911c4f6b 100755 --- a/apps/postgresql-base/bin/generate_runtime_config.sh +++ b/apps/postgresql-base/bin/generate_runtime_config.sh @@ -25,18 +25,28 @@ EOF # -# Update PgBackRest configuration +# Update WAL-G configuration # -MC_POSTGRESQL_PGBACKREST_CONF_PATH="/var/run/postgresql/postgresql-pgbackrest.conf" -MC_BACKREST_CONF_D_S3_CONF_PATH="/etc/pgbackrest/conf.d/s3.conf" +MC_POSTGRESQL_WALG_CONF_PATH="/var/run/postgresql/postgresql-walg.conf" +# Keep in sync with wal-g.sh +MC_POSTGRESQL_WALG_ENV_PATH="/var/run/postgresql/walg.env" -if [ -z ${MC_PGBACKREST_ENABLE+x} ]; then +if [ ! -f "${MC_POSTGRESQL_WALG_CONF_PATH}" ]; then + echo "PostgreSQL WAL-G configuration file does not exist in ${MC_POSTGRESQL_WALG_CONF_PATH}" + exit 1 +fi +if [ ! -f "${MC_POSTGRESQL_WALG_ENV_PATH}" ]; then + echo "PostgreSQL WAL-G environment file does not exist in ${MC_POSTGRESQL_WALG_ENV_PATH}" + exit 1 +fi + +if [ -z ${MC_WALG_ENABLE+x} ]; then - echo "PgBackRest is disabled." + echo "WAL-G is disabled." - cat > "${MC_POSTGRESQL_PGBACKREST_CONF_PATH}" << EOF + cat > "${MC_POSTGRESQL_WALG_CONF_PATH}" << EOF # # Auto-generated, please don't edit! # @@ -44,44 +54,79 @@ if [ -z ${MC_PGBACKREST_ENABLE+x} ]; then archive_mode = off EOF - cat > "${MC_BACKREST_CONF_D_S3_CONF_PATH}" << EOF + cat > "${MC_POSTGRESQL_WALG_ENV_PATH}" << EOF # # Auto-generated, please don't edit! # -# S3 archiving disabled +# WAL-G is disabled. EOF else - echo "PgBackRest is enabled." + echo "WAL-G is enabled." - cat > "${MC_POSTGRESQL_PGBACKREST_CONF_PATH}" << EOF + cat > "${MC_POSTGRESQL_WALG_CONF_PATH}" << EOF # # Auto-generated, please don't edit! # -# Back up with PgBackRest -# (stanzas of all users of postgresql-base are called "main") +# Back up with WAL-G archive_mode = on -archive_command = 'pgbackrest --stanza=main archive-push %p' +archive_command = '/opt/postgresql-base/bin/wal-g.sh wal-push %p' EOF - cat > "${MC_BACKREST_CONF_D_S3_CONF_PATH}" << EOF + if [[ ! "${MC_WALG_S3_BUCKET_PREFIX}" == "s3://"* ]]; then + echo "S3 bucket + prefix must start with 's3://': ${MC_WALG_S3_BUCKET_PREFIX}" + exit 1 + fi + + if [ "${MC_WALG_S3_BUCKET_PREFIX: -1}" == "/" ]; then + echo "S3 bucket + prefix can't end with a slash: ${MC_WALG_S3_BUCKET_PREFIX}" + exit 1 + fi + + if [ -z ${MC_WALG_S3_ENDPOINT+x} ]; then + MC_WALG_S3_ENDPOINT="https://s3.amazonaws.com" + fi + + if [[ ! "${MC_WALG_S3_ENDPOINT}" == "http"* ]]; then + echo "S3 endpoint must be 'https://' or 'http://': ${MC_WALG_S3_ENDPOINT}" + exit 1 + fi + + if [ -z ${MC_WALG_S3_STORAGE_CLASS+x} ]; then + MC_WALG_S3_STORAGE_CLASS="STANDARD" + fi + if [ -z ${MC_WALG_S3_FORCE_PATH_STYLE+x} ]; then + MC_WALG_S3_FORCE_PATH_STYLE="false" + fi + if [ -z ${MC_WALG_S3_USE_LIST_OBJECTS_V1+x} ]; then + MC_WALG_S3_USE_LIST_OBJECTS_V1="false" + fi + + cat > "${MC_POSTGRESQL_WALG_ENV_PATH}" << EOF # # Auto-generated, please don't edit! # -# S3 credentials -[global] -repo1-retention-full=${MC_PGBACKREST_RETENTION_FULL} -repo1-s3-endpoint=${MC_PGBACKREST_S3_ENDPOINT} -repo1-s3-bucket=${MC_PGBACKREST_S3_BUCKET} -repo1-storage-verify-tls=${MC_PGBACKREST_S3_VERIFY_TLS} -repo1-s3-key=${MC_PGBACKREST_S3_KEY} -repo1-s3-key-secret=${MC_PGBACKREST_S3_KEY_SECRET} -repo1-s3-region=${MC_PGBACKREST_S3_REGION} -repo1-path=${MC_PGBACKREST_S3_PATH} +# Keep up to 6 delta backups +export WALG_DELTA_MAX_STEPS=6 + +export AWS_ACCESS_KEY_ID=${MC_WALG_S3_ACCESS_KEY_ID} +export AWS_SECRET_ACCESS_KEY=${MC_WALG_S3_SECRET_ACCESS_KEY} +export AWS_REGION=${MC_WALG_S3_REGION} +export AWS_ENDPOINT=${MC_WALG_S3_ENDPOINT} +export WALG_S3_PREFIX=${MC_WALG_S3_BUCKET_PREFIX} +export WALG_S3_STORAGE_CLASS=${MC_WALG_S3_STORAGE_CLASS} +export AWS_S3_FORCE_PATH_STYLE=${MC_WALG_S3_FORCE_PATH_STYLE} +export S3_USE_LIST_OBJECTS_V1=${MC_WALG_S3_USE_LIST_OBJECTS_V1} EOF + if [ ! -z ${MC_WALG_S3_CA_CERT_BASE64+x} ]; then + MC_WALG_S3_CA_CERT_FILE=/var/run/postgresql/walg.cert + echo "${MC_WALG_S3_CA_CERT_BASE64}" | base64 -d > "${MC_WALG_S3_CA_CERT_FILE}" + echo "export WALG_S3_CA_CERT_FILE=${MC_WALG_S3_CA_CERT_FILE}" >> \ + "${MC_POSTGRESQL_WALG_ENV_PATH}" + fi fi diff --git a/apps/postgresql-base/bin/wal-g.sh b/apps/postgresql-base/bin/wal-g.sh new file mode 100755 index 0000000000..24a8c458fa --- /dev/null +++ b/apps/postgresql-base/bin/wal-g.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Wrapper around "wal-g" binary which reads pre-configured credentials +# + +set -u +set -e + +# Keep in sync with generate_runtime_config.sh +MC_POSTGRESQL_WALG_ENV_PATH="/var/run/postgresql/walg.env" + +if [ ! -f "${MC_POSTGRESQL_WALG_ENV_PATH}" ]; then + echo "WAL-G environment file ${MC_POSTGRESQL_WALG_ENV_PATH} does not exist;" + echo "maybe you haven't run PostgreSQL yet?" + exit 1 +fi + +source /var/run/postgresql/walg.env + +exec /usr/bin/_wal-g "$@" diff --git a/apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore b/apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore deleted file mode 100644 index 5e7d2734cf..0000000000 --- a/apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/apps/postgresql-base/etc/pgbackrest/pgbackrest.conf b/apps/postgresql-base/etc/pgbackrest/pgbackrest.conf deleted file mode 100644 index 40d4eb09bf..0000000000 --- a/apps/postgresql-base/etc/pgbackrest/pgbackrest.conf +++ /dev/null @@ -1,30 +0,0 @@ -[main] -pg1-path=/var/lib/postgresql/13/main - - -[global] -process-max=4 -start-fast=y -delta=y - -# Use fastest compression because the database is huge and network (plus the -# storage space on the other end) is plenty -compress-type=lz4 -compress-level=1 -buffer-size=16777216 - -# Docs say that async archiving is faster -archive-async=y -archive-push-queue-max=16GB -spool-path=/var/spool/pgbackrest - -# Log only to the console, i.e. Docker's log -log-level-console=detail -log-level-file=off - -# Write WALs and backups to S3 -repo1-type=s3 - -# S3 configuration read from environent variables will be in conf.d/s3.conf; -# FIXME WARN: configuration file contains command-line only option 'config-include-path' -config-include-path=/etc/pgbackrest/conf.d diff --git a/apps/postgresql-base/etc/postgresql/13/main/postgresql.conf b/apps/postgresql-base/etc/postgresql/13/main/postgresql.conf index cc472e0a8e..56c54ce8b6 100644 --- a/apps/postgresql-base/etc/postgresql/13/main/postgresql.conf +++ b/apps/postgresql-base/etc/postgresql/13/main/postgresql.conf @@ -89,5 +89,5 @@ max_locks_per_transaction = 1024 # Include memory configuration (updated on every run in wrapper script) include '/var/run/postgresql/postgresql-memory.conf' -# Include PgBackRest configuration (updated on every run in wrapper script) -include '/var/run/postgresql/postgresql-pgbackrest.conf' +# Include WAL-G configuration (updated on every run in wrapper script) +include '/var/run/postgresql/postgresql-walg.conf' diff --git a/apps/postgresql-server/Dockerfile b/apps/postgresql-server/Dockerfile index 617fccdf62..f173c5f139 100644 --- a/apps/postgresql-server/Dockerfile +++ b/apps/postgresql-server/Dockerfile @@ -45,7 +45,10 @@ COPY pgmigrate/ /opt/postgresql-server/pgmigrate # container start, Docker will copy the files from the container to the volume USER postgres -RUN /opt/postgresql-server/bin/initialize_db.sh +RUN \ + unset PGHOST PGPORT && \ + /opt/postgresql-server/bin/initialize_db.sh && \ + true # Remove the init script so that someone doesn't accidentally run it in # production @@ -60,8 +63,6 @@ ENV \ PATH="/opt/postgresql-server/bin:${PATH}" \ # # Make sure that we can connect via "psql" without sudoing into "postgres" user - PGHOST=localhost \ - PGPORT=5432 \ PGUSER=mediacloud \ PGPASSWORD=mediacloud \ PGDATABASE=mediacloud @@ -70,4 +71,4 @@ ENV \ VOLUME /var/lib/postgresql/ # Use our own wrapper script which runs schema upgrades first -CMD ["/opt/postgresql-server/bin/postgresql.sh"] \ No newline at end of file +CMD ["/opt/postgresql-server/bin/postgresql.sh"] diff --git a/apps/postgresql-server/bin/apply_migrations.sh b/apps/postgresql-server/bin/apply_migrations.sh index 54b862ebed..2a406ce8f2 100755 --- a/apps/postgresql-server/bin/apply_migrations.sh +++ b/apps/postgresql-server/bin/apply_migrations.sh @@ -23,7 +23,11 @@ fi # Start PostgreSQL on a temporary port "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ - -o "-c config_file=${MC_POSTGRESQL_CONF_PATH} -p ${TEMP_PORT}" \ + -o "\ + -c config_file=${MC_POSTGRESQL_CONF_PATH} \ + -p ${TEMP_PORT} \ + -c archive_mode=off \ + " \ -D "${MC_POSTGRESQL_DATA_DIR}" \ -t "${PGCTL_START_TIMEOUT}" \ -w \ diff --git a/apps/postgresql-server/docker-compose.tests.yml b/apps/postgresql-server/docker-compose.tests.yml index 9c0b0f2189..4a9d60ae56 100644 --- a/apps/postgresql-server/docker-compose.tests.yml +++ b/apps/postgresql-server/docker-compose.tests.yml @@ -1,4 +1,4 @@ -# Used for manually testing PgBackRest +# Used for manually testing WAL-G version: "3.7" @@ -11,15 +11,16 @@ services: expose: - 5432 environment: - MC_PGBACKREST_ENABLE: "${MC_PGBACKREST_ENABLE}" - MC_PGBACKREST_RETENTION_FULL: "${MC_PGBACKREST_RETENTION_FULL}" - MC_PGBACKREST_S3_ENDPOINT: "${MC_PGBACKREST_S3_ENDPOINT}" - MC_PGBACKREST_S3_BUCKET: "${MC_PGBACKREST_S3_BUCKET}" - MC_PGBACKREST_S3_VERIFY_TLS: "${MC_PGBACKREST_S3_VERIFY_TLS}" - MC_PGBACKREST_S3_KEY: "${MC_PGBACKREST_S3_KEY}" - MC_PGBACKREST_S3_KEY_SECRET: "${MC_PGBACKREST_S3_KEY_SECRET}" - MC_PGBACKREST_S3_REGION: "${MC_PGBACKREST_S3_REGION}" - MC_PGBACKREST_S3_PATH: "${MC_PGBACKREST_S3_PATH}" + MC_WALG_ENABLE: "${MC_WALG_ENABLE}" + MC_WALG_S3_ACCESS_KEY_ID: "${MC_WALG_S3_ACCESS_KEY_ID}" + MC_WALG_S3_SECRET_ACCESS_KEY: "${MC_WALG_S3_SECRET_ACCESS_KEY}" + MC_WALG_S3_REGION: "${MC_WALG_S3_REGION}" + MC_WALG_S3_BUCKET_PREFIX: "${MC_WALG_S3_BUCKET_PREFIX}" + # MC_WALG_S3_ENDPOINT: "${MC_WALG_S3_ENDPOINT}" + # MC_WALG_S3_STORAGE_CLASS: "${MC_WALG_S3_STORAGE_CLASS}" + # MC_WALG_S3_FORCE_PATH_STYLE: "${MC_WALG_S3_FORCE_PATH_STYLE}" + # MC_WALG_S3_USE_LIST_OBJECTS_V1: "${MC_WALG_S3_USE_LIST_OBJECTS_V1}" + # MC_WALG_S3_CA_CERT_BASE64: "${MC_WALG_S3_CA_CERT_BASE64}" volumes: - type: bind source: ./bin/ @@ -33,7 +34,3 @@ services: - type: bind source: ./../postgresql-base/etc/postgresql/ target: /etc/postgresql/ - # Mounting PgBackRest's configuration directory too - - type: bind - source: ./../postgresql-base/etc/pgbackrest/pgbackrest.conf - target: /etc/pgbackrest/pgbackrest.conf diff --git a/apps/temporal-postgresql/Dockerfile b/apps/temporal-postgresql/Dockerfile index 54521994fd..48a4cf7c4c 100644 --- a/apps/temporal-postgresql/Dockerfile +++ b/apps/temporal-postgresql/Dockerfile @@ -66,7 +66,10 @@ USER postgres # schema # If a new empty volume gets mounted to /var/lib/postgresql/ upon # container start, Docker will copy the files from the container to the volume -RUN /opt/temporal-postgresql/bin/initialize_schema.sh +RUN \ + unset PGHOST PGPORT && \ + /opt/temporal-postgresql/bin/initialize_schema.sh && \ + true # Remove the init script, Temporal server and configuration so that someone # doesn't accidentally run it in production @@ -85,8 +88,6 @@ ENV \ PATH="/opt/temporal-postgresql/bin:${PATH}" \ # # Make sure that we can connect via "psql" without sudoing into "postgres" user - PGHOST=localhost \ - PGPORT=5432 \ PGUSER=temporal \ PGPASSWORD=temporal \ PGDATABASE=temporal diff --git a/apps/temporal-postgresql/bin/apply_migrations.sh b/apps/temporal-postgresql/bin/apply_migrations.sh index 27c5fa233f..6676f32b06 100755 --- a/apps/temporal-postgresql/bin/apply_migrations.sh +++ b/apps/temporal-postgresql/bin/apply_migrations.sh @@ -16,7 +16,11 @@ PGCTL_START_TIMEOUT=3600 # Start PostgreSQL on a temporary port "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ - -o "-c config_file=${MC_POSTGRESQL_CONF_PATH} -p ${TEMP_PORT}" \ + -o "\ + -c config_file=${MC_POSTGRESQL_CONF_PATH} \ + -p ${TEMP_PORT} \ + -c archive_mode=off \ + " \ -D "${MC_POSTGRESQL_DATA_DIR}" \ -t "${PGCTL_START_TIMEOUT}" \ -w \ diff --git a/doc/postgresql_pgbackrest.markdown b/doc/postgresql_pgbackrest.markdown deleted file mode 100644 index fe3966b75b..0000000000 --- a/doc/postgresql_pgbackrest.markdown +++ /dev/null @@ -1,90 +0,0 @@ -# PgBackRest - - -## Usage - -A stanza is the [configuration for a PostgreSQL database cluster that defines where it is located, how it will be backed up, archiving options, etc.](https://pgbackrest.org/user-guide.html) - -Create stanza: - -```bash -pgbackrest --stanza=main stanza-create -``` - -Check stanza: - -```bash -pgbackrest --stanza=main check -``` - -Do a full backup: - -```bash -pgbackrest --stanza=main --type=full backup -``` - -Do an incremental backup: - -```bash -pgbackrest --stanza=main --type=incr backup -``` - - -## Creating S3 bucket - -1. Create a bucket: - 1. Name the bucket appropriately, e.g. `mediacloud-pgbackrest` or `mediacloud-pgbackrest-test` - 2. Make sure public access to the bucket is off - 3. Add a tag with name `project` and value `mediacloud-backrest` - -2. Create (update) IAM policy: - 1. Name the policy appropriately, e.g. `mediacloud-pgbackrest` or `mediacloud-pgbackrest-test`; - 2. Allow access to the previously created bucket, e.g.: - - ```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "1", - "Effect": "Allow", - "Action": [ - "s3:ListAllMyBuckets" - ], - "Resource": [ - "arn:aws:s3:::*" - ] - }, - { - "Sid": "2", - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::mediacloud-pgbackrest-test" - ] - }, - { - "Sid": "3", - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject" - ], - "Resource": [ - "arn:aws:s3:::mediacloud-pgbackrest-test/*" - ] - } - ] - } - ``` - - 3. Add a tag with name `project` and value `mediacloud-backrest` - -3. Create (update) IAM user: - 1. Name the user appropriately, e.g. `mediacloud-pgbackrest` or `mediacloud-pgbackrest-test` - 2. Enable only the programmatic access - 3. Attach the newly created / updated policy to the user - 4. Add a tag with name `project` and value `mediacloud-backrest` diff --git a/doc/postgresql_walg.markdown b/doc/postgresql_walg.markdown new file mode 100644 index 0000000000..1c7d7483f0 --- /dev/null +++ b/doc/postgresql_walg.markdown @@ -0,0 +1,94 @@ +# PostgreSQL backups with WAL-G + + +## Usage + +(To be executed from within a running `postgresql-base` container, e.g. `postgresql-server` or `temporal-postgresql`.) + +Make a full backup, or a delta backup if a full backup exists and not more than `WALG_DELTA_MAX_STEPS` delta backups already exist + +```bash +# No slash at the end! +wal-g.sh backup-push /var/lib/postgresql/13/main +``` + +Verify WAL segment storage: + +```bash +wal-g.sh wal-verify timeline +# and / or +wal-g.sh wal-verify integrity +``` + +Leave only two full backups, deleting the rest (including old WALs too): + +```bash +# Dry runs by default; add `--confirm` flag to actually delete the old backups +wal-g.sh delete retain FULL 2 +``` + + +## S3 bucket configuration + +1. Create a bucket: + 1. Name the bucket appropriately, e.g. `mediacloud-postgresql-wal-backups` or `mediacloud-postgresql-wal-backups-test` + 2. Make sure public access to the bucket is off + 3. Add a tag with name `project` and value `mediacloud-postgresql-wal-backups` + +2. Create (update) IAM policy: + 1. Name the policy appropriately, e.g. `mediacloud-postgresql-wal-backups` or `mediacloud-postgresql-wal-backups-test`; + 2. Allow access to the previously created bucket, e.g.: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "1", + "Effect": "Allow", + "Action": [ + "s3:ListAllMyBuckets" + ], + "Resource": [ + "arn:aws:s3:::*" + ] + }, + { + "Sid": "2", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::mediacloud-postgresql-wal-backups-test" + ] + }, + { + "Sid": "3", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::mediacloud-postgresql-wal-backups-test/*" + ] + } + ] + } + ``` + + 3. Add a tag with name `project` and value `mediacloud-postgresql-wal-backups` + 4. Describe the project as *TEST ACCOUNT - WAL backups of postgresql-server, temporal-postgresql and possibly others* + +3. Create (update) IAM user: + 1. Name the user appropriately, e.g. `mediacloud-postgresql-wal-backups` or `mediacloud-postgresql-wal-backups-test` + 2. Enable only the programmatic access + 3. Attach the newly created / updated policy to the user + 4. Add a tag with name `project` and value `mediacloud-postgresql-wal-backups` + + +## Links + +*