Install + set up PgBackRest in postgresql-base

apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml

@@ -54,5 +54,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/common/docker-compose.tests.yml

@@ -97,8 +97,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     solr-shard-01:
         image: gcr.io/mcback/solr-shard:latest

apps/crawler-ap/docker-compose.tests.yml

@@ -93,8 +93,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     rabbitmq-server:
         image: gcr.io/mcback/rabbitmq-server:latest

apps/crawler-fetcher/docker-compose.tests.yml

@@ -56,8 +56,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     rabbitmq-server:
         image: gcr.io/mcback/rabbitmq-server:latest

apps/crawler-provider/docker-compose.tests.yml

@@ -49,5 +49,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/create-missing-partitions/docker-compose.tests.yml

@@ -43,5 +43,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml

@@ -46,5 +46,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-generate-media-health/docker-compose.tests.yml

@@ -52,5 +52,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-generate-user-summary/docker-compose.tests.yml

@@ -46,5 +46,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-print-long-running-job-states/docker-compose.tests.yml

@@ -46,5 +46,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-refresh-stats/docker-compose.tests.yml

@@ -46,5 +46,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-rescrape-due-media/docker-compose.tests.yml

@@ -47,8 +47,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     rabbitmq-server:
         image: gcr.io/mcback/rabbitmq-server:latest

apps/cron-rescraping-changes/docker-compose.tests.yml

@@ -46,5 +46,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-set-media-primary-language/docker-compose.tests.yml

@@ -52,5 +52,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/cron-set-media-subject-country/docker-compose.tests.yml

@@ -52,5 +52,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/docker-compose.dist.yml

@@ -1344,6 +1344,47 @@ services:
         init: true
         networks:
             - default
+        environment:
+
+            # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest
+            # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown
+            MC_PGBACKREST_ENABLE: "1"
+
+            # (required if PgBackRest is enabled) Full backup retention count/time:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full
+            MC_PGBACKREST_RETENTION_FULL: "2"
+
+            # (required if PgBackRest is enabled) S3 repository endpoint:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint
+            MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com"
+
+            # (required if PgBackRest is enabled) S3 repository bucket:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket
+            MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example"
+
+            # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"):
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls
+            MC_PGBACKREST_S3_VERIFY_TLS: "y"
+
+            # (required if PgBackRest is enabled) S3 repository access key (Access Key ID):
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key
+            MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE"
+
+            # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key):
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret
+            MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY"
+
+            # (required if PgBackRest is enabled) S3 repository region:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region
+            MC_PGBACKREST_S3_REGION: "us-east-1"
+
+            # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-path
+            #
+            # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST!
+            #
+            MC_PGBACKREST_S3_PATH: "/postgresql-server"
+
         expose:
             - 5432
         # Allow up to 5 minutes for PostgreSQL to stop so that it manages to
@@ -1841,10 +1882,58 @@ services:
         init: true
         networks:
             - default
+        environment:
+
+            # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest
+            # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown
+            MC_PGBACKREST_ENABLE: "1"
+
+            # (required if PgBackRest is enabled) Full backup retention count/time:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full
+            MC_PGBACKREST_RETENTION_FULL: "2"
+
+            # (required if PgBackRest is enabled) S3 repository endpoint:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint
+            MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com"
+
+            # (required if PgBackRest is enabled) S3 repository bucket:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket
+            MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example"
+
+            # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"):
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls
+            MC_PGBACKREST_S3_VERIFY_TLS: "y"
+
+            # (required if PgBackRest is enabled) S3 repository access key (Access Key ID):
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key
+            MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE"
+
+            # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key):
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret
+            MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY"
+
+            # (required if PgBackRest is enabled) S3 repository region:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region
+            MC_PGBACKREST_S3_REGION: "us-east-1"
+
+            # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash:
+            # https://pgbackrest.org/configuration.html#section-repository/option-repo-path
+            #
+            # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST!
+            #
+            MC_PGBACKREST_S3_PATH: "/temporal-postgresql"
+
         expose:
             - 5432
         volumes:
             - vol_temporal_postgresql_data:/var/lib/postgresql/
+
+            # Provide container with more shared memory than is the default:
+            - type: tmpfs
+              target: /dev/shm
+              tmpfs:
+                size: 4294967296   # 4 GB
+
         deploy:
             <<: *endpoint-mode-dnsrr
             placement:

apps/dump-table/docker-compose.tests.yml

@@ -43,5 +43,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/export-tables-to-backup-crawler/docker-compose.tests.yml

@@ -49,5 +49,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/extract-and-vector/docker-compose.tests.yml

@@ -69,8 +69,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     rabbitmq-server:
         image: gcr.io/mcback/rabbitmq-server:latest

apps/facebook-fetch-story-stats/docker-compose.tests.yml

@@ -53,8 +53,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     rabbitmq-server:
         image: gcr.io/mcback/rabbitmq-server:latest

apps/import-solr-data-for-testing/docker-compose.tests.yml

@@ -50,8 +50,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     solr-shard-01:
         image: gcr.io/mcback/solr-shard:latest

apps/import-solr-data/docker-compose.tests.yml

@@ -75,8 +75,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     solr-shard-01:
         image: gcr.io/mcback/solr-shard:latest

apps/import-stories-feedly/docker-compose.tests.yml

@@ -52,5 +52,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/import-stories-scrapehtml/docker-compose.tests.yml

@@ -52,5 +52,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/munin-cron/docker-compose.tests.yml

@@ -63,8 +63,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     solr-shard-01:
         image: gcr.io/mcback/solr-shard:latest

apps/munin-httpd/docker-compose.tests.yml

@@ -90,8 +90,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     solr-shard-01:
         image: gcr.io/mcback/solr-shard:latest

apps/munin-node/docker-compose.tests.yml

@@ -43,8 +43,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     solr-shard-01:
         image: gcr.io/mcback/solr-shard:latest

apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml

@@ -52,5 +52,5 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/

apps/podcast-transcribe-episode/docker-compose.tests.yml

@@ -68,8 +68,8 @@ services:
               source: ./../postgresql-server/pgmigrate/
               target: /opt/postgresql-server/pgmigrate/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     rabbitmq-server:
         image: gcr.io/mcback/rabbitmq-server:latest
@@ -121,8 +121,8 @@ services:
               source: ./../temporal-postgresql/bin/
               target: /opt/temporal-postgresql/bin/
             - type: bind
-              source: ./../postgresql-base/conf/
-              target: /etc/postgresql/13/main/
+              source: ./../postgresql-base/etc/postgresql/
+              target: /etc/postgresql/
 
     temporal-elasticsearch:
         image: gcr.io/mcback/temporal-elasticsearch:latest