From b2fdad9be22d143ef2eea89af2f4704ce2bc3eac Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Thu, 7 Oct 2021 21:57:07 +0100 Subject: [PATCH 1/3] Increase /dev/shm size for temporal-postgresql --- apps/docker-compose.dist.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/docker-compose.dist.yml b/apps/docker-compose.dist.yml index 33d739781e..5f9f6d3690 100644 --- a/apps/docker-compose.dist.yml +++ b/apps/docker-compose.dist.yml @@ -1845,6 +1845,13 @@ services: - 5432 volumes: - vol_temporal_postgresql_data:/var/lib/postgresql/ + + # Provide container with more shared memory than is the default: + - type: tmpfs + target: /dev/shm + tmpfs: + size: 4294967296 # 4 GB + deploy: <<: *endpoint-mode-dnsrr placement: From 86bfa2821b2a6604f112d8356d0fffdf158340e8 Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Thu, 7 Oct 2021 23:22:45 +0100 Subject: [PATCH 2/3] Install PgBackRest into postgresql-base --- .../docker-compose.tests.yml | 4 +- apps/common/docker-compose.tests.yml | 4 +- apps/crawler-ap/docker-compose.tests.yml | 4 +- apps/crawler-fetcher/docker-compose.tests.yml | 4 +- .../crawler-provider/docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- apps/docker-compose.dist.yml | 82 +++++++++++++++++ apps/dump-table/docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../import-solr-data/docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- apps/munin-cron/docker-compose.tests.yml | 4 +- apps/munin-httpd/docker-compose.tests.yml | 4 +- apps/munin-node/docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 8 +- apps/postgresql-base/Dockerfile | 47 ++++++++-- .../bin/generate_runtime_config.sh | 87 ++++++++++++++++++ apps/postgresql-base/bin/postgresql.sh | 2 +- .../bin/update_memory_config.sh | 20 ----- .../etc/pgbackrest/conf.d/.gitignore | 4 + .../etc/pgbackrest/pgbackrest.conf | 30 +++++++ .../postgresql/13/main}/environment | 0 .../postgresql/13/main}/pg_ctl.conf | 0 .../postgresql/13/main}/pg_hba.conf | 0 .../postgresql/13/main}/pg_ident.conf | 0 .../postgresql/13/main}/postgresql.conf | 3 + .../postgresql/13/main}/start.conf | 0 apps/postgresql-server/bin/initialize_db.sh | 2 +- apps/postgresql-server/bin/postgresql.sh | 2 +- .../docker-compose.tests.yml | 39 ++++++++ .../bin/postgresql_upgrade.py | 2 +- .../docker-compose.tests.yml | 4 +- apps/rescrape-media/docker-compose.tests.yml | 4 +- .../bin/initialize_schema.sh | 2 +- apps/temporal-postgresql/bin/postgresql.sh | 2 +- apps/temporal-server/docker-compose.tests.yml | 4 +- apps/tools/docker-compose.tests.yml | 4 +- apps/topics-base/docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- apps/topics-map/docker-compose.tests.yml | 4 +- apps/topics-mine/docker-compose.tests.yml | 4 +- apps/topics-snapshot/docker-compose.tests.yml | 4 +- apps/webapp-api/docker-compose.tests.yml | 4 +- apps/webapp-httpd/docker-compose.tests.yml | 4 +- .../docker-compose.tests.yml | 4 +- doc/postgresql_pgbackrest.markdown | 88 +++++++++++++++++++ 62 files changed, 465 insertions(+), 119 deletions(-) create mode 100755 apps/postgresql-base/bin/generate_runtime_config.sh delete mode 100755 apps/postgresql-base/bin/update_memory_config.sh create mode 100644 apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore create mode 100644 apps/postgresql-base/etc/pgbackrest/pgbackrest.conf rename apps/postgresql-base/{conf => etc/postgresql/13/main}/environment (100%) rename apps/postgresql-base/{conf => etc/postgresql/13/main}/pg_ctl.conf (100%) rename apps/postgresql-base/{conf => etc/postgresql/13/main}/pg_hba.conf (100%) rename apps/postgresql-base/{conf => etc/postgresql/13/main}/pg_ident.conf (100%) rename apps/postgresql-base/{conf => etc/postgresql/13/main}/postgresql.conf (93%) rename apps/postgresql-base/{conf => etc/postgresql/13/main}/start.conf (100%) create mode 100644 apps/postgresql-server/docker-compose.tests.yml create mode 100644 doc/postgresql_pgbackrest.markdown diff --git a/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml b/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml index ce2028d844..95d7696d9e 100644 --- a/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml +++ b/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml @@ -54,5 +54,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/common/docker-compose.tests.yml b/apps/common/docker-compose.tests.yml index 512d478d8b..2f88641ce5 100644 --- a/apps/common/docker-compose.tests.yml +++ b/apps/common/docker-compose.tests.yml @@ -97,8 +97,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/crawler-ap/docker-compose.tests.yml b/apps/crawler-ap/docker-compose.tests.yml index d43c351f3c..59c012115e 100644 --- a/apps/crawler-ap/docker-compose.tests.yml +++ b/apps/crawler-ap/docker-compose.tests.yml @@ -93,8 +93,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/crawler-fetcher/docker-compose.tests.yml b/apps/crawler-fetcher/docker-compose.tests.yml index ae3ad80d71..704c6e4e53 100644 --- a/apps/crawler-fetcher/docker-compose.tests.yml +++ b/apps/crawler-fetcher/docker-compose.tests.yml @@ -56,8 +56,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/crawler-provider/docker-compose.tests.yml b/apps/crawler-provider/docker-compose.tests.yml index 6145990302..5670c466ac 100644 --- a/apps/crawler-provider/docker-compose.tests.yml +++ b/apps/crawler-provider/docker-compose.tests.yml @@ -49,5 +49,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/create-missing-partitions/docker-compose.tests.yml b/apps/create-missing-partitions/docker-compose.tests.yml index d9db7c2e53..e2254584aa 100644 --- a/apps/create-missing-partitions/docker-compose.tests.yml +++ b/apps/create-missing-partitions/docker-compose.tests.yml @@ -43,5 +43,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml b/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml index 4f554d3ab4..0061507da7 100644 --- a/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml +++ b/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml @@ -46,5 +46,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-generate-media-health/docker-compose.tests.yml b/apps/cron-generate-media-health/docker-compose.tests.yml index 56d2a317d4..5b63b504d2 100644 --- a/apps/cron-generate-media-health/docker-compose.tests.yml +++ b/apps/cron-generate-media-health/docker-compose.tests.yml @@ -52,5 +52,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-generate-user-summary/docker-compose.tests.yml b/apps/cron-generate-user-summary/docker-compose.tests.yml index bec9e68355..c89cdfab0c 100644 --- a/apps/cron-generate-user-summary/docker-compose.tests.yml +++ b/apps/cron-generate-user-summary/docker-compose.tests.yml @@ -46,5 +46,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-print-long-running-job-states/docker-compose.tests.yml b/apps/cron-print-long-running-job-states/docker-compose.tests.yml index 24431fde60..854dd85898 100644 --- a/apps/cron-print-long-running-job-states/docker-compose.tests.yml +++ b/apps/cron-print-long-running-job-states/docker-compose.tests.yml @@ -46,5 +46,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-refresh-stats/docker-compose.tests.yml b/apps/cron-refresh-stats/docker-compose.tests.yml index f381c76f5c..09c716987d 100644 --- a/apps/cron-refresh-stats/docker-compose.tests.yml +++ b/apps/cron-refresh-stats/docker-compose.tests.yml @@ -46,5 +46,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-rescrape-due-media/docker-compose.tests.yml b/apps/cron-rescrape-due-media/docker-compose.tests.yml index 5163cd2fc8..1bbd6949ca 100644 --- a/apps/cron-rescrape-due-media/docker-compose.tests.yml +++ b/apps/cron-rescrape-due-media/docker-compose.tests.yml @@ -47,8 +47,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/cron-rescraping-changes/docker-compose.tests.yml b/apps/cron-rescraping-changes/docker-compose.tests.yml index 729095d782..46e5130046 100644 --- a/apps/cron-rescraping-changes/docker-compose.tests.yml +++ b/apps/cron-rescraping-changes/docker-compose.tests.yml @@ -46,5 +46,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-set-media-primary-language/docker-compose.tests.yml b/apps/cron-set-media-primary-language/docker-compose.tests.yml index e152c23ad5..c546eb5923 100644 --- a/apps/cron-set-media-primary-language/docker-compose.tests.yml +++ b/apps/cron-set-media-primary-language/docker-compose.tests.yml @@ -52,5 +52,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/cron-set-media-subject-country/docker-compose.tests.yml b/apps/cron-set-media-subject-country/docker-compose.tests.yml index 2a2872d961..73a4de27d4 100644 --- a/apps/cron-set-media-subject-country/docker-compose.tests.yml +++ b/apps/cron-set-media-subject-country/docker-compose.tests.yml @@ -52,5 +52,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/docker-compose.dist.yml b/apps/docker-compose.dist.yml index 5f9f6d3690..599685ac06 100644 --- a/apps/docker-compose.dist.yml +++ b/apps/docker-compose.dist.yml @@ -1344,6 +1344,47 @@ services: init: true networks: - default + environment: + + # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest + # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown + MC_PGBACKREST_ENABLE: "1" + + # (required if PgBackRest is enabled) Full backup retention count/time: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full + MC_PGBACKREST_RETENTION_FULL: "2" + + # (required if PgBackRest is enabled) S3 repository endpoint: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint + MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com" + + # (required if PgBackRest is enabled) S3 repository bucket: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket + MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example" + + # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"): + # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls + MC_PGBACKREST_S3_VERIFY_TLS: "y" + + # (required if PgBackRest is enabled) S3 repository access key (Access Key ID): + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key + MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE" + + # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key): + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret + MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY" + + # (required if PgBackRest is enabled) S3 repository region: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region + MC_PGBACKREST_S3_REGION: "us-east-1" + + # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-path + # + # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST! + # + MC_PGBACKREST_S3_PATH: "/postgresql-server" + expose: - 5432 # Allow up to 5 minutes for PostgreSQL to stop so that it manages to @@ -1841,6 +1882,47 @@ services: init: true networks: - default + environment: + + # (optional) Set to "1" (string "1", not integer 1!) to enable PgBackRest + # backups to S3; for more information, refer to doc/postgresql_pgbackrest.markdown + MC_PGBACKREST_ENABLE: "1" + + # (required if PgBackRest is enabled) Full backup retention count/time: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-retention-full + MC_PGBACKREST_RETENTION_FULL: "2" + + # (required if PgBackRest is enabled) S3 repository endpoint: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-endpoint + MC_PGBACKREST_S3_ENDPOINT: "s3.amazonaws.com" + + # (required if PgBackRest is enabled) S3 repository bucket: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-bucket + MC_PGBACKREST_S3_BUCKET: "mediacloud-pgbackrest-example" + + # (required if PgBackRest is enabled) Whether or not to verify storage's TLS certificates ("y" / "n"): + # https://pgbackrest.org/configuration.html#section-repository/option-repo-storage-verify-tls + MC_PGBACKREST_S3_VERIFY_TLS: "y" + + # (required if PgBackRest is enabled) S3 repository access key (Access Key ID): + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key + MC_PGBACKREST_S3_KEY: "AKIAIOSFODNN7EXAMPLE" + + # (required if PgBackRest is enabled) S3 repository secret access key (Secret Access Key): + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-key-secret + MC_PGBACKREST_S3_KEY_SECRET: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY" + + # (required if PgBackRest is enabled) S3 repository region: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-s3-region + MC_PGBACKREST_S3_REGION: "us-east-1" + + # (required if PgBackRest is enabled) Prefix path on S3 bucket; must start with a slash: + # https://pgbackrest.org/configuration.html#section-repository/option-repo-path + # + # MAKE SURE IT'S DIFFERENT FROM OTHER USERS OF PGBACKREST! + # + MC_PGBACKREST_S3_PATH: "/temporal-postgresql" + expose: - 5432 volumes: diff --git a/apps/dump-table/docker-compose.tests.yml b/apps/dump-table/docker-compose.tests.yml index 5f53609d5d..c3d804106e 100644 --- a/apps/dump-table/docker-compose.tests.yml +++ b/apps/dump-table/docker-compose.tests.yml @@ -43,5 +43,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/export-tables-to-backup-crawler/docker-compose.tests.yml b/apps/export-tables-to-backup-crawler/docker-compose.tests.yml index 31f916dbdd..ca5d2cc8b0 100644 --- a/apps/export-tables-to-backup-crawler/docker-compose.tests.yml +++ b/apps/export-tables-to-backup-crawler/docker-compose.tests.yml @@ -49,5 +49,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/extract-and-vector/docker-compose.tests.yml b/apps/extract-and-vector/docker-compose.tests.yml index e3c8b5d01f..c5d3b745e0 100644 --- a/apps/extract-and-vector/docker-compose.tests.yml +++ b/apps/extract-and-vector/docker-compose.tests.yml @@ -69,8 +69,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/facebook-fetch-story-stats/docker-compose.tests.yml b/apps/facebook-fetch-story-stats/docker-compose.tests.yml index b5a8d3f727..f457ab13d2 100644 --- a/apps/facebook-fetch-story-stats/docker-compose.tests.yml +++ b/apps/facebook-fetch-story-stats/docker-compose.tests.yml @@ -53,8 +53,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/import-solr-data-for-testing/docker-compose.tests.yml b/apps/import-solr-data-for-testing/docker-compose.tests.yml index 0b7e048a91..8e22225ce8 100644 --- a/apps/import-solr-data-for-testing/docker-compose.tests.yml +++ b/apps/import-solr-data-for-testing/docker-compose.tests.yml @@ -50,8 +50,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/import-solr-data/docker-compose.tests.yml b/apps/import-solr-data/docker-compose.tests.yml index 2527cc48ba..a78868c504 100644 --- a/apps/import-solr-data/docker-compose.tests.yml +++ b/apps/import-solr-data/docker-compose.tests.yml @@ -75,8 +75,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/import-stories-feedly/docker-compose.tests.yml b/apps/import-stories-feedly/docker-compose.tests.yml index 5398a84035..0e963675e5 100644 --- a/apps/import-stories-feedly/docker-compose.tests.yml +++ b/apps/import-stories-feedly/docker-compose.tests.yml @@ -52,5 +52,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/import-stories-scrapehtml/docker-compose.tests.yml b/apps/import-stories-scrapehtml/docker-compose.tests.yml index d42815d441..65598806e0 100644 --- a/apps/import-stories-scrapehtml/docker-compose.tests.yml +++ b/apps/import-stories-scrapehtml/docker-compose.tests.yml @@ -52,5 +52,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/munin-cron/docker-compose.tests.yml b/apps/munin-cron/docker-compose.tests.yml index 095a1a2be0..c6258d9cc1 100644 --- a/apps/munin-cron/docker-compose.tests.yml +++ b/apps/munin-cron/docker-compose.tests.yml @@ -63,8 +63,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/munin-httpd/docker-compose.tests.yml b/apps/munin-httpd/docker-compose.tests.yml index d50207945e..481efeec99 100644 --- a/apps/munin-httpd/docker-compose.tests.yml +++ b/apps/munin-httpd/docker-compose.tests.yml @@ -90,8 +90,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/munin-node/docker-compose.tests.yml b/apps/munin-node/docker-compose.tests.yml index d6d94c6287..d691db8570 100644 --- a/apps/munin-node/docker-compose.tests.yml +++ b/apps/munin-node/docker-compose.tests.yml @@ -43,8 +43,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml b/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml index b9ed705eea..307dcb6548 100644 --- a/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml +++ b/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml @@ -52,5 +52,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/podcast-transcribe-episode/docker-compose.tests.yml b/apps/podcast-transcribe-episode/docker-compose.tests.yml index 55de596db7..ab47916b87 100644 --- a/apps/podcast-transcribe-episode/docker-compose.tests.yml +++ b/apps/podcast-transcribe-episode/docker-compose.tests.yml @@ -68,8 +68,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest @@ -121,8 +121,8 @@ services: source: ./../temporal-postgresql/bin/ target: /opt/temporal-postgresql/bin/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ temporal-elasticsearch: image: gcr.io/mcback/temporal-elasticsearch:latest diff --git a/apps/postgresql-base/Dockerfile b/apps/postgresql-base/Dockerfile index b869d58cf2..6401567c48 100644 --- a/apps/postgresql-base/Dockerfile +++ b/apps/postgresql-base/Dockerfile @@ -6,6 +6,8 @@ FROM gcr.io/mcback/postgresql-repo-base:latest # Install PostgreSQL RUN \ + # FIXME + apt-get -y update && \ if [ "$(dpkg --print-architecture)" = "arm64" ]; then \ echo "Installing Graviton2-optimized PostgreSQL..." && \ /dl_to_stdout.sh "https://github.com/mediacloud/postgresql-aws-graviton2/releases/download/13.3-1.pgdg20.04%2B1/postgresql-13_13.3-1.pgdg20.04+1_arm64.deb" > /var/tmp/postgresql-13.deb && \ @@ -39,6 +41,14 @@ RUN \ fi; \ true +# Install pgBackRest for backing up PostgreSQL +RUN \ + apt-get -y --no-install-recommends install pgbackrest && \ + # Remove default configuration + rm -rf /etc/pgbackrest.conf /etc/pgbackrest/ && \ + # + true + # Make some run directories RUN \ mkdir -p /var/run/postgresql/13-main.pg_stat_tmp && \ @@ -47,19 +57,41 @@ RUN \ # Write our own configuration RUN rm -rf /etc/postgresql/13/main/ -COPY conf/ /etc/postgresql/13/main/ +COPY etc/postgresql/13/main/ /etc/postgresql/13/main/ +COPY etc/pgbackrest/ /etc/pgbackrest/ -# This is where "update_memory_config.sh" script will write its memory settings -# which it will auto-determine from available RAM on every run. RUN \ + # + # This is where "generate_runtime_config.sh" script will write its memory settings + # which it will auto-determine from available RAM on every run. touch /var/run/postgresql/postgresql-memory.conf && \ chown postgres:postgres /var/run/postgresql/postgresql-memory.conf && \ + # + # This is where "generate_runtime_config.sh" script will write PgBackRest-related + # configuration + touch /var/run/postgresql/postgresql-pgbackrest.conf && \ + chown postgres:postgres /var/run/postgresql/postgresql-pgbackrest.conf && \ + # + # We'll write runtime S3 credentials there + chown postgres:postgres /etc/pgbackrest/conf.d/ && \ + # + # Get rid of /var/lib/pgbackrest/ as we won't be using it + rm -rf /var/lib/pgbackrest/ && \ + # + # (Re)-create /var/spool/pgbackrest/ + rm -rf "/var/spool/pgbackrest/" && \ + mkdir -p "/var/spool/pgbackrest/" && \ + chown postgres:postgres "/var/spool/pgbackrest/" && \ + chmod 750 "/var/spool/pgbackrest/" && \ + # true # Copy helper scripts RUN mkdir -p /opt/postgresql-base/ COPY bin/* /opt/postgresql-base/bin/ +ENV PATH="/opt/postgresql-base/bin:${PATH}" + USER postgres RUN \ @@ -68,10 +100,9 @@ RUN \ # locale, doesn't use checksums etc. rm -rf /var/lib/postgresql/13/main/ && \ # - # Update memory configuration in case we decide to start PostgreSQL at + # Generate memory configuration in case we decide to start PostgreSQL at # build time - # Update memory configuration - /opt/postgresql-base/bin/update_memory_config.sh && \ + /opt/postgresql-base/bin/generate_runtime_config.sh && \ # # Run initdb mkdir -p /var/lib/postgresql/13/main/ && \ @@ -82,11 +113,13 @@ RUN \ --lc-collate='en_US.UTF-8' \ --lc-ctype='en_US.UTF-8' \ && \ + # true # VOLUME doesn't get set here as children of this image might amend the initial # data directory somehow (e.g. pre-initialize it with some schema). Once you do -# that in the sub-image, don't forget to define VOLUME afterwards! +# that in the sub-image, don't forget to define "VOLUME /var/lib/postgresql/" +# afterwards! # SIGTERM (Docker's default) will initiate PostgreSQL's "Smart Shutdown" mode # which will then wait for the current transactions to finish. If there are diff --git a/apps/postgresql-base/bin/generate_runtime_config.sh b/apps/postgresql-base/bin/generate_runtime_config.sh new file mode 100755 index 0000000000..3653f1fb3d --- /dev/null +++ b/apps/postgresql-base/bin/generate_runtime_config.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -u +set -e + + +# +# Update memory configuration based on the amount of RAM available to the +# container +# + +MC_POSTGRESQL_MEMORY_CONF_PATH="/var/run/postgresql/postgresql-memory.conf" +MC_RAM_SIZE=$(/container_memory_limit.sh) +MC_POSTGRESQL_CONF_SHARED_BUFFERS=$((MC_RAM_SIZE / 3)) +MC_POSTGRESQL_CONF_EFFECTIVE_CACHE_SIZE=$((MC_RAM_SIZE / 3)) + +cat > "${MC_POSTGRESQL_MEMORY_CONF_PATH}" << EOF +# +# Auto-generated, please don't edit! +# + +shared_buffers = ${MC_POSTGRESQL_CONF_SHARED_BUFFERS}MB +effective_cache_size = ${MC_POSTGRESQL_CONF_EFFECTIVE_CACHE_SIZE}MB +EOF + + +# +# Update PgBackRest configuration +# + +MC_POSTGRESQL_PGBACKREST_CONF_PATH="/var/run/postgresql/postgresql-pgbackrest.conf" +MC_BACKREST_CONF_D_S3_CONF_PATH="/etc/pgbackrest/conf.d/s3.conf" + + +if [ -z ${MC_PGBACKREST_ENABLE+x} ]; then + + echo "PgBackRest is disabled." + + cat > "${MC_POSTGRESQL_PGBACKREST_CONF_PATH}" << EOF +# +# Auto-generated, please don't edit! +# + +archive_mode = off +EOF + + cat > "${MC_BACKREST_CONF_D_S3_CONF_PATH}" << EOF +# +# Auto-generated, please don't edit! +# + +# S3 archiving disabled +EOF + +else + + echo "PgBackRest is enabled." + + cat > "${MC_POSTGRESQL_PGBACKREST_CONF_PATH}" << EOF +# +# Auto-generated, please don't edit! +# + +# Back up with PgBackRest +# (stanzas of all users of postgresql-base are called "main") +archive_mode = on +archive_command = 'pgbackrest --stanza=main archive-push %p' +EOF + + cat > "${MC_BACKREST_CONF_D_S3_CONF_PATH}" << EOF +# +# Auto-generated, please don't edit! +# + +# S3 credentials +[global] +repo1-retention-full=${MC_PGBACKREST_RETENTION_FULL} +repo1-s3-endpoint=${MC_PGBACKREST_S3_ENDPOINT} +repo1-s3-bucket=${MC_PGBACKREST_S3_BUCKET} +repo1-storage-verify-tls=${MC_PGBACKREST_S3_VERIFY_TLS} +repo1-s3-key=${MC_PGBACKREST_S3_KEY} +repo1-s3-key-secret=${MC_PGBACKREST_S3_KEY_SECRET} +repo1-s3-region=${MC_PGBACKREST_S3_REGION} +repo1-path=${MC_PGBACKREST_S3_PATH} +EOF + +fi diff --git a/apps/postgresql-base/bin/postgresql.sh b/apps/postgresql-base/bin/postgresql.sh index 032c9e0a38..fbdf5d886d 100755 --- a/apps/postgresql-base/bin/postgresql.sh +++ b/apps/postgresql-base/bin/postgresql.sh @@ -8,7 +8,7 @@ MC_POSTGRESQL_DATA_DIR="/var/lib/postgresql/13/main/" MC_POSTGRESQL_CONF_PATH="/etc/postgresql/13/main/postgresql.conf" # Update memory configuration -/opt/postgresql-base/bin/update_memory_config.sh +/opt/postgresql-base/bin/generate_runtime_config.sh # Start PostgreSQL exec "${MC_POSTGRESQL_BIN_DIR}/postgres" \ diff --git a/apps/postgresql-base/bin/update_memory_config.sh b/apps/postgresql-base/bin/update_memory_config.sh deleted file mode 100755 index a0456d51c0..0000000000 --- a/apps/postgresql-base/bin/update_memory_config.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -u -set -e - -MC_POSTGRESQL_MEMORY_CONF_PATH="/var/run/postgresql/postgresql-memory.conf" - -# Adjust configuration based on amount of RAM -MC_RAM_SIZE=$(/container_memory_limit.sh) -MC_POSTGRESQL_CONF_SHARED_BUFFERS=$((MC_RAM_SIZE / 3)) -MC_POSTGRESQL_CONF_EFFECTIVE_CACHE_SIZE=$((MC_RAM_SIZE / 3)) - -cat > "${MC_POSTGRESQL_MEMORY_CONF_PATH}" << EOF -# -# Auto-generated, please don't edit! -# - -shared_buffers = ${MC_POSTGRESQL_CONF_SHARED_BUFFERS}MB -effective_cache_size = ${MC_POSTGRESQL_CONF_EFFECTIVE_CACHE_SIZE}MB -EOF diff --git a/apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore b/apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore new file mode 100644 index 0000000000..5e7d2734cf --- /dev/null +++ b/apps/postgresql-base/etc/pgbackrest/conf.d/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/apps/postgresql-base/etc/pgbackrest/pgbackrest.conf b/apps/postgresql-base/etc/pgbackrest/pgbackrest.conf new file mode 100644 index 0000000000..40d4eb09bf --- /dev/null +++ b/apps/postgresql-base/etc/pgbackrest/pgbackrest.conf @@ -0,0 +1,30 @@ +[main] +pg1-path=/var/lib/postgresql/13/main + + +[global] +process-max=4 +start-fast=y +delta=y + +# Use fastest compression because the database is huge and network (plus the +# storage space on the other end) is plenty +compress-type=lz4 +compress-level=1 +buffer-size=16777216 + +# Docs say that async archiving is faster +archive-async=y +archive-push-queue-max=16GB +spool-path=/var/spool/pgbackrest + +# Log only to the console, i.e. Docker's log +log-level-console=detail +log-level-file=off + +# Write WALs and backups to S3 +repo1-type=s3 + +# S3 configuration read from environent variables will be in conf.d/s3.conf; +# FIXME WARN: configuration file contains command-line only option 'config-include-path' +config-include-path=/etc/pgbackrest/conf.d diff --git a/apps/postgresql-base/conf/environment b/apps/postgresql-base/etc/postgresql/13/main/environment similarity index 100% rename from apps/postgresql-base/conf/environment rename to apps/postgresql-base/etc/postgresql/13/main/environment diff --git a/apps/postgresql-base/conf/pg_ctl.conf b/apps/postgresql-base/etc/postgresql/13/main/pg_ctl.conf similarity index 100% rename from apps/postgresql-base/conf/pg_ctl.conf rename to apps/postgresql-base/etc/postgresql/13/main/pg_ctl.conf diff --git a/apps/postgresql-base/conf/pg_hba.conf b/apps/postgresql-base/etc/postgresql/13/main/pg_hba.conf similarity index 100% rename from apps/postgresql-base/conf/pg_hba.conf rename to apps/postgresql-base/etc/postgresql/13/main/pg_hba.conf diff --git a/apps/postgresql-base/conf/pg_ident.conf b/apps/postgresql-base/etc/postgresql/13/main/pg_ident.conf similarity index 100% rename from apps/postgresql-base/conf/pg_ident.conf rename to apps/postgresql-base/etc/postgresql/13/main/pg_ident.conf diff --git a/apps/postgresql-base/conf/postgresql.conf b/apps/postgresql-base/etc/postgresql/13/main/postgresql.conf similarity index 93% rename from apps/postgresql-base/conf/postgresql.conf rename to apps/postgresql-base/etc/postgresql/13/main/postgresql.conf index cbd4c22669..cc472e0a8e 100644 --- a/apps/postgresql-base/conf/postgresql.conf +++ b/apps/postgresql-base/etc/postgresql/13/main/postgresql.conf @@ -88,3 +88,6 @@ max_locks_per_transaction = 1024 # Include memory configuration (updated on every run in wrapper script) include '/var/run/postgresql/postgresql-memory.conf' + +# Include PgBackRest configuration (updated on every run in wrapper script) +include '/var/run/postgresql/postgresql-pgbackrest.conf' diff --git a/apps/postgresql-base/conf/start.conf b/apps/postgresql-base/etc/postgresql/13/main/start.conf similarity index 100% rename from apps/postgresql-base/conf/start.conf rename to apps/postgresql-base/etc/postgresql/13/main/start.conf diff --git a/apps/postgresql-server/bin/initialize_db.sh b/apps/postgresql-server/bin/initialize_db.sh index 921a2410c7..541208722a 100755 --- a/apps/postgresql-server/bin/initialize_db.sh +++ b/apps/postgresql-server/bin/initialize_db.sh @@ -8,7 +8,7 @@ MC_POSTGRESQL_DATA_DIR="/var/lib/postgresql/13/main/" MC_POSTGRESQL_CONF_PATH="/etc/postgresql/13/main/postgresql.conf" # Update memory configuration -/opt/postgresql-base/bin/update_memory_config.sh +/opt/postgresql-base/bin/generate_runtime_config.sh "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ -o "-c config_file=${MC_POSTGRESQL_CONF_PATH}" \ diff --git a/apps/postgresql-server/bin/postgresql.sh b/apps/postgresql-server/bin/postgresql.sh index cf7e7c5c57..31bf3f9d6c 100755 --- a/apps/postgresql-server/bin/postgresql.sh +++ b/apps/postgresql-server/bin/postgresql.sh @@ -4,7 +4,7 @@ set -u set -e # Update memory configuration -/opt/postgresql-base/bin/update_memory_config.sh +/opt/postgresql-base/bin/generate_runtime_config.sh # Run schema migrations if needed if [ -e /var/lib/postgresql/first_run ]; then diff --git a/apps/postgresql-server/docker-compose.tests.yml b/apps/postgresql-server/docker-compose.tests.yml new file mode 100644 index 0000000000..9c0b0f2189 --- /dev/null +++ b/apps/postgresql-server/docker-compose.tests.yml @@ -0,0 +1,39 @@ +# Used for manually testing PgBackRest + +version: "3.7" + +services: + + postgresql-server: + image: gcr.io/mcback/postgresql-server:latest + init: true + stop_signal: SIGKILL + expose: + - 5432 + environment: + MC_PGBACKREST_ENABLE: "${MC_PGBACKREST_ENABLE}" + MC_PGBACKREST_RETENTION_FULL: "${MC_PGBACKREST_RETENTION_FULL}" + MC_PGBACKREST_S3_ENDPOINT: "${MC_PGBACKREST_S3_ENDPOINT}" + MC_PGBACKREST_S3_BUCKET: "${MC_PGBACKREST_S3_BUCKET}" + MC_PGBACKREST_S3_VERIFY_TLS: "${MC_PGBACKREST_S3_VERIFY_TLS}" + MC_PGBACKREST_S3_KEY: "${MC_PGBACKREST_S3_KEY}" + MC_PGBACKREST_S3_KEY_SECRET: "${MC_PGBACKREST_S3_KEY_SECRET}" + MC_PGBACKREST_S3_REGION: "${MC_PGBACKREST_S3_REGION}" + MC_PGBACKREST_S3_PATH: "${MC_PGBACKREST_S3_PATH}" + volumes: + - type: bind + source: ./bin/ + target: /opt/postgresql-server/bin/ + - type: bind + source: ./pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ + - type: bind + source: ./../postgresql-base/bin/ + target: /opt/postgresql-base/bin/ + - type: bind + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ + # Mounting PgBackRest's configuration directory too + - type: bind + source: ./../postgresql-base/etc/pgbackrest/pgbackrest.conf + target: /etc/pgbackrest/pgbackrest.conf diff --git a/apps/postgresql-upgrade/bin/postgresql_upgrade.py b/apps/postgresql-upgrade/bin/postgresql_upgrade.py index 48c1b928a2..25796bba40 100755 --- a/apps/postgresql-upgrade/bin/postgresql_upgrade.py +++ b/apps/postgresql-upgrade/bin/postgresql_upgrade.py @@ -295,7 +295,7 @@ def postgres_upgrade(source_version: int, target_version: int) -> None: ) logging.info("Updating memory configuration...") - subprocess.check_call(['/opt/mediacloud/bin/update_memory_config.sh']) + subprocess.check_call(['/opt/mediacloud/bin/generate_runtime_config.sh']) # Remove cruft that might have been left over from last attempt to do the upgrade patterns = [ diff --git a/apps/purge-object-caches/docker-compose.tests.yml b/apps/purge-object-caches/docker-compose.tests.yml index cdc6037a60..3c5c7f0b89 100644 --- a/apps/purge-object-caches/docker-compose.tests.yml +++ b/apps/purge-object-caches/docker-compose.tests.yml @@ -43,5 +43,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/rescrape-media/docker-compose.tests.yml b/apps/rescrape-media/docker-compose.tests.yml index 5a7ce7926e..4c40f69c0a 100644 --- a/apps/rescrape-media/docker-compose.tests.yml +++ b/apps/rescrape-media/docker-compose.tests.yml @@ -50,8 +50,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/temporal-postgresql/bin/initialize_schema.sh b/apps/temporal-postgresql/bin/initialize_schema.sh index df22aba2b3..154674c144 100755 --- a/apps/temporal-postgresql/bin/initialize_schema.sh +++ b/apps/temporal-postgresql/bin/initialize_schema.sh @@ -11,7 +11,7 @@ MC_POSTGRESQL_DATA_DIR="/var/lib/postgresql/13/main/" MC_POSTGRESQL_CONF_PATH="/etc/postgresql/13/main/postgresql.conf" # Update memory configuration -/opt/postgresql-base/bin/update_memory_config.sh +/opt/postgresql-base/bin/generate_runtime_config.sh "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ -o "-c config_file=${MC_POSTGRESQL_CONF_PATH}" \ diff --git a/apps/temporal-postgresql/bin/postgresql.sh b/apps/temporal-postgresql/bin/postgresql.sh index 4b7af3a946..005f5fc5fb 100755 --- a/apps/temporal-postgresql/bin/postgresql.sh +++ b/apps/temporal-postgresql/bin/postgresql.sh @@ -4,7 +4,7 @@ set -u set -e # Update memory configuration -/opt/postgresql-base/bin/update_memory_config.sh +/opt/postgresql-base/bin/generate_runtime_config.sh # Run schema migrations if needed if [ -e /var/lib/postgresql/first_run ]; then diff --git a/apps/temporal-server/docker-compose.tests.yml b/apps/temporal-server/docker-compose.tests.yml index 89ab2f7968..0607ea223e 100644 --- a/apps/temporal-server/docker-compose.tests.yml +++ b/apps/temporal-server/docker-compose.tests.yml @@ -86,8 +86,8 @@ services: source: ./../temporal-postgresql/bin/ target: /opt/temporal-postgresql/bin/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ temporal-elasticsearch: image: gcr.io/mcback/temporal-elasticsearch:latest diff --git a/apps/tools/docker-compose.tests.yml b/apps/tools/docker-compose.tests.yml index 665a29f6e3..5ee1e180b5 100644 --- a/apps/tools/docker-compose.tests.yml +++ b/apps/tools/docker-compose.tests.yml @@ -45,8 +45,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/topics-base/docker-compose.tests.yml b/apps/topics-base/docker-compose.tests.yml index 6dc66c83e9..4ae712a1b3 100644 --- a/apps/topics-base/docker-compose.tests.yml +++ b/apps/topics-base/docker-compose.tests.yml @@ -93,8 +93,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/topics-extract-story-links/docker-compose.tests.yml b/apps/topics-extract-story-links/docker-compose.tests.yml index 94aea4a828..2e45fa3a9b 100644 --- a/apps/topics-extract-story-links/docker-compose.tests.yml +++ b/apps/topics-extract-story-links/docker-compose.tests.yml @@ -75,5 +75,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/topics-fetch-link/docker-compose.tests.yml b/apps/topics-fetch-link/docker-compose.tests.yml index c9ec6f61ad..d46a331dd9 100644 --- a/apps/topics-fetch-link/docker-compose.tests.yml +++ b/apps/topics-fetch-link/docker-compose.tests.yml @@ -93,8 +93,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/topics-fetch-twitter-urls/docker-compose.tests.yml b/apps/topics-fetch-twitter-urls/docker-compose.tests.yml index 49726d6525..6ba9b0e2b3 100644 --- a/apps/topics-fetch-twitter-urls/docker-compose.tests.yml +++ b/apps/topics-fetch-twitter-urls/docker-compose.tests.yml @@ -97,8 +97,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/topics-map/docker-compose.tests.yml b/apps/topics-map/docker-compose.tests.yml index 607c9ca4a1..58eca72c55 100644 --- a/apps/topics-map/docker-compose.tests.yml +++ b/apps/topics-map/docker-compose.tests.yml @@ -55,5 +55,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/apps/topics-mine/docker-compose.tests.yml b/apps/topics-mine/docker-compose.tests.yml index 011ff01846..7f1815b8cc 100644 --- a/apps/topics-mine/docker-compose.tests.yml +++ b/apps/topics-mine/docker-compose.tests.yml @@ -111,8 +111,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ rabbitmq-server: image: gcr.io/mcback/rabbitmq-server:latest diff --git a/apps/topics-snapshot/docker-compose.tests.yml b/apps/topics-snapshot/docker-compose.tests.yml index 22f1e877d0..6b5013b67e 100644 --- a/apps/topics-snapshot/docker-compose.tests.yml +++ b/apps/topics-snapshot/docker-compose.tests.yml @@ -100,8 +100,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/webapp-api/docker-compose.tests.yml b/apps/webapp-api/docker-compose.tests.yml index 5e2affe8ec..cfec04b720 100644 --- a/apps/webapp-api/docker-compose.tests.yml +++ b/apps/webapp-api/docker-compose.tests.yml @@ -69,8 +69,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/webapp-httpd/docker-compose.tests.yml b/apps/webapp-httpd/docker-compose.tests.yml index 0689c6747f..3a27891aad 100644 --- a/apps/webapp-httpd/docker-compose.tests.yml +++ b/apps/webapp-httpd/docker-compose.tests.yml @@ -98,8 +98,8 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ solr-shard-01: image: gcr.io/mcback/solr-shard:latest diff --git a/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml b/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml index 2b3396af8e..e60db066a5 100644 --- a/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml +++ b/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml @@ -49,5 +49,5 @@ services: source: ./../postgresql-server/pgmigrate/ target: /opt/postgresql-server/pgmigrate/ - type: bind - source: ./../postgresql-base/conf/ - target: /etc/postgresql/13/main/ + source: ./../postgresql-base/etc/postgresql/ + target: /etc/postgresql/ diff --git a/doc/postgresql_pgbackrest.markdown b/doc/postgresql_pgbackrest.markdown new file mode 100644 index 0000000000..b3fb0b339c --- /dev/null +++ b/doc/postgresql_pgbackrest.markdown @@ -0,0 +1,88 @@ +# PgBackRest + + +## Usage + +Create stanza: + +```bash +pgbackrest --stanza=main stanza-create +``` + +Check stanza: + +```bash +pgbackrest --stanza=main check +``` + +Do a full backup: + +```bash +pgbackrest --stanza=main --type=full backup +``` + +Do an incremental backup: + +```bash +pgbackrest --stanza=main --type=incr backup +``` + + +## Creating S3 bucket + +1. Create a bucket: + 1. Name the bucket appropriately, e.g. `mediacloud-pgbackrest` or `mediacloud-pgbackrest-test` + 2. Make sure public access to the bucket is off + 3. Add a tag with name `project` and value `mediacloud-backrest` + +2. Create (update) IAM policy: + 1. Name the policy appropriately, e.g. `mediacloud-pgbackrest` or `mediacloud-pgbackrest-test`; + 2. Allow access to the previously created bucket, e.g.: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "1", + "Effect": "Allow", + "Action": [ + "s3:ListAllMyBuckets" + ], + "Resource": [ + "arn:aws:s3:::*" + ] + }, + { + "Sid": "2", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::mediacloud-pgbackrest-test" + ] + }, + { + "Sid": "3", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::mediacloud-pgbackrest-test/*" + ] + } + ] + } + ``` + + 3. Add a tag with name `project` and value `mediacloud-backrest` + +3. Create (update) IAM user: + 1. Name the user appropriately, e.g. `mediacloud-pgbackrest` or `mediacloud-pgbackrest-test` + 2. Enable only the programmatic access + 3. Attach the newly created / updated policy to the user + 4. Add a tag with name `project` and value `mediacloud-backrest` From 57c8633dbb555f3481e0907836883a01a473bb35 Mon Sep 17 00:00:00 2001 From: Linas Valiukas Date: Fri, 8 Oct 2021 22:30:05 +0100 Subject: [PATCH 3/3] Specify what a stanza is [ci skip] --- doc/postgresql_pgbackrest.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/postgresql_pgbackrest.markdown b/doc/postgresql_pgbackrest.markdown index b3fb0b339c..fe3966b75b 100644 --- a/doc/postgresql_pgbackrest.markdown +++ b/doc/postgresql_pgbackrest.markdown @@ -3,6 +3,8 @@ ## Usage +A stanza is the [configuration for a PostgreSQL database cluster that defines where it is located, how it will be backed up, archiving options, etc.](https://pgbackrest.org/user-guide.html) + Create stanza: ```bash