From b722248ed7c4a7aba2b047855b983d72a9725209 Mon Sep 17 00:00:00 2001 From: Paul B Date: Thu, 28 May 2020 18:41:49 +0200 Subject: standby-clone: Allow cloning of standby server with pg_basebackup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right now the role assumes you have a base backup available with Barman. However if you don't have an initial barman backup you might want to clone the primary server directly to setup your standby server. This PR adds a new `primary.pg_basebackup` option to the cluster configuration which if enabled (set to `true`) will create a `/root/standby-clone-{{ postgres_version }}-{{ postgres_cluster_name }}.sh` script on the standby server which helps to initialise a standby server. ⚠️ Breaking change: the current role behavior which creates a cloning script fetching the initial backup from barman will not be enabled by default anymore. You will need to add the new `primary.restore_barman_directory` option in your role configuration to do so. ⚠️ --- tasks/postgres-cluster.yml | 6 +++- tasks/postgres-standby-barman.yml | 4 +-- tasks/postgres-standby-basebackup.yml | 4 +++ templates/standby-barman-clone.sh.j2 | 45 ++++++++++++++++++++++++++ templates/standby-clone.sh.j2 | 45 -------------------------- templates/standby-pg_basebackup-clone.sh.j2 | 49 +++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 48 deletions(-) create mode 100644 tasks/postgres-standby-basebackup.yml create mode 100755 templates/standby-barman-clone.sh.j2 delete mode 100755 templates/standby-clone.sh.j2 create mode 100755 templates/standby-pg_basebackup-clone.sh.j2 diff --git a/tasks/postgres-cluster.yml b/tasks/postgres-cluster.yml index a8e96c1..4f62d31 100644 --- a/tasks/postgres-cluster.yml +++ b/tasks/postgres-cluster.yml @@ -60,7 +60,11 @@ - name: Add standby clone from barman script include: postgres-standby-barman.yml - when: postgres_barman_server is defined + when: postgres_barman_server is defined and postgres_primary.restore_barman_directory is defined + +- name: Add standby clone from pg_basebackup + include: postgres-standby-basebackup.yml + when: postgres_primary and postgres_primary.pg_basebackup is defined - name: Determine SSD or rotational disks raw: 'lsblk --noheadings --nodeps --raw --output=rota | grep -q 1' diff --git a/tasks/postgres-standby-barman.yml b/tasks/postgres-standby-barman.yml index 6a94b75..fb71d32 100644 --- a/tasks/postgres-standby-barman.yml +++ b/tasks/postgres-standby-barman.yml @@ -11,8 +11,8 @@ path: "{{ postgres_barman_path_prefix | default('~') }}" rsync_password_file: "{{ postgres_barman_rsync_enabled | ternary(' --password-file=/var/lib/postgresql/.rsync_pass ', '') }}" -- name: Copy secondary script - template: src=standby-clone.sh.j2 dest=/root/standby-clone-{{ postgres_version }}-{{ postgres_cluster_name }}.sh mode=0755 +- name: Copy secondary barman clone script + template: src=standby-barman-clone.sh.j2 dest=/root/standby-clone-{{ postgres_version }}-{{ postgres_cluster_name }}.sh mode=0755 - name: Copy rsync password file copy: diff --git a/tasks/postgres-standby-basebackup.yml b/tasks/postgres-standby-basebackup.yml new file mode 100644 index 0000000..db52d55 --- /dev/null +++ b/tasks/postgres-standby-basebackup.yml @@ -0,0 +1,4 @@ +--- +- name: Copy secondary pg_basebackup clone script + template: src=standby-pg_basebackup-clone.sh.j2 dest=/root/standby-clone-{{ postgres_version }}-{{ postgres_cluster_name }}.sh mode=0755 + no_log: true diff --git a/templates/standby-barman-clone.sh.j2 b/templates/standby-barman-clone.sh.j2 new file mode 100755 index 0000000..2db44b6 --- /dev/null +++ b/templates/standby-barman-clone.sh.j2 @@ -0,0 +1,45 @@ +#!/bin/bash +# {{ ansible_managed }} + +set -eo pipefail + +BARMAN_DATABASE=$1 +BARMAN_BACKUP_VERSION=$2 + +if [[ -z $BARMAN_DATABASE || -z $BARMAN_BACKUP_VERSION ]] +then + echo "Usage : $0 BARMAN_DATABASE BARMAN_BACKUP_VERSION" >&2 + echo " Example: $0 25kv 20161118T002405" >&2 + exit 1 +fi + +BACKUP_DATE=`date +%s` + +echo "accept key if necessary" +sudo -u postgres ssh barman@{{ postgres_barman_server }} echo "" + +echo Stopping PostgreSQL +pg_ctlcluster {{ postgres_version }} {{ postgres_cluster_name }} stop || true + +echo Cleaning up old cluster directory +sudo -u postgres mv /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}{,_$BACKUP_DATE} + +echo Creating new directory +sudo -u postgres mkdir -p /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }} + +echo Get previous backup from backups server +sudo -u postgres \ + time rsync --progress -pvia --exclude='*.conf' --exclude='server.crt' --exclude='server.key' --delete {{ rsync_options }} \ + {{ barman_remote_url }}/$BARMAN_DATABASE/base/$BARMAN_BACKUP_VERSION/data/ \ + /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}/ + +echo Restoring .conf and server certificate +sudo -u postgres cp -a /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}_$BACKUP_DATE/{*.conf,server.crt,server.key} /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}/ + +echo Ensure rights are correcly set +chown -R postgres:postgres /var/lib/postgresql/{{ postgres_version }}/ +chmod 0700 /var/lib/postgresql/{{ postgres_version }}/ +chmod -R o-rwx /var/lib/postgresql/{{ postgres_version }}/ + +echo Starting PostgreSQL +sudo pg_ctlcluster {{ postgres_version }} {{ postgres_cluster_name }} start diff --git a/templates/standby-clone.sh.j2 b/templates/standby-clone.sh.j2 deleted file mode 100755 index 2db44b6..0000000 --- a/templates/standby-clone.sh.j2 +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# {{ ansible_managed }} - -set -eo pipefail - -BARMAN_DATABASE=$1 -BARMAN_BACKUP_VERSION=$2 - -if [[ -z $BARMAN_DATABASE || -z $BARMAN_BACKUP_VERSION ]] -then - echo "Usage : $0 BARMAN_DATABASE BARMAN_BACKUP_VERSION" >&2 - echo " Example: $0 25kv 20161118T002405" >&2 - exit 1 -fi - -BACKUP_DATE=`date +%s` - -echo "accept key if necessary" -sudo -u postgres ssh barman@{{ postgres_barman_server }} echo "" - -echo Stopping PostgreSQL -pg_ctlcluster {{ postgres_version }} {{ postgres_cluster_name }} stop || true - -echo Cleaning up old cluster directory -sudo -u postgres mv /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}{,_$BACKUP_DATE} - -echo Creating new directory -sudo -u postgres mkdir -p /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }} - -echo Get previous backup from backups server -sudo -u postgres \ - time rsync --progress -pvia --exclude='*.conf' --exclude='server.crt' --exclude='server.key' --delete {{ rsync_options }} \ - {{ barman_remote_url }}/$BARMAN_DATABASE/base/$BARMAN_BACKUP_VERSION/data/ \ - /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}/ - -echo Restoring .conf and server certificate -sudo -u postgres cp -a /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}_$BACKUP_DATE/{*.conf,server.crt,server.key} /var/lib/postgresql/{{ postgres_version }}/{{ postgres_cluster_name }}/ - -echo Ensure rights are correcly set -chown -R postgres:postgres /var/lib/postgresql/{{ postgres_version }}/ -chmod 0700 /var/lib/postgresql/{{ postgres_version }}/ -chmod -R o-rwx /var/lib/postgresql/{{ postgres_version }}/ - -echo Starting PostgreSQL -sudo pg_ctlcluster {{ postgres_version }} {{ postgres_cluster_name }} start diff --git a/templates/standby-pg_basebackup-clone.sh.j2 b/templates/standby-pg_basebackup-clone.sh.j2 new file mode 100755 index 0000000..ca34a05 --- /dev/null +++ b/templates/standby-pg_basebackup-clone.sh.j2 @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# {{ ansible_managed }} + +set -eo pipefail + +CLUSTER_VERSION="{{ postgres_version }}" +CLUSTER_NAME="{{ postgres_cluster_name }}" +PRIMARY_CLUSTER_HOST="{{ postgres_primary.host }}" +PRIMARY_CLUSTER_PORT="{{ postgres_primary.port }}" +PRIMARY_CLUSTER_USER="{{ postgres_primary.replication_user }}" +PRIMARY_CLUSTER_PASSWORD="{{ postgres_primary.replication_password }}" +{% if postgres_primary.replication_dbname is defined %} +PRIMARY_CLUSTER_DBNAME=" dbname='{{ postgres_primary.replication_dbname }}'" +{% endif %} + +read -p "You are about to clone the primary cluster (${PRIMARY_CLUSTER_HOST}:${PRIMARY_CLUSTER_PORT}) with pg_basebackup. This will replace the local ${CLUSTER_VERSION}/${CLUSTER_NAME} cluster. Are you sure? [y/n] " -r go + +if [ "$go" != "y" ]; then + echo "Aborted." + exit 1 +fi + +BACKUP_DATE=$(date +%s) + +echo "Stopping PostgreSQL" +pg_ctlcluster "${CLUSTER_VERSION}" "${CLUSTER_NAME}" stop || true + +echo "Cleaning up old cluster directory" +sudo -u postgres mv "/var/lib/postgresql/${CLUSTER_VERSION}/${CLUSTER_NAME}"{,"_${BACKUP_DATE}"} + +echo "Creating new directory" +sudo -u postgres mkdir -p "/var/lib/postgresql/${CLUSTER_VERSION}/${CLUSTER_NAME}" + +echo "Get backup from primary server" +sudo -u postgres \ + time "/usr/lib/postgresql/${CLUSTER_VERSION}/bin/pg_basebackup" --format=p -R \ + -d "host='${PRIMARY_CLUSTER_HOST}' port='${PRIMARY_CLUSTER_PORT}' user='${PRIMARY_CLUSTER_USER}' password='${PRIMARY_CLUSTER_PASSWORD}' ${PRIMARY_CLUSTER_DBNAME} sslmode=require" \ + -D "/var/lib/postgresql/${CLUSTER_VERSION}/${CLUSTER_NAME}/" + +echo "Restoring .conf and server certificate" +sudo -u postgres cp -a "/var/lib/postgresql/${CLUSTER_VERSION}/${CLUSTER_NAME}_${BACKUP_DATE}/"{*.conf,server.crt,server.key} "/var/lib/postgresql/${CLUSTER_VERSION}/${CLUSTER_NAME}/" + +echo "Ensure rights are correcly set" +chown -R postgres:postgres "/var/lib/postgresql/${CLUSTER_VERSION}/" +chmod 0700 "/var/lib/postgresql/${CLUSTER_VERSION}/" +chmod -R o-rwx "/var/lib/postgresql/${CLUSTER_VERSION}/" + +echo "Starting PostgreSQL" +sudo pg_ctlcluster "${CLUSTER_VERSION}" "${CLUSTER_NAME}" start -- cgit v1.2.3