Merge branch 'refactoring' into dev

9 files changed, 589 insertions, 0 deletions

diff --git a/modules/profile/manifests/postgresql/backup_dump.pp b/modules/profile/manifests/postgresql/backup_dump.pp
new file mode 100644
index 0000000..10e349a
--- /dev/null
+++ b/modules/profile/manifests/postgresql/backup_dump.pp
@@ -0,0 +1,53 @@
+define profile::postgresql::backup_dump (
+  String $pg_user  = "postgres",
+  String $pg_group = "postgres",
+) {
+  $base_path        = $title
+  $pg_path          = "$base_path/postgresql"
+  $pg_backup_path   = "$base_path/postgresql_backup"
+  $pg_host          = split($base_path, "/")[-1]
+
+  ensure_packages(["python", "python-pip"])
+  ensure_resource("package", "pylog2rotate", {
+    source   => "git+https://github.com/avian2/pylog2rotate",
+    ensure   => present,
+    provider => "pip3",
+    require  => Package["python-pip"],
+  })
+
+  file { $pg_backup_path:
+    ensure  => directory,
+    owner   => $pg_user,
+    group   => $pg_group,
+    mode    => "0700",
+    require => File[$base_path],
+  }
+
+  cron::job::multiple { "backup_psql_$pg_host":
+    ensure  => "present",
+    require => [File[$pg_backup_path], File[$pg_path]],
+    jobs    => [
+      {
+        command     => "/usr/bin/pg_dumpall -h $pg_path -f $pg_backup_path/\$(date -Iseconds).sql",
+        user        => $pg_user,
+        hour        => "22,4,10,16",
+        minute      => 0,
+        description => "Backup the database",
+      },
+      {
+        command     => "/usr/bin/rm -f $(ls -1 $pg_backup_path/*.sql | grep -v 'T22:' | sort -r | sed -e '1,12d')",
+        user        => $pg_user,
+        hour        => 3,
+        minute      => 0,
+        description => "Cleanup the database backups",
+      },
+      {
+        command     => "cd $pg_backup_path ; /usr/bin/rm -f $(ls -1 *T22*.sql | log2rotate --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M:%S+02:00.sql')",
+        user        => $pg_user,
+        hour        => 3,
+        minute      => 1,
+        description => "Cleanup the database backups exponentially",
+      },
+    ]
+  }
+}
diff --git a/modules/profile/manifests/postgresql/backup_pgbouncer.pp b/modules/profile/manifests/postgresql/backup_pgbouncer.pp
new file mode 100644
index 0000000..45b8ed5
--- /dev/null
+++ b/modules/profile/manifests/postgresql/backup_pgbouncer.pp
@@ -0,0 +1,92 @@
+define profile::postgresql::backup_pgbouncer (
+  String $base_path,
+  Hash   $pg_infos,
+  String $pg_user  = "postgres",
+  String $pg_group = "postgres",
+) {
+  include "profile::postgresql::pam_ldap_pgbouncer"
+  ensure_packages(["pgbouncer"])
+
+  $host_cn = $title
+
+  $host = find_host($facts["ldapvar"]["other"], $host_cn)
+  if empty($host) {
+    fail("No host found for pgbouncer")
+  } elsif has_key($host["vars"], "host") {
+    $pg_backup_host = $host["vars"]["host"][0]
+  } else {
+    $pg_backup_host = $host["vars"]["real_hostname"][0]
+  }
+
+  $pg_path = "$base_path/$pg_backup_host/postgresql"
+
+  if has_key($host["vars"], "postgresql_backup_port") {
+    $pg_port = " port=${host[vars][postgresql_backup_port][0]}"
+  } else {
+    $pg_port = ""
+  }
+
+  # Config
+  ensure_resource("concat", "/etc/pgbouncer/pgbouncer.ini", {
+    mode           => "0644",
+    owner          => "root",
+    group          => "root",
+    ensure_newline => true,
+    notify         => Service["pgbouncer"],
+    before         => Service["pgbouncer"],
+  })
+
+  ensure_resource("concat::fragment", "pgbouncer_head", {
+    target => "/etc/pgbouncer/pgbouncer.ini",
+    order  => 01,
+    source => "puppet:///modules/profile/postgresql/pgbouncer_head.ini",
+  })
+
+  concat::fragment { "pgbouncer_$pg_backup_host":
+    target  => "/etc/pgbouncer/pgbouncer.ini",
+    order   => 02,
+    content => "${pg_infos[pgbouncer_dbname]} = host=$pg_path$pg_port user=${pg_infos[dbuser]} dbname=${pg_infos[dbname]}",
+  }
+
+  # pg_hba for accessed cluster
+  postgresql::server::pg_hba_rule { "$pg_backup_host - local access as ${pg_infos[dbuser]} user":
+    description => "Allow local access to ${pg_infos[dbuser]} user",
+    type        => 'local',
+    database    => $pg_infos["dbname"],
+    user        => $pg_infos["dbuser"],
+    auth_method => 'trust',
+    order       => "01-00",
+    target      => "$pg_path/pg_hba.conf",
+    postgresql_version => "10",
+  }
+
+  # service
+  ensure_resource("file", "/etc/systemd/system/pgbouncer.service.d", {
+    ensure => "directory",
+    mode   => "0644",
+    owner  => "root",
+    group  => "root",
+  })
+
+  ensure_resource("file", "/etc/systemd/system/pgbouncer.service.d/override.conf", {
+    ensure  => "present",
+    mode    => "0644",
+    owner   => "root",
+    group   => "root",
+    content => "[Service]\nUser=\nUser=$pg_user\n",
+    notify  => Service["pgbouncer"],
+    before  => Service["pgbouncer"],
+  })
+
+  ensure_resource("service", "pgbouncer", {
+    ensure  => "running",
+    enable  => true,
+    require => [
+      Package["pgbouncer"],
+      File["/etc/systemd/system/pgbouncer.service.d/override.conf"],
+      Concat["/etc/pgbouncer/pgbouncer.ini"]
+    ],
+  })
+
+
+}
diff --git a/modules/profile/manifests/postgresql/backup_replication.pp b/modules/profile/manifests/postgresql/backup_replication.pp
new file mode 100644
index 0000000..a4edb8f
--- /dev/null
+++ b/modules/profile/manifests/postgresql/backup_replication.pp
@@ -0,0 +1,135 @@
+define profile::postgresql::backup_replication (
+  String $base_path,
+  Hash   $pg_infos,
+  String $pg_user  = "postgres",
+  String $pg_group = "postgres",
+) {
+  $host_cn = $title
+
+  $host = find_host($facts["ldapvar"]["other"], $host_cn)
+  if empty($host) {
+    $pg_backup_host = $host_cn
+  } elsif has_key($host["vars"], "host") {
+    $pg_backup_host = $host["vars"]["host"][0]
+  } else {
+    $pg_backup_host = $host["vars"]["real_hostname"][0]
+  }
+
+  $pg_path = "$base_path/$pg_backup_host/postgresql"
+
+  # Replication folder
+  ensure_resource("file", "$base_path/$pg_backup_host", {
+    ensure => directory,
+  })
+
+  file { $pg_path:
+    ensure  => directory,
+    owner   => $pg_user,
+    group   => $pg_group,
+    mode    => "0700",
+    require => File["$base_path/$pg_backup_host"],
+  }
+
+  # pg_hba.conf
+  profile::postgresql::base_pg_hba_rules { $pg_backup_host:
+    pg_path => $pg_path
+  }
+
+  # postgresql.conf file and ssl
+  concat { "$pg_path/postgresql.conf":
+    owner => $pg_user,
+    group => $pg_group,
+    mode  => '0640',
+    warn  => true,
+  }
+
+  if !empty($host) and has_key($host["vars"], "postgresql_backup_port") {
+    $pg_listen_port = $host["vars"]["postgresql_backup_port"][0]
+
+    profile::postgresql::ssl { $pg_path:
+      certname             => $host_cn,
+      handle_concat_config => true,
+      before               => Service["postgresql_backup@$pg_backup_host"]
+    }
+
+    concat::fragment { "$pg_path/postgresql.conf listen":
+      target  => "$pg_path/postgresql.conf",
+      content => "listen_addresses = '*'\nport = $pg_listen_port\n",
+    }
+
+    profile::postgresql::replication { $host_cn:
+      target => "$pg_path/pg_hba.conf",
+    }
+  } else {
+    concat::fragment { "$pg_path/postgresql.conf listen":
+      target  => "$pg_path/postgresql.conf",
+      content => "listen_addresses = ''\n",
+    }
+  }
+
+  concat::fragment { "$pg_path/postgresql.conf paths":
+    target  => "$pg_path/postgresql.conf",
+    content => "unix_socket_directories = '$pg_path'\ndata_directory = '$pg_path'\nwal_level = logical\n",
+  }
+
+  $password_seed = lookup("base_installation::puppet_pass_seed")
+  $pg_host = $pg_backup_host
+  $pg_port = $pg_infos["dbport"]
+  $ldap_cn = lookup("base_installation::ldap_cn")
+  $ldap_password = generate_password(24, $password_seed, "ldap")
+  $pg_slot = regsubst($ldap_cn, '-', "_", "G")
+
+  # recovery.conf file
+  $primary_conninfo  = "host=$pg_host port=$pg_port user=$ldap_cn password=$ldap_password sslmode=require"
+  $primary_slot_name = $pg_slot
+  $standby_mode      = "on"
+
+  file { "$pg_path/recovery.conf":
+    owner   => $pg_user,
+    group   => $pg_group,
+    mode    => '0640',
+    content => template('postgresql/recovery.conf.erb'),
+  }
+
+  # Initial replication
+  exec { "pg_basebackup $pg_path":
+    cwd         => $pg_path,
+    user        => $pg_user,
+    creates     => "$pg_path/PG_VERSION",
+    environment => ["PGPASSWORD=$ldap_password"],
+    command     => "/usr/bin/pg_basebackup -w -h $pg_host -p $pg_port -U $ldap_cn -D $pg_path -S $pg_slot",
+    before      => [
+      Concat["$pg_path/pg_hba.conf"],
+      File["$pg_path/recovery.conf"],
+      Concat["$pg_path/postgresql.conf"],
+    ]
+  }
+
+  # Service
+  ensure_resource("file", "/etc/systemd/system/postgresql_backup@.service", {
+    mode    => "0644",
+    owner   => "root",
+    group   => "root",
+    content => template("profile/postgresql/postgresql_backup@.service.erb"),
+  })
+
+  service { "postgresql_backup@$pg_backup_host":
+    enable  => true,
+    ensure  => "running",
+    require => [
+      File["/etc/systemd/system/postgresql_backup@.service"],
+      Concat["$pg_path/pg_hba.conf"],
+      File["$pg_path/recovery.conf"],
+      Concat["$pg_path/postgresql.conf"],
+    ],
+    subscribe => [
+      Concat["$pg_path/pg_hba.conf"],
+      File["$pg_path/recovery.conf"],
+      Concat["$pg_path/postgresql.conf"],
+    ]
+  }
+
+  # Dumps
+  profile::postgresql::backup_dump { "$base_path/$pg_backup_host": }
+
+}
diff --git a/modules/profile/manifests/postgresql/base_pg_hba_rules.pp b/modules/profile/manifests/postgresql/base_pg_hba_rules.pp
new file mode 100644
index 0000000..07c4bb6
--- /dev/null
+++ b/modules/profile/manifests/postgresql/base_pg_hba_rules.pp
@@ -0,0 +1,74 @@
+define profile::postgresql::base_pg_hba_rules (
+  Optional[String] $pg_path  = undef,
+  String           $pg_user  = "postgres",
+  String           $pg_group = "postgres",
+) {
+  unless empty($pg_path) {
+    concat { "$pg_path/pg_hba.conf":
+      owner   => $pg_user,
+      group   => $pg_group,
+      mode    => '0640',
+      warn    => true,
+      require => File[$pg_path],
+    }
+
+    Postgresql::Server::Pg_hba_rule {
+      target             => "$pg_path/pg_hba.conf",
+      postgresql_version => "10",
+    }
+  }
+
+  postgresql::server::pg_hba_rule { "$title - local access as postgres user":
+    description => 'Allow local access to postgres user',
+    type        => 'local',
+    database    => 'all',
+    user        => $pg_user,
+    auth_method => 'ident',
+    order       => "00-01",
+  }
+  postgresql::server::pg_hba_rule { "$title - localhost access as postgres user":
+    description => 'Allow localhost access to postgres user',
+    type        => 'host',
+    database    => 'all',
+    user        => $pg_user,
+    address     => "127.0.0.1/32",
+    auth_method => 'md5',
+    order       => "00-02",
+  }
+  postgresql::server::pg_hba_rule { "$title - localhost ip6 access as postgres user":
+    description => 'Allow localhost access to postgres user',
+    type        => 'host',
+    database    => 'all',
+    user        => $pg_user,
+    address     => "::1/128",
+    auth_method => 'md5',
+    order       => "00-03",
+  }
+  postgresql::server::pg_hba_rule { "$title - deny access to postgresql user":
+    description => 'Deny remote access to postgres user',
+    type        => 'host',
+    database    => 'all',
+    user        => $pg_user,
+    address     => "0.0.0.0/0",
+    auth_method => 'reject',
+    order       => "00-04",
+  }
+  postgresql::server::pg_hba_rule { "$title - local access":
+    description => 'Allow local access with password',
+    type        => 'local',
+    database    => 'all',
+    user        => 'all',
+    auth_method => 'md5',
+    order       => "10-01",
+  }
+
+  postgresql::server::pg_hba_rule { "$title - local access with same name":
+    description => 'Allow local access with same name',
+    type        => 'local',
+    database    => 'all',
+    user        => 'all',
+    auth_method => 'ident',
+    order       => "10-02",
+  }
+
+}
diff --git a/modules/profile/manifests/postgresql/master.pp b/modules/profile/manifests/postgresql/master.pp
new file mode 100644
index 0000000..969905f
--- /dev/null
+++ b/modules/profile/manifests/postgresql/master.pp
@@ -0,0 +1,20 @@
+define profile::postgresql::master (
+  $letsencrypt_host = undef,
+  $backup_hosts     = [],
+) {
+  profile::postgresql::ssl { "/var/lib/postgres/data":
+    cert                => "/etc/letsencrypt/live/$letsencrypt_host/cert.pem",
+    key                 => "/etc/letsencrypt/live/$letsencrypt_host/privkey.pem",
+    require             => Letsencrypt::Certonly[$letsencrypt_host],
+    handle_config_entry => true,
+  }
+
+  $backup_hosts.each |$backup_host| {
+    profile::postgresql::replication { $backup_host:
+      handle_config => true,
+      handle_role   => true,
+      handle_slot   => true,
+      add_self_role => true,
+    }
+  }
+}
diff --git a/modules/profile/manifests/postgresql/pam_ldap.pp b/modules/profile/manifests/postgresql/pam_ldap.pp
new file mode 100644
index 0000000..f068245
--- /dev/null
+++ b/modules/profile/manifests/postgresql/pam_ldap.pp
@@ -0,0 +1,28 @@
+class profile::postgresql::pam_ldap (
+  String $pg_user = "postgres"
+) {
+  include "profile::pam_ldap"
+
+  $password_seed = lookup("base_installation::puppet_pass_seed")
+  $ldap_server = lookup("base_installation::ldap_server")
+  $ldap_base   = lookup("base_installation::ldap_base")
+  $ldap_dn     = lookup("base_installation::ldap_dn")
+  $ldap_password = generate_password(24, $password_seed, "ldap")
+  $ldap_attribute = "cn"
+
+  file { "/etc/pam_ldap.d/postgresql.conf":
+    ensure  => "present",
+    mode    => "0400",
+    owner   => $pg_user,
+    group   => "root",
+    content => template("profile/postgresql/pam_ldap_postgresql.conf.erb"),
+    require => File["/etc/pam_ldap.d"],
+  } ->
+  file { "/etc/pam.d/postgresql":
+    ensure => "present",
+    mode   => "0644",
+    owner  => "root",
+    group  => "root",
+    source => "puppet:///modules/profile/postgresql/pam_postgresql"
+  }
+}
diff --git a/modules/profile/manifests/postgresql/pam_ldap_pgbouncer.pp b/modules/profile/manifests/postgresql/pam_ldap_pgbouncer.pp
new file mode 100644
index 0000000..67714f2
--- /dev/null
+++ b/modules/profile/manifests/postgresql/pam_ldap_pgbouncer.pp
@@ -0,0 +1,33 @@
+class profile::postgresql::pam_ldap_pgbouncer (
+  String $pg_user = "postgres"
+) {
+  include "profile::pam_ldap"
+
+  $password_seed = lookup("base_installation::puppet_pass_seed")
+  $ldap_server = lookup("base_installation::ldap_server")
+  $ldap_base   = lookup("base_installation::ldap_base")
+  $ldap_dn     = lookup("base_installation::ldap_dn")
+  $ldap_password = generate_password(24, $password_seed, "ldap")
+  $ldap_attribute = "uid"
+  $ldap_filter = lookup("role::backup::postgresql::pgbouncer_access_filter", { "default_value" => undef })
+
+  if empty($ldap_filter) {
+    fail("need ldap filter for pgbouncer")
+  }
+
+  file { "/etc/pam_ldap.d/pgbouncer.conf":
+    ensure  => "present",
+    mode    => "0600",
+    owner   => $pg_user,
+    group   => "root",
+    content => template("profile/postgresql/pam_ldap_pgbouncer.conf.erb"),
+    require => File["/etc/pam_ldap.d"],
+  } ->
+  file { "/etc/pam.d/pgbouncer":
+    ensure => "present",
+    mode   => "0644",
+    owner  => "root",
+    group  => "root",
+    source => "puppet:///modules/profile/postgresql/pam_pgbouncer"
+  }
+}
diff --git a/modules/profile/manifests/postgresql/replication.pp b/modules/profile/manifests/postgresql/replication.pp
new file mode 100644
index 0000000..2fcb71c
--- /dev/null
+++ b/modules/profile/manifests/postgresql/replication.pp
@@ -0,0 +1,72 @@
+define profile::postgresql::replication (
+  Boolean          $handle_role   = false,
+  Boolean          $handle_config = false,
+  Boolean          $add_self_role = false,
+  Boolean          $handle_slot   = false,
+  Optional[String] $target        = undef,
+) {
+  include "profile::postgresql::pam_ldap"
+
+  $host_cn = $title
+  $host_infos = find_host($facts["ldapvar"]["other"], $host_cn)
+
+  if empty($host_infos) {
+    fail("Unable to find host for replication")
+  }
+
+  if empty($target) {
+    $pg_version = undef
+  } else {
+    $pg_version = "10"
+  }
+
+  $host_infos["ipHostNumber"].each |$ip| {
+    $infos = split($ip, "/")
+    $ipaddress = $infos[0]
+    if (length($infos) == 1 and $ipaddress =~ /:/) {
+      $mask = "128"
+    } elsif (length($infos) == 1) {
+      $mask = "32"
+    } else {
+      $mask = $infos[1]
+    }
+
+    postgresql::server::pg_hba_rule { "allow TCP access for replication to user $host_cn from $ipaddress/$mask":
+      type               => 'hostssl',
+      database           => 'replication',
+      user               => $host_cn,
+      address            => "$ipaddress/$mask",
+      auth_method        => 'pam',
+      order              => "06-01",
+      target             => $target,
+      postgresql_version => $pg_version,
+    }
+  }
+
+  if $handle_config {
+    ensure_resource("postgresql::server::config_entry", "wal_level", {
+      value => "logical",
+    })
+  }
+
+  if $handle_role {
+    postgresql::server::role { $host_cn:
+      replication => true,
+    }
+
+    if $add_self_role {
+      $ldap_cn = lookup("base_installation::ldap_cn")
+
+      # Needed to be replicated to the backup and be able to recover later
+      ensure_resource("postgresql::server::role", $ldap_cn, {
+        replication => true,
+      })
+    }
+  }
+
+  if $handle_slot {
+    postgresql_replication_slot { regsubst($host_cn, '-', "_", "G"):
+      ensure => present
+    }
+  }
+}
diff --git a/modules/profile/manifests/postgresql/ssl.pp b/modules/profile/manifests/postgresql/ssl.pp
new file mode 100644
index 0000000..dc56c0b
--- /dev/null
+++ b/modules/profile/manifests/postgresql/ssl.pp
@@ -0,0 +1,82 @@
+define profile::postgresql::ssl (
+  Optional[String]  $cert                 = undef,
+  Optional[String]  $key                  = undef,
+  Optional[String]  $certname             = undef,
+  Optional[Boolean] $copy_keys            = true,
+  Optional[Boolean] $handle_config_entry  = false,
+  Optional[Boolean] $handle_concat_config = false,
+  Optional[String]  $pg_user              = "postgres",
+  Optional[String]  $pg_group             = "postgres",
+) {
+  $datadir = $title
+
+  file { "$datadir/certs":
+    ensure  => directory,
+    mode    => "0700",
+    owner   => $pg_user,
+    group   => $pg_group,
+    require => File[$datadir],
+  }
+
+  if empty($cert) or empty($key) {
+    if empty($certname) {
+      fail("A certificate name is necessary to generate ssl certificate")
+    }
+
+    ssl::self_signed_certificate { $certname:
+      common_name  => $certname,
+      country      => "FR",
+      days         => "3650",
+      organization => "Immae",
+      owner        => $pg_user,
+      group        => $pg_group,
+      directory    => "$datadir/certs",
+    }
+
+    $ssl_key  = "$datadir/certs/$certname.key"
+    $ssl_cert = "$datadir/certs/$certname.crt"
+  } elsif $copy_keys {
+    $ssl_key  = "$datadir/certs/privkey.pem"
+    $ssl_cert = "$datadir/certs/cert.pem"
+
+    file { $ssl_cert:
+      source  => "file://$cert",
+      mode    => "0600",
+      links   => "follow",
+      owner   => $pg_user,
+      group   => $pg_group,
+      require => File["$datadir/certs"],
+    }
+    file { $ssl_key:
+      source  => "file://$key",
+      mode    => "0600",
+      links   => "follow",
+      owner   => $pg_user,
+      group   => $pg_group,
+      require => File["$datadir/certs"],
+    }
+  } else {
+    $ssl_key  = $key
+    $ssl_cert = $cert
+  }
+
+  if $handle_config_entry {
+    postgresql::server::config_entry { "ssl":
+      value => "on",
+    }
+
+    postgresql::server::config_entry { "ssl_cert_file":
+      value => $ssl_cert,
+    }
+
+    postgresql::server::config_entry { "ssl_key_file":
+      value => $ssl_key,
+    }
+  } elsif $handle_concat_config {
+    concat::fragment { "$datadir/postgresql.conf ssl config":
+      target  => "$datadir/postgresql.conf",
+      content => "ssl = on\nssl_key_file = '$ssl_key'\nssl_cert_file = '$ssl_cert'\n"
+    }
+  }
+
+}