From 80225f1917ac5919135af2a4815fce5faf3fa080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sun, 24 Jun 2018 21:25:13 +0200 Subject: Add pylog2rotate for backups --- modules/role/manifests/backup.pp | 8 ++++++++ modules/role/manifests/backup/postgresql.pp | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp index b35c542..51b689d 100644 --- a/modules/role/manifests/backup.pp +++ b/modules/role/manifests/backup.pp @@ -17,6 +17,14 @@ class role::backup ( include "role::backup::postgresql" + ensure_packages(["python", "python-pip"]) + package { "pylog2rotate": + source => "git+https://github.com/avian2/pylog2rotate", + ensure => present, + provider => "pip3", + require => Package["python-pip"], + } + ensure_packages(["rsync"]) ssh_keygen { $user: diff --git a/modules/role/manifests/backup/postgresql.pp b/modules/role/manifests/backup/postgresql.pp index e635611..e891195 100644 --- a/modules/role/manifests/backup/postgresql.pp +++ b/modules/role/manifests/backup/postgresql.pp @@ -153,12 +153,19 @@ class role::backup::postgresql inherits role::backup { description => "Backup the database", }, { - command => "/usr/bin/rm -f $(ls -1 $pg_backup_path/*.sql | sort -r | sed -e '1,16d')", + command => "/usr/bin/rm -f $(ls -1 $pg_backup_path/*.sql | grep -v 'T22:' | sort -r | sed -e '1,12d')", user => $pg_user, hour => 3, minute => 0, description => "Cleanup the database backups", }, + { + command => "cd $pg_backup_path ; /usr/bin/rm -f $(ls -1 *T22*.sql | log2rotate --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M:%S+02:00.sql')", + user => $pg_user, + hour => 3, + minute => 1, + description => "Cleanup the database backups exponentially", + }, ] } -- cgit v1.2.3 From 96cda4dc280242305aba5aa509f95cd981784a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 25 Jun 2018 23:29:23 +0200 Subject: Add real hostname puppetVar to ldap --- modules/base_installation/templates/puppet/host_ldap.info.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/base_installation/templates/puppet/host_ldap.info.erb b/modules/base_installation/templates/puppet/host_ldap.info.erb index 9c79d3c..7afdf2d 100644 --- a/modules/base_installation/templates/puppet/host_ldap.info.erb +++ b/modules/base_installation/templates/puppet/host_ldap.info.erb @@ -12,6 +12,7 @@ objectclass: ipHost <% unless @ips["v6"].nil? -%>ipHostNumber: <%= @ips["v6"]["ipAddress"] %>/<%= @ips["v6"]["mask"] %><%- end %> <%- end -%> environment: <%= @environment %> +puppetVar: real_hostname=<%= @real_hostname %> userpassword: {SSHA}<%= Base64.encode64(Digest::SHA1.digest(@ldap_password+@ssha_ldap_seed)+@ssha_ldap_seed).chomp! %> EOF #### Or modify an existing entry: -- cgit v1.2.3 From 2ec6746cf2564470f514319cb73c36d867be4078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 26 Jun 2018 00:39:14 +0200 Subject: Cleanup backup hosts information lookup --- .../lib/puppet/parser/functions/find_host.rb | 18 ++++++++++++++++++ modules/role/manifests/backup/postgresql.pp | 10 +++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 modules/base_installation/lib/puppet/parser/functions/find_host.rb diff --git a/modules/base_installation/lib/puppet/parser/functions/find_host.rb b/modules/base_installation/lib/puppet/parser/functions/find_host.rb new file mode 100644 index 0000000..79aae5c --- /dev/null +++ b/modules/base_installation/lib/puppet/parser/functions/find_host.rb @@ -0,0 +1,18 @@ +module Puppet::Parser::Functions + newfunction(:find_host, :type => :rvalue, :doc => <<-EOS +This function searches through the hosts to find the one with matching cn + EOS + ) do |arguments| + + if (arguments.size != 2) then + raise(Puppet::ParseError, "grep(): Wrong number of arguments given #{arguments.size} for 2") + end + + hosts = arguments[0] + cn = arguments[1] + + hosts.find { |host| host["cn"].any? { |cn_| cn_ == cn } } + end +end + +# vim: set ts=2 sw=2 et : diff --git a/modules/role/manifests/backup/postgresql.pp b/modules/role/manifests/backup/postgresql.pp index e891195..b26773c 100644 --- a/modules/role/manifests/backup/postgresql.pp +++ b/modules/role/manifests/backup/postgresql.pp @@ -94,7 +94,15 @@ class role::backup::postgresql inherits role::backup { } } - $pg_backup_hosts.each |$pg_backup_host, $pg_infos| { + $pg_backup_hosts.each |$backup_host_cn, $pg_infos| { + $host = find_host($facts["ldapvar"]["other"], $backup_host_cn) + if empty($host) { + $pg_backup_host = $backup_host_cn + } elsif has_key($host["vars"], "host") { + $pg_backup_host = $host["vars"]["host"][0] + } else { + $pg_backup_host = $host["vars"]["real_hostname"][0] + } $pg_path = "$mountpoint/$pg_backup_host/postgresql" $pg_backup_path = "$mountpoint/$pg_backup_host/postgresql_backup" $pg_host = "$pg_backup_host" -- cgit v1.2.3 From c230c6632aba600c34301e1664a4b16acec050e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 5 Jun 2018 23:53:22 +0200 Subject: Add etherpad role --- environments/global/roles/etherpad.yaml | 3 + modules/role/manifests/etherpad.pp | 129 ++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 environments/global/roles/etherpad.yaml create mode 100644 modules/role/manifests/etherpad.pp diff --git a/environments/global/roles/etherpad.yaml b/environments/global/roles/etherpad.yaml new file mode 100644 index 0000000..f8781e1 --- /dev/null +++ b/environments/global/roles/etherpad.yaml @@ -0,0 +1,3 @@ +--- +classes: + role::etherpad: ~ diff --git a/modules/role/manifests/etherpad.pp b/modules/role/manifests/etherpad.pp new file mode 100644 index 0000000..826525e --- /dev/null +++ b/modules/role/manifests/etherpad.pp @@ -0,0 +1,129 @@ +class role::etherpad ( +) { + $password_seed = lookup("base_installation::puppet_pass_seed") + + include "base_installation" + + include "profile::tools" + include "profile::postgresql" + include "profile::apache" + + ensure_packages(["npm"]) + ensure_packages(["abiword"]) + ensure_packages(["libreoffice-fresh", "libreoffice-fresh-fr", "java-runtime-common", "jre8-openjdk"]) + ensure_packages(["tidy"]) + aur::package { "etherpad-lite": } + + $modules = [ + "ep_aa_file_menu_toolbar", + "ep_adminpads", + "ep_align", + "ep_bookmark", + "ep_clear_formatting", + "ep_colors", + "ep_copy_paste_select_all", + "ep_cursortrace", + "ep_embedmedia", + "ep_font_family", + "ep_font_size", + "ep_headings2", + "ep_ldapauth", + "ep_line_height", + "ep_markdown", + "ep_previewimages", + "ep_ruler", + "ep_scrollto", + "ep_set_title_on_pad", + "ep_subscript_and_superscript", + "ep_timesliderdiff" + ] + + $modules.each |$module| { + exec { "npm_install_$module": + command => "/usr/bin/npm install $module", + unless => "/usr/bin/test -d /usr/share/etherpad-lite/node_modules/$module", + cwd => "/usr/share/etherpad-lite/", + environment => "HOME=/root", + require => Aur::Package["etherpad-lite"], + before => Service["etherpad-lite"], + notify => Service["etherpad-lite"], + } + -> + file { "/usr/share/etherpad-lite/node_modules/$module/.ep_initialized": + ensure => present, + mode => "0644", + before => Service["etherpad-lite"], + } + } + + service { "etherpad-lite": + enable => true, + ensure => "running", + require => Aur::Package["etherpad-lite"], + subscribe => Aur::Package["etherpad-lite"], + } + + $web_host = "outils-1.v.immae.eu" + $pg_db = "etherpad-lite" + $pg_user = "etherpad-lite" + $pg_password = generate_password(24, $password_seed, "postgres_etherpad") + + file { "/var/lib/postgres/data/certs": + ensure => directory, + mode => "0700", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => File["/var/lib/postgres"], + } + + file { "/var/lib/postgres/data/certs/cert.pem": + source => "file:///etc/letsencrypt/live/$web_host/cert.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] + } + + file { "/var/lib/postgres/data/certs/privkey.pem": + source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] + } + + postgresql::server::config_entry { "wal_level": + value => "logical", + } + + postgresql::server::config_entry { "ssl": + value => "on", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::config_entry { "ssl_cert_file": + value => "/var/lib/postgres/data/certs/cert.pem", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::config_entry { "ssl_key_file": + value => "/var/lib/postgres/data/certs/privkey.pem", + require => Letsencrypt::Certonly[$web_host], + } + + postgresql::server::db { $pg_db: + user => $pg_user, + password => postgresql_password($pg_user, $pg_password), + } + + postgresql::server::pg_hba_rule { "allow local access to $pg_user user": + type => 'local', + database => $pg_db, + user => $pg_user, + auth_method => 'ident', + order => "05-01", + } + +} -- cgit v1.2.3 From a1c3146595f8f6c7b78adfca8388dd35083b4c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 18 Jun 2018 10:40:00 +0200 Subject: Patch libreoffice --- .gitmodules | 3 +++ external_modules/patch | 1 + modules/role/files/etherpad/libreoffice_patch.diff | 11 +++++++++++ modules/role/manifests/etherpad.pp | 3 +++ 4 files changed, 18 insertions(+) create mode 160000 external_modules/patch create mode 100644 modules/role/files/etherpad/libreoffice_patch.diff diff --git a/.gitmodules b/.gitmodules index 0f21622..d68cf4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -64,3 +64,6 @@ [submodule "external_modules/slack"] path = external_modules/slack url = git://git.immae.eu/github/cegeka/puppet-slack +[submodule "external_modules/patch"] + path = external_modules/patch + url = git://git.immae.eu/github/tohuwabohu/puppet-patch.git diff --git a/external_modules/patch b/external_modules/patch new file mode 160000 index 0000000..fed3814 --- /dev/null +++ b/external_modules/patch @@ -0,0 +1 @@ +Subproject commit fed3814e759347d72b919ebeb1b6837c5d13373f diff --git a/modules/role/files/etherpad/libreoffice_patch.diff b/modules/role/files/etherpad/libreoffice_patch.diff new file mode 100644 index 0000000..dbfdf1a --- /dev/null +++ b/modules/role/files/etherpad/libreoffice_patch.diff @@ -0,0 +1,11 @@ +--- a/LibreOffice.js 2018-06-18 09:54:15.087161212 +0200 ++++ b/LibreOffice.js 2018-06-18 10:33:27.534055021 +0200 +@@ -63,6 +63,7 @@ + '--invisible', + '--nologo', + '--nolockcheck', ++ '-env:UserInstallation=file:///tmp/', + '--convert-to', task.type, + task.srcFile, + '--outdir', tmpDir + diff --git a/modules/role/manifests/etherpad.pp b/modules/role/manifests/etherpad.pp index 826525e..476a210 100644 --- a/modules/role/manifests/etherpad.pp +++ b/modules/role/manifests/etherpad.pp @@ -13,6 +13,9 @@ class role::etherpad ( ensure_packages(["libreoffice-fresh", "libreoffice-fresh-fr", "java-runtime-common", "jre8-openjdk"]) ensure_packages(["tidy"]) aur::package { "etherpad-lite": } + -> patch::file { "/usr/share/etherpad-lite/src/node/utils/LibreOffice.js": + diff_source => "puppet:///modules/role/etherpad/libreoffice_patch.diff", + } $modules = [ "ep_aa_file_menu_toolbar", -- cgit v1.2.3 From f568173a3d8a43ac30fa9294a75c260042b9e415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Mon, 18 Jun 2018 14:09:05 +0200 Subject: Add postgresql_master profile --- .../profile/files/postgresql_master/pam_postgresql | 3 + modules/profile/manifests/postgresql_master.pp | 116 +++++++++++++++++++++ .../postgresql_master/pam_ldap_postgresql.conf.erb | 6 ++ modules/role/manifests/etherpad.pp | 52 ++------- 4 files changed, 131 insertions(+), 46 deletions(-) create mode 100644 modules/profile/files/postgresql_master/pam_postgresql create mode 100644 modules/profile/manifests/postgresql_master.pp create mode 100644 modules/profile/templates/postgresql_master/pam_ldap_postgresql.conf.erb diff --git a/modules/profile/files/postgresql_master/pam_postgresql b/modules/profile/files/postgresql_master/pam_postgresql new file mode 100644 index 0000000..70a90ae --- /dev/null +++ b/modules/profile/files/postgresql_master/pam_postgresql @@ -0,0 +1,3 @@ +auth required pam_ldap.so config=/etc/pam_ldap.d/postgresql.conf +account required pam_ldap.so config=/etc/pam_ldap.d/postgresql.conf + diff --git a/modules/profile/manifests/postgresql_master.pp b/modules/profile/manifests/postgresql_master.pp new file mode 100644 index 0000000..3f68890 --- /dev/null +++ b/modules/profile/manifests/postgresql_master.pp @@ -0,0 +1,116 @@ +define profile::postgresql_master ( + $letsencrypt_host = undef, + $backup_hosts = [], +) { + $password_seed = lookup("base_installation::puppet_pass_seed") + + ensure_resource("file", "/var/lib/postgres/data/certs", { + ensure => directory, + mode => "0700", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => File["/var/lib/postgres"], + }) + + ensure_resource("file", "/var/lib/postgres/data/certs/cert.pem", { + source => "file:///etc/letsencrypt/live/$letsencrypt_host/cert.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$letsencrypt_host], File["/var/lib/postgres/data/certs"]] + }) + + ensure_resource("file", "/var/lib/postgres/data/certs/privkey.pem", { + source => "file:///etc/letsencrypt/live/$letsencrypt_host/privkey.pem", + mode => "0600", + links => "follow", + owner => $::profile::postgresql::pg_user, + group => $::profile::postgresql::pg_user, + require => [Letsencrypt::Certonly[$letsencrypt_host], File["/var/lib/postgres/data/certs"]] + }) + + ensure_resource("postgresql::server::config_entry", "wal_level", { + value => "logical", + }) + + ensure_resource("postgresql::server::config_entry", "ssl", { + value => "on", + require => Letsencrypt::Certonly[$letsencrypt_host], + }) + + ensure_resource("postgresql::server::config_entry", "ssl_cert_file", { + value => "/var/lib/postgres/data/certs/cert.pem", + require => Letsencrypt::Certonly[$letsencrypt_host], + }) + + ensure_resource("postgresql::server::config_entry", "ssl_key_file", { + value => "/var/lib/postgres/data/certs/privkey.pem", + require => Letsencrypt::Certonly[$letsencrypt_host], + }) + + $backup_hosts.each |$backup_host| { + ensure_packages(["pam_ldap"]) + + $facts["ldapvar"]["other"].each |$host| { + if ($host["cn"][0] == $backup_host) { + $host["ipHostNumber"].each |$ip| { + $infos = split($ip, "/") + $ipaddress = $infos[0] + if (length($infos) == 1 and $ipaddress =~ /:/) { + $mask = "128" + } elsif (length($infos) == 1) { + $mask = "32" + } else { + $mask = $infos[1] + } + + postgresql::server::pg_hba_rule { "allow TCP access to replication user from backup for replication from $ipaddress/$mask": + type => 'hostssl', + database => 'replication', + user => $backup_host, + address => "$ipaddress/$mask", + auth_method => 'pam', + order => "06-01", + } + } + + postgresql::server::role { $backup_host: + replication => true, + } + + postgresql_replication_slot { regsubst($backup_host, '-', "_", "G"): + ensure => present + } + } + } + + $ldap_server = lookup("base_installation::ldap_server") + $ldap_base = lookup("base_installation::ldap_base") + $ldap_dn = lookup("base_installation::ldap_dn") + $ldap_password = generate_password(24, $password_seed, "ldap") + $ldap_attribute = "cn" + + file { "/etc/pam_ldap.d": + ensure => directory, + mode => "0755", + owner => "root", + group => "root", + } -> + file { "/etc/pam_ldap.d/postgresql.conf": + ensure => "present", + mode => "0600", + owner => $::profile::postgresql::pg_user, + group => "root", + content => template("profile/postgresql_master/pam_ldap_postgresql.conf.erb"), + } -> + file { "/etc/pam.d/postgresql": + ensure => "present", + mode => "0644", + owner => "root", + group => "root", + source => "puppet:///modules/profile/postgresql_master/pam_postgresql" + } + } + +} diff --git a/modules/profile/templates/postgresql_master/pam_ldap_postgresql.conf.erb b/modules/profile/templates/postgresql_master/pam_ldap_postgresql.conf.erb new file mode 100644 index 0000000..f3d9674 --- /dev/null +++ b/modules/profile/templates/postgresql_master/pam_ldap_postgresql.conf.erb @@ -0,0 +1,6 @@ +host <%= @ldap_server %> + +base <%= @ldap_base %> +binddn <%= @ldap_dn %> +bindpw <%= @ldap_password %> +pam_login_attribute <%= @ldap_attribute %> diff --git a/modules/role/manifests/etherpad.pp b/modules/role/manifests/etherpad.pp index 476a210..a43f146 100644 --- a/modules/role/manifests/etherpad.pp +++ b/modules/role/manifests/etherpad.pp @@ -66,54 +66,14 @@ class role::etherpad ( subscribe => Aur::Package["etherpad-lite"], } - $web_host = "outils-1.v.immae.eu" - $pg_db = "etherpad-lite" - $pg_user = "etherpad-lite" + $web_host = "outils-1.v.immae.eu" + $pg_db = "etherpad-lite" + $pg_user = "etherpad-lite" $pg_password = generate_password(24, $password_seed, "postgres_etherpad") - file { "/var/lib/postgres/data/certs": - ensure => directory, - mode => "0700", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => File["/var/lib/postgres"], - } - - file { "/var/lib/postgres/data/certs/cert.pem": - source => "file:///etc/letsencrypt/live/$web_host/cert.pem", - mode => "0600", - links => "follow", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] - } - - file { "/var/lib/postgres/data/certs/privkey.pem": - source => "file:///etc/letsencrypt/live/$web_host/privkey.pem", - mode => "0600", - links => "follow", - owner => $::profile::postgresql::pg_user, - group => $::profile::postgresql::pg_user, - require => [Letsencrypt::Certonly[$web_host], File["/var/lib/postgres/data/certs"]] - } - - postgresql::server::config_entry { "wal_level": - value => "logical", - } - - postgresql::server::config_entry { "ssl": - value => "on", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::config_entry { "ssl_cert_file": - value => "/var/lib/postgres/data/certs/cert.pem", - require => Letsencrypt::Certonly[$web_host], - } - - postgresql::server::config_entry { "ssl_key_file": - value => "/var/lib/postgres/data/certs/privkey.pem", - require => Letsencrypt::Certonly[$web_host], + profile::postgresql_master { "postgresql master for etherpad": + letsencrypt_host => $web_host, + backup_hosts => ["backup-1"], } postgresql::server::db { $pg_db: -- cgit v1.2.3 From 580bd7fc5d4b078f8dec2fd440e5989b5f963f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 23 Jun 2018 14:02:48 +0200 Subject: Add json file --- modules/role/manifests/etherpad.pp | 26 +++++-- modules/role/templates/etherpad/settings.json.erb | 93 +++++++++++++++++++++++ 2 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 modules/role/templates/etherpad/settings.json.erb diff --git a/modules/role/manifests/etherpad.pp b/modules/role/manifests/etherpad.pp index a43f146..109da96 100644 --- a/modules/role/manifests/etherpad.pp +++ b/modules/role/manifests/etherpad.pp @@ -1,6 +1,20 @@ class role::etherpad ( ) { $password_seed = lookup("base_installation::puppet_pass_seed") + $web_host = lookup("base_installation::real_hostname") + $web_listen = "0.0.0.0" + $web_port = 18000 + $pg_db = "etherpad-lite" + $pg_user = "etherpad-lite" + $pg_password = generate_password(24, $password_seed, "postgres_etherpad") + + $ldap_server = lookup("base_installation::ldap_server") + $ldap_base = lookup("base_installation::ldap_base") + $ldap_dn = lookup("base_installation::ldap_dn") + $ldap_account_pattern = "(&(memberOf=cn=users,cn=etherpad,ou=services,dc=immae,dc=eu)(uid={{username}}))" + $ldap_group_pattern = "(memberOf=cn=groups,cn=etherpad,ou=services,dc=immae,dc=eu)" + $ldap_password = generate_password(24, $password_seed, "ldap") + include "base_installation" @@ -16,6 +30,13 @@ class role::etherpad ( -> patch::file { "/usr/share/etherpad-lite/src/node/utils/LibreOffice.js": diff_source => "puppet:///modules/role/etherpad/libreoffice_patch.diff", } + -> file { "/etc/etherpad-lite/settings.json": + ensure => present, + owner => "etherpad-lite", + group => "etherpad-lite", + notify => Service["etherpad-lite"], + content => template("role/etherpad/settings.json.erb"), + } $modules = [ "ep_aa_file_menu_toolbar", @@ -66,11 +87,6 @@ class role::etherpad ( subscribe => Aur::Package["etherpad-lite"], } - $web_host = "outils-1.v.immae.eu" - $pg_db = "etherpad-lite" - $pg_user = "etherpad-lite" - $pg_password = generate_password(24, $password_seed, "postgres_etherpad") - profile::postgresql_master { "postgresql master for etherpad": letsencrypt_host => $web_host, backup_hosts => ["backup-1"], diff --git a/modules/role/templates/etherpad/settings.json.erb b/modules/role/templates/etherpad/settings.json.erb new file mode 100644 index 0000000..dfd69c1 --- /dev/null +++ b/modules/role/templates/etherpad/settings.json.erb @@ -0,0 +1,93 @@ +{ + "title": "Etherpad", + "favicon": "favicon.ico", + + "ip": "<%= @web_listen %>", + "port" : <%= @web_port %>, + "showSettingsInAdminPage" : false, + "dbType" : "postgres", + "dbSettings" : { + "user" : "<%= @pg_user %>", + "host" : "/run/postgresql", + "password": "", + "database": "<%= @pg_db %>", + "charset" : "utf8mb4" + }, + + "defaultPadText" : "Welcome to Etherpad!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nGet involved with Etherpad at http:\/\/etherpad.org\n", + "padOptions": { + "noColors": false, + "showControls": true, + "showChat": true, + "showLineNumbers": true, + "useMonospaceFont": false, + "userName": false, + "userColor": false, + "rtl": false, + "alwaysShowChat": false, + "chatAndUsers": false, + "lang": "en-gb" + }, + + "suppressErrorsInPadText" : false, + "requireSession" : false, + "editOnly" : false, + "sessionNoPassword" : false, + "minify" : true, + "maxAge" : 21600, + "abiword" : "/usr/bin/abiword", + "soffice" : "/usr/bin/soffice", + "tidyHtml" : "/usr/bin/tidy", + "allowUnknownFileEnds" : true, + "requireAuthentication" : false, + "requireAuthorization" : false, + "trustProxy" : false, + "disableIPlogging" : false, + "automaticReconnectionTimeout" : 0, + "scrollWhenFocusLineIsOutOfViewport": { + "percentage": { + "editionAboveViewport": 0, + "editionBelowViewport": 0 + }, + "duration": 0, + "scrollWhenCaretIsInTheLastLineOfViewport": false, + "percentageToScrollWhenUserPressesArrowUp": 0 + }, + "users": { + "ldapauth": { + "url": "ldaps://<%= @ldap_server %>", + "accountBase": "<%= @ldap_base %>", + "accountPattern": "<%= @ldap_account_pattern %>", + "displayNameAttribute": "cn", + "searchDN": "<%= @ldap_dn %>", + "searchPWD": "<%= @ldap_password %>", + "groupSearchBase": "<%= @ldap_base %>", + "groupAttribute": "member", + "groupAttributeIsDN": true, + "searchScope": "sub", + "groupSearch": "<%= @ldap_group_pattern %>", + "anonymousReadonly": false + } + }, + "socketTransportProtocols" : ["xhr-polling", "jsonp-polling", "htmlfile"], + "loadTest": false, + "indentationOnNewLine": false, + "toolbar": { + "left": [ + ["bold", "italic", "underline", "strikethrough"], + ["orderedlist", "unorderedlist", "indent", "outdent"], + ["undo", "redo"], + ["clearauthorship"] + ], + "right": [ + ["importexport", "timeslider", "savedrevision"], + ["settings", "embed"], + ["showusers"] + ], + "timeslider": [ + ["timeslider_export", "timeslider_returnToPad"] + ] + }, + "loglevel": "INFO", + "logconfig" : { "appenders": [ { "type": "console" } ] } +} -- cgit v1.2.3 From 9313fa2ea3c7b796b448f6249f13a588c6618889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 26 Jun 2018 00:27:26 +0200 Subject: Add find_host function to help finding host --- modules/profile/manifests/postgresql_master.pp | 51 +++++++++++++------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/modules/profile/manifests/postgresql_master.pp b/modules/profile/manifests/postgresql_master.pp index 3f68890..9966f0d 100644 --- a/modules/profile/manifests/postgresql_master.pp +++ b/modules/profile/manifests/postgresql_master.pp @@ -52,36 +52,35 @@ define profile::postgresql_master ( $backup_hosts.each |$backup_host| { ensure_packages(["pam_ldap"]) - $facts["ldapvar"]["other"].each |$host| { - if ($host["cn"][0] == $backup_host) { - $host["ipHostNumber"].each |$ip| { - $infos = split($ip, "/") - $ipaddress = $infos[0] - if (length($infos) == 1 and $ipaddress =~ /:/) { - $mask = "128" - } elsif (length($infos) == 1) { - $mask = "32" - } else { - $mask = $infos[1] - } - - postgresql::server::pg_hba_rule { "allow TCP access to replication user from backup for replication from $ipaddress/$mask": - type => 'hostssl', - database => 'replication', - user => $backup_host, - address => "$ipaddress/$mask", - auth_method => 'pam', - order => "06-01", - } + $host = find_host($facts["ldapvar"]["other"], $backup_host) + unless empty($host) { + $host["ipHostNumber"].each |$ip| { + $infos = split($ip, "/") + $ipaddress = $infos[0] + if (length($infos) == 1 and $ipaddress =~ /:/) { + $mask = "128" + } elsif (length($infos) == 1) { + $mask = "32" + } else { + $mask = $infos[1] } - postgresql::server::role { $backup_host: - replication => true, + postgresql::server::pg_hba_rule { "allow TCP access to replication user from backup for replication from $ipaddress/$mask": + type => 'hostssl', + database => 'replication', + user => $backup_host, + address => "$ipaddress/$mask", + auth_method => 'pam', + order => "06-01", } + } - postgresql_replication_slot { regsubst($backup_host, '-', "_", "G"): - ensure => present - } + postgresql::server::role { $backup_host: + replication => true, + } + + postgresql_replication_slot { regsubst($backup_host, '-', "_", "G"): + ensure => present } } -- cgit v1.2.3 From 5feedbb4f3e35cfb63201a360a685127b2608345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Tue, 26 Jun 2018 01:04:09 +0200 Subject: Allow host to replicate the cluster from backup --- modules/profile/manifests/postgresql_master.pp | 50 ++++++++--------- modules/role/manifests/backup/postgresql.pp | 62 +++++++++++++++++++--- .../templates/backup/pam_ldap_pgbouncer.conf.erb | 2 +- python/reboot_ovh_cloud_instance.py | 3 ++ 4 files changed, 84 insertions(+), 33 deletions(-) diff --git a/modules/profile/manifests/postgresql_master.pp b/modules/profile/manifests/postgresql_master.pp index 9966f0d..6f8854d 100644 --- a/modules/profile/manifests/postgresql_master.pp +++ b/modules/profile/manifests/postgresql_master.pp @@ -83,33 +83,33 @@ define profile::postgresql_master ( ensure => present } } + } - $ldap_server = lookup("base_installation::ldap_server") - $ldap_base = lookup("base_installation::ldap_base") - $ldap_dn = lookup("base_installation::ldap_dn") - $ldap_password = generate_password(24, $password_seed, "ldap") - $ldap_attribute = "cn" + $ldap_server = lookup("base_installation::ldap_server") + $ldap_base = lookup("base_installation::ldap_base") + $ldap_dn = lookup("base_installation::ldap_dn") + $ldap_password = generate_password(24, $password_seed, "ldap") + $ldap_attribute = "cn" - file { "/etc/pam_ldap.d": - ensure => directory, - mode => "0755", - owner => "root", - group => "root", - } -> - file { "/etc/pam_ldap.d/postgresql.conf": - ensure => "present", - mode => "0600", - owner => $::profile::postgresql::pg_user, - group => "root", - content => template("profile/postgresql_master/pam_ldap_postgresql.conf.erb"), - } -> - file { "/etc/pam.d/postgresql": - ensure => "present", - mode => "0644", - owner => "root", - group => "root", - source => "puppet:///modules/profile/postgresql_master/pam_postgresql" - } + file { "/etc/pam_ldap.d": + ensure => directory, + mode => "0755", + owner => "root", + group => "root", + } -> + file { "/etc/pam_ldap.d/postgresql.conf": + ensure => "present", + mode => "0600", + owner => $::profile::postgresql::pg_user, + group => "root", + content => template("profile/postgresql_master/pam_ldap_postgresql.conf.erb"), + } -> + file { "/etc/pam.d/postgresql": + ensure => "present", + mode => "0644", + owner => "root", + group => "root", + source => "puppet:///modules/profile/postgresql_master/pam_postgresql" } } diff --git a/modules/role/manifests/backup/postgresql.pp b/modules/role/manifests/backup/postgresql.pp index b26773c..9e1c9f7 100644 --- a/modules/role/manifests/backup/postgresql.pp +++ b/modules/role/manifests/backup/postgresql.pp @@ -13,7 +13,7 @@ class role::backup::postgresql inherits role::backup { $ldap_server = lookup("base_installation::ldap_server") $ldap_base = lookup("base_installation::ldap_base") $ldap_dn = lookup("base_installation::ldap_dn") - $ldap_attribute = "uid" + $pgbouncer_ldap_attribute = "uid" $pg_slot = regsubst($ldap_cn, '-', "_", "G") @@ -71,18 +71,13 @@ class role::backup::postgresql inherits role::backup { ], } - file { "/etc/pam_ldap.d": - ensure => directory, - mode => "0755", - owner => "root", - group => "root", - } -> file { "/etc/pam_ldap.d/pgbouncer.conf": ensure => "present", mode => "0600", owner => $pg_user, group => "root", content => template("role/backup/pam_ldap_pgbouncer.conf.erb"), + require => File["/etc/pam_ldap.d"], } -> file { "/etc/pam.d/pgbouncer": ensure => "present", @@ -94,6 +89,29 @@ class role::backup::postgresql inherits role::backup { } } + $ldap_attribute = "cn" + + file { "/etc/pam_ldap.d": + ensure => directory, + mode => "0755", + owner => "root", + group => "root", + } -> + file { "/etc/pam_ldap.d/postgresql.conf": + ensure => "present", + mode => "0600", + owner => $pg_user, + group => "root", + content => template("profile/postgresql_master/pam_ldap_postgresql.conf.erb"), + } -> + file { "/etc/pam.d/postgresql": + ensure => "present", + mode => "0644", + owner => "root", + group => "root", + source => "puppet:///modules/profile/postgresql_master/pam_postgresql" + } + $pg_backup_hosts.each |$backup_host_cn, $pg_infos| { $host = find_host($facts["ldapvar"]["other"], $backup_host_cn) if empty($host) { @@ -108,6 +126,31 @@ class role::backup::postgresql inherits role::backup { $pg_host = "$pg_backup_host" $pg_port = $pg_infos["dbport"] + unless empty($host) { + $host["ipHostNumber"].each |$ip| { + $infos = split($ip, "/") + $ipaddress = $infos[0] + if (length($infos) == 1 and $ipaddress =~ /:/) { + $mask = "128" + } elsif (length($infos) == 1) { + $mask = "32" + } else { + $mask = $infos[1] + } + + postgresql::server::pg_hba_rule { "allow TCP access for initial replication from $ipaddress/$mask": + type => 'hostssl', + database => 'replication', + user => $backup_host_cn, + address => "$ipaddress/$mask", + auth_method => 'pam', + order => "06-01", + target => "$pg_path/pg_hba.conf", + postgresql_version => "10", + } + } + } + if !empty($ldap_filter) and ($pg_infos["pgbouncer"]) { concat::fragment { "pgbouncer_$pg_backup_host": target => "/etc/pgbouncer/pgbouncer.ini", @@ -292,6 +335,11 @@ class role::backup::postgresql inherits role::backup { Concat["$pg_path/pg_hba.conf"], Concat["$pg_path/recovery.conf"], File["$pg_path/postgresql.conf"], + ], + subscribe => [ + Concat["$pg_path/pg_hba.conf"], + Concat["$pg_path/recovery.conf"], + File["$pg_path/postgresql.conf"], ] } } diff --git a/modules/role/templates/backup/pam_ldap_pgbouncer.conf.erb b/modules/role/templates/backup/pam_ldap_pgbouncer.conf.erb index 12fa9bb..384a418 100644 --- a/modules/role/templates/backup/pam_ldap_pgbouncer.conf.erb +++ b/modules/role/templates/backup/pam_ldap_pgbouncer.conf.erb @@ -3,5 +3,5 @@ host <%= @ldap_server %> base <%= @ldap_base %> binddn <%= @ldap_dn %> bindpw <%= @ldap_password %> -pam_login_attribute <%= @ldap_attribute %> +pam_login_attribute <%= @pgbouncer_ldap_attribute %> pam_filter <%= @ldap_filter %> diff --git a/python/reboot_ovh_cloud_instance.py b/python/reboot_ovh_cloud_instance.py index bd70e0b..1e88c89 100644 --- a/python/reboot_ovh_cloud_instance.py +++ b/python/reboot_ovh_cloud_instance.py @@ -25,6 +25,9 @@ if netboot_mode is not None: result = client.post("/cloud/project/{}/instance/{}/rescueMode".format(project, instance["id"]), imageId=instance["imageId"], rescue=(netboot_mode == "rescue")) print(result) +else: + result = client.post("/cloud/project/{}/instance/{}/reboot".format(project, instance["id"]), type="soft") + print(result) # reboot normal: #result = client.post("/cloud/project/{}/instance/{}/reboot".format(project, instance["id"]), type="soft") -- cgit v1.2.3 From cc278743bffea94197755c0d114389f5fd69596a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 27 Jun 2018 12:37:18 +0200 Subject: Listen to port depending on host variable --- modules/role/manifests/backup/postgresql.pp | 14 +++++++++++++- modules/role/templates/backup/postgresql.conf.erb | 7 ++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/modules/role/manifests/backup/postgresql.pp b/modules/role/manifests/backup/postgresql.pp index 9e1c9f7..aef177b 100644 --- a/modules/role/manifests/backup/postgresql.pp +++ b/modules/role/manifests/backup/postgresql.pp @@ -121,6 +121,12 @@ class role::backup::postgresql inherits role::backup { } else { $pg_backup_host = $host["vars"]["real_hostname"][0] } + if has_key($host["vars"], "postgresql_backup_port") { + $pg_listen_port = $host["vars"]["postgresql_backup_port"][0] + } else { + $pg_listen_port = undef + } + $pg_path = "$mountpoint/$pg_backup_host/postgresql" $pg_backup_path = "$mountpoint/$pg_backup_host/postgresql_backup" $pg_host = "$pg_backup_host" @@ -152,10 +158,16 @@ class role::backup::postgresql inherits role::backup { } if !empty($ldap_filter) and ($pg_infos["pgbouncer"]) { + if empty($pg_listen_port) { + $pg_listen_port_key = "" + } else { + $pg_listen_port_key = "port=$pg_listen_port" + } + concat::fragment { "pgbouncer_$pg_backup_host": target => "/etc/pgbouncer/pgbouncer.ini", order => 02, - content => "${pg_infos[pgbouncer_dbname]} = host=$mountpoint/$pg_backup_host/postgresql user=${pg_infos[dbuser]} dbname=${pg_infos[dbname]}", + content => "${pg_infos[pgbouncer_dbname]} = host=$mountpoint/$pg_backup_host/postgresql $pg_listen_port_key user=${pg_infos[dbuser]} dbname=${pg_infos[dbname]}", } postgresql::server::pg_hba_rule { "$pg_backup_host - local access as ${pg_infos[dbuser]} user": diff --git a/modules/role/templates/backup/postgresql.conf.erb b/modules/role/templates/backup/postgresql.conf.erb index 860089b..c4d223e 100644 --- a/modules/role/templates/backup/postgresql.conf.erb +++ b/modules/role/templates/backup/postgresql.conf.erb @@ -1,5 +1,10 @@ +<%- if !@pg_listen_port.nil? -%> +listen_addresses= '*' +port = <%= @pg_listen_port %> +ssl = on +<%- else -%> listen_addresses= '' +<%- end %> unix_socket_directories = '<%= @pg_path %>' data_directory = '<%= @pg_path %>' wal_level = logical - -- cgit v1.2.3 From 2f3d3a34ab0b3fd31bd84e4c935954740313dbed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 27 Jun 2018 13:10:32 +0200 Subject: Add ssl certificate for postgresql connection --- modules/role/manifests/backup/postgresql.pp | 30 ++++++++++++++++++++--- modules/role/templates/backup/postgresql.conf.erb | 2 ++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/role/manifests/backup/postgresql.pp b/modules/role/manifests/backup/postgresql.pp index aef177b..ee62a00 100644 --- a/modules/role/manifests/backup/postgresql.pp +++ b/modules/role/manifests/backup/postgresql.pp @@ -121,16 +121,38 @@ class role::backup::postgresql inherits role::backup { } else { $pg_backup_host = $host["vars"]["real_hostname"][0] } + + $pg_path = "$mountpoint/$pg_backup_host/postgresql" + $pg_backup_path = "$mountpoint/$pg_backup_host/postgresql_backup" + $pg_host = "$pg_backup_host" + $pg_port = $pg_infos["dbport"] + if has_key($host["vars"], "postgresql_backup_port") { $pg_listen_port = $host["vars"]["postgresql_backup_port"][0] + file { "$pg_path/certs": + ensure => directory, + mode => "0700", + owner => $pg_user, + group => $pg_group, + } -> + ssl::self_signed_certificate { $backup_host_cn: + common_name => $backup_host_cn, + country => "FR", + days => "3650", + organization => "Immae", + owner => $pg_user, + group => $pg_group, + directory => "$pg_path/certs", + before => File["$pg_path/postgresql.conf"], + } + $ssl_key = "$pg_path/certs/$backup_host_cn.key" + $ssl_cert = "$pg_path/certs/$backup_host_cn.crt" } else { $pg_listen_port = undef + $ssl_key = undef + $ssl_cert = undef } - $pg_path = "$mountpoint/$pg_backup_host/postgresql" - $pg_backup_path = "$mountpoint/$pg_backup_host/postgresql_backup" - $pg_host = "$pg_backup_host" - $pg_port = $pg_infos["dbport"] unless empty($host) { $host["ipHostNumber"].each |$ip| { diff --git a/modules/role/templates/backup/postgresql.conf.erb b/modules/role/templates/backup/postgresql.conf.erb index c4d223e..8741507 100644 --- a/modules/role/templates/backup/postgresql.conf.erb +++ b/modules/role/templates/backup/postgresql.conf.erb @@ -2,6 +2,8 @@ listen_addresses= '*' port = <%= @pg_listen_port %> ssl = on +ssl_key_file = '<%= @ssl_key %>' +ssl_cert_file = '<%= @ssl_cert %>' <%- else -%> listen_addresses= '' <%- end %> -- cgit v1.2.3 From 6840de8261fda8832964098a5f685eb79640f5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 27 Jun 2018 09:20:32 +0200 Subject: Add own ldap login as replication --- modules/profile/manifests/postgresql_master.pp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/profile/manifests/postgresql_master.pp b/modules/profile/manifests/postgresql_master.pp index 6f8854d..067345a 100644 --- a/modules/profile/manifests/postgresql_master.pp +++ b/modules/profile/manifests/postgresql_master.pp @@ -88,9 +88,15 @@ define profile::postgresql_master ( $ldap_server = lookup("base_installation::ldap_server") $ldap_base = lookup("base_installation::ldap_base") $ldap_dn = lookup("base_installation::ldap_dn") + $ldap_cn = lookup("base_installation::ldap_cn") $ldap_password = generate_password(24, $password_seed, "ldap") $ldap_attribute = "cn" + # This is to be replicated to the backup + postgresql::server::role { $ldap_cn: + replication => true, + } + file { "/etc/pam_ldap.d": ensure => directory, mode => "0755", -- cgit v1.2.3 From f1d583bfdaf881116e5f9ca9e050307e7acdc28e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 27 Jun 2018 13:30:37 +0200 Subject: Allow to pass arguments in puppet_reset_and_apply --- modules/base_installation/files/scripts/puppet_reset_and_apply | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/base_installation/files/scripts/puppet_reset_and_apply b/modules/base_installation/files/scripts/puppet_reset_and_apply index 0350e6e..7ec7053 100644 --- a/modules/base_installation/files/scripts/puppet_reset_and_apply +++ b/modules/base_installation/files/scripts/puppet_reset_and_apply @@ -6,9 +6,10 @@ git fetch origin branch="master" if [ -n "$1" ]; then branch="$1" + shift fi git reset --hard origin/$branch git submodule update --init -puppet_apply --test manifests/site.pp +puppet_apply "$@" --test manifests/site.pp -- cgit v1.2.3