Merge branch 'backup' into dev

22 files changed, 383 insertions, 5 deletions

diff --git a/.gitmodules b/.gitmodules
index fa5163a..735ca8c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -46,6 +46,9 @@
 [submodule "python/ovh"]
         path = python/ovh
         url = git://git.immae.eu/github/ovh/python-ovh
+[submodule "modules/ssh_keygen"]
+        path = modules/ssh_keygen
+        url = git://git.immae.eu/github/voxpupuli/puppet-ssh_keygen
 [submodule "modules/ssl"]
         path = modules/ssl
         url = git://git.immae.eu/github/fnerdwq/puppet-ssl
diff --git a/environments/global/common.yaml b/environments/global/common.yaml
index 5911194..4836f6e 100644
--- a/environments/global/common.yaml
+++ b/environments/global/common.yaml
@@ -8,6 +8,10 @@ lookup_options:
     merge: unique
   letsencrypt::hosts:
     merge: unique
+  role::backup::backups:
+    merge: unique
+  profile::known_hosts::hosts:
+    merge: unique
 
 classes:
   stdlib: ~
@@ -30,5 +34,9 @@ base_installation::system_timezone: "Europe/Paris"
 base_installation::system_users: [] # Fetched via ldap
 profile::xmr_stak::mining_pool: "" # Fetched via ldap
 profile::xmr_stak::wallet: "" # Fetched via ldap
+profile::mail::mailhub: "" # Fetched via ldap
+role::backup::mailto: "" # Fetched via ldap
+role::backup::backups: [] # Fetched via ldap
+profile::known_hosts::hosts: [] # Fetched via ldap
 letsencrypt::email: ~ # Fetched via ldap
 letsencrypt::try_for_real_hostname: true
diff --git a/environments/global/roles/backup.yaml b/environments/global/roles/backup.yaml
new file mode 100644
index 0000000..52befe2
--- /dev/null
+++ b/environments/global/roles/backup.yaml
@@ -0,0 +1,11 @@
+---
+classes:
+  role::backup: ~
+role::backup::user: "backup"
+role::backup::group: "backup"
+base_installation::system_users:
+  - username: "%{lookup('role::backup::user')}"
+    userid: 976
+    system: true
+    password: "!!"
+
diff --git a/environments/hiera.yaml b/environments/hiera.yaml
index eda5eb3..61d40d8 100644
--- a/environments/hiera.yaml
+++ b/environments/hiera.yaml
@@ -8,6 +8,7 @@ defaults:
 hierarchy:
   - name: "Initialization variables"
     path: "/root/puppet_variables.json"
+    data_hash: json_data
 
   - name: "Puppet ldap variables"
     data_hash: ldap_data
diff --git a/modules/base_installation/files/cronie/puppet-post-merge b/modules/base_installation/files/cronie/puppet-post-merge
index ac5e3ff..f5c21a7 100644
--- a/modules/base_installation/files/cronie/puppet-post-merge
+++ b/modules/base_installation/files/cronie/puppet-post-merge
@@ -1,7 +1,7 @@
 #!/bin/bash
-## Run Puppet locally using puppet apply
+## Run Puppet locally using puppet_apply
 git submodule update --init
-/usr/bin/puppet apply `pwd`/manifests/site.pp
+/usr/local/sbin/puppet_apply `pwd`/manifests/site.pp
 
 ## Log status of the Puppet run
 if [ $? -eq 0 ]
diff --git a/modules/base_installation/files/scripts/puppet_apply b/modules/base_installation/files/scripts/puppet_apply
new file mode 100644
index 0000000..69673cc
--- /dev/null
+++ b/modules/base_installation/files/scripts/puppet_apply
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+lockfile=/var/run/puppet-apply.lock
+path=`dirname $0`
+path=`cd $path/..; pwd`
+
+if [ $(id -u) -gt 0 ]; then
+  echo "You must be root to run this script." >&2
+  exit 2
+fi
+
+if (set -o noclobber; echo "$$" > "$lockfile") 2> /dev/null; then
+  trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT
+
+  puppet apply "$@"
+
+  rm -f "$lockfile"
+  trap - INT TERM EXIT
+else
+  echo "Failed to acquire lockfile: $lockfile." >&2
+  echo "Held by $(cat $lockfile 2>/dev/null)" >&2
+  exit 1
+fi
diff --git a/modules/base_installation/files/scripts/puppet_reset_and_apply b/modules/base_installation/files/scripts/puppet_reset_and_apply
index 6743044..0350e6e 100644
--- a/modules/base_installation/files/scripts/puppet_reset_and_apply
+++ b/modules/base_installation/files/scripts/puppet_reset_and_apply
@@ -11,4 +11,4 @@ fi
 git reset --hard origin/$branch
 
 git submodule update --init
-puppet apply --test manifests/site.pp
+puppet_apply --test manifests/site.pp
diff --git a/modules/base_installation/manifests/cronie.pp b/modules/base_installation/manifests/cronie.pp
index e8d5dfd..e8f3e20 100644
--- a/modules/base_installation/manifests/cronie.pp
+++ b/modules/base_installation/manifests/cronie.pp
@@ -19,13 +19,13 @@ class base_installation::cronie inherits base_installation {
     }
     cron { 'puppet-apply':
       ensure  => present,
-      command => "cd $base_installation::puppet_code_path ; puppet apply $base_installation::puppet_code_path/manifests/site.pp",
+      command => "cd $base_installation::puppet_code_path ; puppet_apply $base_installation::puppet_code_path/manifests/site.pp",
       user    => root,
       minute  => '*/20'
     }
     cron { 'puppet-apply-reboot':
       ensure  => present,
-      command => "cd $base_installation::puppet_code_path ; puppet apply $base_installation::puppet_code_path/manifests/site.pp",
+      command => "cd $base_installation::puppet_code_path ; puppet_apply $base_installation::puppet_code_path/manifests/site.pp",
       user    => root,
       special => "reboot"
     }
diff --git a/modules/base_installation/manifests/puppet.pp b/modules/base_installation/manifests/puppet.pp
index 6f7732d..0cb43bc 100644
--- a/modules/base_installation/manifests/puppet.pp
+++ b/modules/base_installation/manifests/puppet.pp
@@ -39,6 +39,12 @@ class base_installation::puppet (
     source => "puppet:///modules/base_installation/scripts/report_print.rb"
   }
 
+  file { '/usr/local/sbin/puppet_apply':
+    mode   => "0755",
+    ensure => present,
+    source => "puppet:///modules/base_installation/scripts/puppet_apply",
+  }
+
   unless empty(find_file($password_seed)) {
     $ldap_password = generate_password(24, $password_seed, "ldap")
     $ssha_ldap_seed = generate_password(5, $password_seed, "ldap_seed")
diff --git a/modules/profile/manifests/known_hosts.pp b/modules/profile/manifests/known_hosts.pp
new file mode 100644
index 0000000..ed9ec8e
--- /dev/null
+++ b/modules/profile/manifests/known_hosts.pp
@@ -0,0 +1,11 @@
+class profile::known_hosts (
+  Optional[Array]  $hosts = []
+) {
+  $hosts.each |$host| {
+    sshkey { $host["name"]:
+      ensure => "present",
+      key    => $host["key"],
+      type   => $host["type"],
+    }
+  }
+}
diff --git a/modules/profile/manifests/mail.pp b/modules/profile/manifests/mail.pp
new file mode 100644
index 0000000..cc47b77
--- /dev/null
+++ b/modules/profile/manifests/mail.pp
@@ -0,0 +1,14 @@
+class profile::mail (
+  String            $mailhub,
+  Optional[Integer] $mailhub_port = 25,
+) {
+  ensure_packages(["s-nail", "ssmtp"])
+
+  $hostname = lookup("base_installation::real_hostname")
+
+  file { "/etc/ssmtp/ssmtp.conf":
+    ensure  => "present",
+    content =>  template("profile/mail/ssmtp.conf.erb"),
+  }
+}
+
diff --git a/modules/profile/templates/mail/ssmtp.conf.erb b/modules/profile/templates/mail/ssmtp.conf.erb
new file mode 100644
index 0000000..e7a0410
--- /dev/null
+++ b/modules/profile/templates/mail/ssmtp.conf.erb
@@ -0,0 +1,14 @@
+#
+# /etc/ssmtp.conf -- a config file for sSMTP sendmail.
+#
+# The person who gets all mail for userids < 1000
+# Make this empty to disable rewriting.
+root=postmaster
+# The place where the mail goes. The actual machine name is required
+# no MX records are consulted. Commonly mailhosts are named mail.domain.com
+# The example will fit if you are in domain.com and you mailhub is so named.
+mailhub=<%= @mailhub %>:<%= @mailhub_port %>
+# Where will the mail seem to come from?
+#rewriteDomain=y
+# The full hostname
+hostname=<%= @hostname %>
diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp
new file mode 100644
index 0000000..edfd5e0
--- /dev/null
+++ b/modules/role/manifests/backup.pp
@@ -0,0 +1,122 @@
+class role::backup (
+  String            $user,
+  String            $group,
+  String            $mailto,
+  Optional[Array]   $backups = [],
+  Optional[String]  $mountpoint = "/backup1",
+  Optional[String]  $backup_script = "/usr/local/bin/backup.sh",
+) {
+  include "base_installation"
+
+  include "profile::mail"
+  include "profile::tools"
+  include "profile::xmr_stak"
+  include "profile::known_hosts"
+
+  ssh_keygen { $user:
+    notify => Notify_refresh["notify-backup-sshkey-change"]
+  }
+
+  $hosts = $backups.map |$backup| { $backup["host"] }
+
+  notify_refresh { "notify-backup-sshkey-change":
+    message     => template("role/backup/ssh_key_changed.info.erb"),
+    refreshonly => true
+  }
+
+  $hosts.each |$host| {
+    notify_refresh { "notify-backup-sshhost-$host-changed":
+      message     => template("role/backup/ssh_host_changed.info.erb"),
+      refreshonly => true,
+      subscribe   => Sshkey[$host],
+    }
+  }
+
+  concat { $backup_script:
+    ensure         => "present",
+    ensure_newline => true,
+    mode           => "0755",
+  }
+
+  cron { "backup":
+    ensure  => present,
+    command => $backup_script,
+    user    => $user,
+    minute  => 25,
+    hour    => 3,
+    require => Concat[$backup_script],
+  }
+
+  concat::fragment { "backup_head":
+    target  => $backup_script,
+    content => template("role/backup/backup_head.sh.erb"),
+    order   => "01-50",
+  }
+
+  concat::fragment { "backup_tail":
+    target  => $backup_script,
+    content => template("role/backup/backup_tail.sh.erb"),
+    order   => "99-50",
+  }
+
+  $backups.each |$infos| {
+    $dirname = $infos["name"]
+    $login = $infos["login"]
+    $host = $infos["host"]
+    $dest = "$login@$host"
+    $base = "$mountpoint/$dirname"
+    $nbr  = $infos["nbr"]
+    $order_dirname = $infos["order"]
+
+    file { $base:
+      ensure  => "directory",
+      owner   => $user,
+      group   => $group,
+      require => Mount[$mountpoint],
+    } ->
+    file { "$base/older":
+      ensure  => "directory",
+      owner   => $user,
+      group   => $group,
+    } ->
+    file { "$base/rsync_output":
+      ensure  => "directory",
+      owner   => $user,
+      group   => $group,
+    }
+
+    concat::fragment { "backup_${dirname}_head":
+      target  => $backup_script,
+      content => template("role/backup/backup_dirname_head.sh.erb"),
+      order   => "$order_dirname-01",
+    }
+
+    concat::fragment { "backup_${dirname}_tail":
+      target  => $backup_script,
+      content => template("role/backup/backup_dirname_tail.sh.erb"),
+      order   => "$order_dirname-99",
+    }
+
+    $infos["parts"].each |$part| {
+      $local_folder = $part["local_folder"]
+      $remote_folder = $part["remote_folder"]
+      $exclude_from = $part["exclude_from"]
+      $files_from = $part["files_from"]
+      $args = $part["args"]
+      $order_part = $part["order"]
+
+      file { "$base/$local_folder":
+        ensure  => "directory",
+        owner   => $user,
+        group   => $group,
+        require => File[$base],
+      }
+
+      concat::fragment { "backup_${dirname}_${local_folder}":
+        target  => $backup_script,
+        content => template("role/backup/backup_dirname_part.sh.erb"),
+        order   => "$order_dirname-$order_part",
+      }
+    }
+  }
+}
diff --git a/modules/role/templates/backup/backup_dirname_head.sh.erb b/modules/role/templates/backup/backup_dirname_head.sh.erb
new file mode 100644
index 0000000..e20cfd3
--- /dev/null
+++ b/modules/role/templates/backup/backup_dirname_head.sh.erb
@@ -0,0 +1,27 @@
+##### <%= @dirname %> #####
+DEST="<%= @dest %>"
+BASE="<%= @base %>"
+OLD_BAK_BASE=$BASE/older/j
+BAK_BASE=${OLD_BAK_BASE}0
+RSYNC_OUTPUT=$BASE/rsync_output
+NBR=<%= @nbr %>
+
+if ! ssh \
+    -o PreferredAuthentications=publickey \
+    -o StrictHostKeyChecking=yes \
+    -o ClearAllForwardings=yes \
+    $DEST backup; then
+  echo "Fichier de verrouillage backup sur $DEST ou impossible de se connecter" >&2
+  skip=$DEST
+fi
+
+rm -rf ${OLD_BAK_BASE}${NBR}
+for j in `seq -w $(($NBR-1)) -1 0`; do
+  [ ! -d ${OLD_BAK_BASE}$j ] && continue
+  mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1))
+done
+mkdir $BAK_BASE
+mv $RSYNC_OUTPUT $BAK_BASE
+mkdir $RSYNC_OUTPUT
+
+if [ "$skip" != "$DEST" ]; then
diff --git a/modules/role/templates/backup/backup_dirname_part.sh.erb b/modules/role/templates/backup/backup_dirname_part.sh.erb
new file mode 100644
index 0000000..ec662c4
--- /dev/null
+++ b/modules/role/templates/backup/backup_dirname_part.sh.erb
@@ -0,0 +1,26 @@
+### <%= @dirname %> <%= @local_folder %> ###
+LOCAL="<%= @local_folder %>"
+REMOTE="<%= @remote_folder %>"
+
+cd $BASE/$LOCAL
+cat > $EXCL_FROM <<EOF
+<%= @exclude_from.join("\n") %>
+EOF
+cat > $FILES_FROM <<EOF
+<%= @files_from.join("\n") %>
+EOF
+
+OUT=$RSYNC_OUTPUT/$LOCAL
+rsync -XAavbrz --fake-super -e ssh --numeric-ids --delete \
+  --backup-dir=$BAK_BASE/$LOCAL \
+<%- unless @args.empty? -%>
+  <%= @args %>\
+<% end -%>
+<%- unless @exclude_from.empty? -%>
+  --exclude-from=$EXCL_FROM \
+<% end -%>
+<%- unless @files_from.empty? -%>
+  --files-from=$FILES_FROM \
+<% end -%>
+  $DEST:$REMOTE . > $OUT || true
+### End <%= @dirname %> <%= @local_folder %> ###
diff --git a/modules/role/templates/backup/backup_dirname_tail.sh.erb b/modules/role/templates/backup/backup_dirname_tail.sh.erb
new file mode 100644
index 0000000..6b16c9d
--- /dev/null
+++ b/modules/role/templates/backup/backup_dirname_tail.sh.erb
@@ -0,0 +1,4 @@
+
+  ssh $DEST sh -c "date > .last_backup"
+fi # [ "$skip" != "$DEST" ]
+##### End <%= @dirname %> #####
diff --git a/modules/role/templates/backup/backup_head.sh.erb b/modules/role/templates/backup/backup_head.sh.erb
new file mode 100644
index 0000000..be9f5bf
--- /dev/null
+++ b/modules/role/templates/backup/backup_head.sh.erb
@@ -0,0 +1,20 @@
+#!/bin/bash
+MAILTO="<%= @mailto %>"
+
+EXCL_FROM=`mktemp`
+FILES_FROM=`mktemp`
+TMP_STDERR=`mktemp`
+
+on_exit() {
+  if [ -s "$TMP_STDERR" ]; then
+    cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" 
+  fi
+  rm -f $TMP_STDERR $EXCL_FROM $FILES_FROM
+}
+
+trap "on_exit" EXIT
+
+exec 2> "$TMP_STDERR"
+exec < /dev/null
+
+set -e
diff --git a/modules/role/templates/backup/backup_immae_eu.sh.erb b/modules/role/templates/backup/backup_immae_eu.sh.erb
new file mode 100644
index 0000000..4fab30e
--- /dev/null
+++ b/modules/role/templates/backup/backup_immae_eu.sh.erb
@@ -0,0 +1,79 @@
+#!/bin/bash
+DEST="<%= @dest %>"
+MAILTO="<%= @mailto %>"
+BASE="<%= @base %>"
+OLD_BAK_BASE=$BASE/older/j
+BAK_BASE=${OLD_BAK_BASE}0
+RSYNC_OUTPUT=$BASE/rsync_output
+NBR=7
+
+TMP=`mktemp`
+TMP_STDERR=`mktemp`
+
+trap "rm -f $TMP $TMP_STDERR" EXIT
+
+exec 2> "$TMP_STDERR"
+
+set -e
+if ! `ssh -o ClearAllForwardings=yes $DEST backup`; then
+  echo "Fichier de verrouillage backup sur $DEST"
+  exit 1
+fi
+
+rm -rf ${OLD_BAK_BASE}${NBR}
+for j in `seq -w $(($NBR-1)) -1 0`; do
+  [ ! -d ${OLD_BAK_BASE}$j ] && continue
+  mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1))
+done
+mkdir $BAK_BASE
+mv $RSYNC_OUTPUT $BAK_BASE
+mkdir $RSYNC_OUTPUT
+
+##############
+NAME="home"
+FOLDER="/home/immae"
+
+cd $BASE/$NAME
+cat > $TMP <<EOF
+/.no_backup/
+/hosts/florian/nobackup/
+/hosts/connexionswing.com/
+/hosts/connexionswing.immae.eu/
+/hosts/ludivine.immae.eu/
+/hosts/ludivinecassal.com/
+/hosts/piedsjaloux.fr/
+/hosts/piedsjaloux.immae.eu/
+/hosts/spip/sites/*/
+/hosts/spip/spip*
+EOF
+OUT=$RSYNC_OUTPUT/$NAME
+rsync -XAavbrz --fake-super -e ssh --numeric-ids --delete \
+  --backup-dir=$BAK_BASE/$NAME --exclude-from=$TMP \
+  $DEST:$FOLDER . > $OUT || true
+
+##############
+NAME="system"
+FOLDER="/"
+
+cd $BASE/$NAME
+cat > $TMP <<EOF
+/etc/
+/srv/
+/var/lib/
+/var/spool/
+/var/named/
+/usr/local/
+EOF
+OUT=$RSYNC_OUTPUT/$NAME
+rsync -XAavbrz -R --fake-super -e ssh --numeric-ids --delete \
+  --rsync-path='sudo rsync' \
+  --backup-dir=$BAK_BASE/$NAME \
+  --files-from=$TMP \
+  $DEST:$FOLDER . > $OUT || true
+
+##############
+ssh $DEST sh -c "date > .last_backup"
+
+if [ -s "$TMP_STDERR" ]; then
+  cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" 
+fi
diff --git a/modules/role/templates/backup/backup_tail.sh.erb b/modules/role/templates/backup/backup_tail.sh.erb
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/modules/role/templates/backup/backup_tail.sh.erb
diff --git a/modules/role/templates/backup/ssh_host_changed.info.erb b/modules/role/templates/backup/ssh_host_changed.info.erb
new file mode 100644
index 0000000..ebf202e
--- /dev/null
+++ b/modules/role/templates/backup/ssh_host_changed.info.erb
@@ -0,0 +1,4 @@
+Host <%= @host %> added, please send <%= @user %> key if necessary.
+<%- if File.exist?("/home/#{@user}/.ssh/id_rsa.pub") %>
+  <%= File.read("/home/#{@user}/.ssh/id_rsa.pub") %>
+<% end -%>
diff --git a/modules/role/templates/backup/ssh_key_changed.info.erb b/modules/role/templates/backup/ssh_key_changed.info.erb
new file mode 100644
index 0000000..43fd2ec
--- /dev/null
+++ b/modules/role/templates/backup/ssh_key_changed.info.erb
@@ -0,0 +1,5 @@
+ssh key of <%= @user %> changed,
+please update hosts:
+<%- @hosts.each do |host| %>
+  - <%= host %>
+<% end -%>
diff --git a/modules/ssh_keygen b/modules/ssh_keygen
new file mode 160000
+Subproject ca53363249b58af96f90cb810c7c51dda8ba803