From d87a489f9585d10f0a185beb59ae16a10f27a7bd Mon Sep 17 00:00:00 2001 From: =?utf8?q?Isma=C3=ABl=20Bouya?= Date: Fri, 16 Mar 2018 20:40:27 +0100 Subject: [PATCH] Add backup role --- .gitmodules | 3 + environments/global/common.yaml | 7 + environments/global/roles/backup.yaml | 11 ++ environments/hiera.yaml | 1 + modules/profile/manifests/known_hosts.pp | 11 ++ modules/role/manifests/backup.pp | 122 ++++++++++++++++++ .../backup/backup_dirname_head.sh.erb | 27 ++++ .../backup/backup_dirname_part.sh.erb | 26 ++++ .../backup/backup_dirname_tail.sh.erb | 4 + .../role/templates/backup/backup_head.sh.erb | 20 +++ .../templates/backup/backup_immae_eu.sh.erb | 79 ++++++++++++ .../role/templates/backup/backup_tail.sh.erb | 0 .../backup/ssh_host_changed.info.erb | 4 + .../templates/backup/ssh_key_changed.info.erb | 5 + modules/ssh_keygen | 1 + 15 files changed, 321 insertions(+) create mode 100644 environments/global/roles/backup.yaml create mode 100644 modules/profile/manifests/known_hosts.pp create mode 100644 modules/role/manifests/backup.pp create mode 100644 modules/role/templates/backup/backup_dirname_head.sh.erb create mode 100644 modules/role/templates/backup/backup_dirname_part.sh.erb create mode 100644 modules/role/templates/backup/backup_dirname_tail.sh.erb create mode 100644 modules/role/templates/backup/backup_head.sh.erb create mode 100644 modules/role/templates/backup/backup_immae_eu.sh.erb create mode 100644 modules/role/templates/backup/backup_tail.sh.erb create mode 100644 modules/role/templates/backup/ssh_host_changed.info.erb create mode 100644 modules/role/templates/backup/ssh_key_changed.info.erb create mode 160000 modules/ssh_keygen diff --git a/.gitmodules b/.gitmodules index fa5163a..735ca8c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,6 +46,9 @@ [submodule "python/ovh"] path = python/ovh url = git://git.immae.eu/github/ovh/python-ovh +[submodule "modules/ssh_keygen"] + path = modules/ssh_keygen + url = git://git.immae.eu/github/voxpupuli/puppet-ssh_keygen [submodule "modules/ssl"] path = modules/ssl url = git://git.immae.eu/github/fnerdwq/puppet-ssl diff --git a/environments/global/common.yaml b/environments/global/common.yaml index e7c70d4..4836f6e 100644 --- a/environments/global/common.yaml +++ b/environments/global/common.yaml @@ -8,6 +8,10 @@ lookup_options: merge: unique letsencrypt::hosts: merge: unique + role::backup::backups: + merge: unique + profile::known_hosts::hosts: + merge: unique classes: stdlib: ~ @@ -31,5 +35,8 @@ base_installation::system_users: [] # Fetched via ldap profile::xmr_stak::mining_pool: "" # Fetched via ldap profile::xmr_stak::wallet: "" # Fetched via ldap profile::mail::mailhub: "" # Fetched via ldap +role::backup::mailto: "" # Fetched via ldap +role::backup::backups: [] # Fetched via ldap +profile::known_hosts::hosts: [] # Fetched via ldap letsencrypt::email: ~ # Fetched via ldap letsencrypt::try_for_real_hostname: true diff --git a/environments/global/roles/backup.yaml b/environments/global/roles/backup.yaml new file mode 100644 index 0000000..52befe2 --- /dev/null +++ b/environments/global/roles/backup.yaml @@ -0,0 +1,11 @@ +--- +classes: + role::backup: ~ +role::backup::user: "backup" +role::backup::group: "backup" +base_installation::system_users: + - username: "%{lookup('role::backup::user')}" + userid: 976 + system: true + password: "!!" + diff --git a/environments/hiera.yaml b/environments/hiera.yaml index eda5eb3..61d40d8 100644 --- a/environments/hiera.yaml +++ b/environments/hiera.yaml @@ -8,6 +8,7 @@ defaults: hierarchy: - name: "Initialization variables" path: "/root/puppet_variables.json" + data_hash: json_data - name: "Puppet ldap variables" data_hash: ldap_data diff --git a/modules/profile/manifests/known_hosts.pp b/modules/profile/manifests/known_hosts.pp new file mode 100644 index 0000000..ed9ec8e --- /dev/null +++ b/modules/profile/manifests/known_hosts.pp @@ -0,0 +1,11 @@ +class profile::known_hosts ( + Optional[Array] $hosts = [] +) { + $hosts.each |$host| { + sshkey { $host["name"]: + ensure => "present", + key => $host["key"], + type => $host["type"], + } + } +} diff --git a/modules/role/manifests/backup.pp b/modules/role/manifests/backup.pp new file mode 100644 index 0000000..edfd5e0 --- /dev/null +++ b/modules/role/manifests/backup.pp @@ -0,0 +1,122 @@ +class role::backup ( + String $user, + String $group, + String $mailto, + Optional[Array] $backups = [], + Optional[String] $mountpoint = "/backup1", + Optional[String] $backup_script = "/usr/local/bin/backup.sh", +) { + include "base_installation" + + include "profile::mail" + include "profile::tools" + include "profile::xmr_stak" + include "profile::known_hosts" + + ssh_keygen { $user: + notify => Notify_refresh["notify-backup-sshkey-change"] + } + + $hosts = $backups.map |$backup| { $backup["host"] } + + notify_refresh { "notify-backup-sshkey-change": + message => template("role/backup/ssh_key_changed.info.erb"), + refreshonly => true + } + + $hosts.each |$host| { + notify_refresh { "notify-backup-sshhost-$host-changed": + message => template("role/backup/ssh_host_changed.info.erb"), + refreshonly => true, + subscribe => Sshkey[$host], + } + } + + concat { $backup_script: + ensure => "present", + ensure_newline => true, + mode => "0755", + } + + cron { "backup": + ensure => present, + command => $backup_script, + user => $user, + minute => 25, + hour => 3, + require => Concat[$backup_script], + } + + concat::fragment { "backup_head": + target => $backup_script, + content => template("role/backup/backup_head.sh.erb"), + order => "01-50", + } + + concat::fragment { "backup_tail": + target => $backup_script, + content => template("role/backup/backup_tail.sh.erb"), + order => "99-50", + } + + $backups.each |$infos| { + $dirname = $infos["name"] + $login = $infos["login"] + $host = $infos["host"] + $dest = "$login@$host" + $base = "$mountpoint/$dirname" + $nbr = $infos["nbr"] + $order_dirname = $infos["order"] + + file { $base: + ensure => "directory", + owner => $user, + group => $group, + require => Mount[$mountpoint], + } -> + file { "$base/older": + ensure => "directory", + owner => $user, + group => $group, + } -> + file { "$base/rsync_output": + ensure => "directory", + owner => $user, + group => $group, + } + + concat::fragment { "backup_${dirname}_head": + target => $backup_script, + content => template("role/backup/backup_dirname_head.sh.erb"), + order => "$order_dirname-01", + } + + concat::fragment { "backup_${dirname}_tail": + target => $backup_script, + content => template("role/backup/backup_dirname_tail.sh.erb"), + order => "$order_dirname-99", + } + + $infos["parts"].each |$part| { + $local_folder = $part["local_folder"] + $remote_folder = $part["remote_folder"] + $exclude_from = $part["exclude_from"] + $files_from = $part["files_from"] + $args = $part["args"] + $order_part = $part["order"] + + file { "$base/$local_folder": + ensure => "directory", + owner => $user, + group => $group, + require => File[$base], + } + + concat::fragment { "backup_${dirname}_${local_folder}": + target => $backup_script, + content => template("role/backup/backup_dirname_part.sh.erb"), + order => "$order_dirname-$order_part", + } + } + } +} diff --git a/modules/role/templates/backup/backup_dirname_head.sh.erb b/modules/role/templates/backup/backup_dirname_head.sh.erb new file mode 100644 index 0000000..e20cfd3 --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_head.sh.erb @@ -0,0 +1,27 @@ +##### <%= @dirname %> ##### +DEST="<%= @dest %>" +BASE="<%= @base %>" +OLD_BAK_BASE=$BASE/older/j +BAK_BASE=${OLD_BAK_BASE}0 +RSYNC_OUTPUT=$BASE/rsync_output +NBR=<%= @nbr %> + +if ! ssh \ + -o PreferredAuthentications=publickey \ + -o StrictHostKeyChecking=yes \ + -o ClearAllForwardings=yes \ + $DEST backup; then + echo "Fichier de verrouillage backup sur $DEST ou impossible de se connecter" >&2 + skip=$DEST +fi + +rm -rf ${OLD_BAK_BASE}${NBR} +for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ${OLD_BAK_BASE}$j ] && continue + mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1)) +done +mkdir $BAK_BASE +mv $RSYNC_OUTPUT $BAK_BASE +mkdir $RSYNC_OUTPUT + +if [ "$skip" != "$DEST" ]; then diff --git a/modules/role/templates/backup/backup_dirname_part.sh.erb b/modules/role/templates/backup/backup_dirname_part.sh.erb new file mode 100644 index 0000000..ec662c4 --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_part.sh.erb @@ -0,0 +1,26 @@ +### <%= @dirname %> <%= @local_folder %> ### +LOCAL="<%= @local_folder %>" +REMOTE="<%= @remote_folder %>" + +cd $BASE/$LOCAL +cat > $EXCL_FROM < +EOF +cat > $FILES_FROM < +EOF + +OUT=$RSYNC_OUTPUT/$LOCAL +rsync -XAavbrz --fake-super -e ssh --numeric-ids --delete \ + --backup-dir=$BAK_BASE/$LOCAL \ +<%- unless @args.empty? -%> + <%= @args %>\ +<% end -%> +<%- unless @exclude_from.empty? -%> + --exclude-from=$EXCL_FROM \ +<% end -%> +<%- unless @files_from.empty? -%> + --files-from=$FILES_FROM \ +<% end -%> + $DEST:$REMOTE . > $OUT || true +### End <%= @dirname %> <%= @local_folder %> ### diff --git a/modules/role/templates/backup/backup_dirname_tail.sh.erb b/modules/role/templates/backup/backup_dirname_tail.sh.erb new file mode 100644 index 0000000..6b16c9d --- /dev/null +++ b/modules/role/templates/backup/backup_dirname_tail.sh.erb @@ -0,0 +1,4 @@ + + ssh $DEST sh -c "date > .last_backup" +fi # [ "$skip" != "$DEST" ] +##### End <%= @dirname %> ##### diff --git a/modules/role/templates/backup/backup_head.sh.erb b/modules/role/templates/backup/backup_head.sh.erb new file mode 100644 index 0000000..be9f5bf --- /dev/null +++ b/modules/role/templates/backup/backup_head.sh.erb @@ -0,0 +1,20 @@ +#!/bin/bash +MAILTO="<%= @mailto %>" + +EXCL_FROM=`mktemp` +FILES_FROM=`mktemp` +TMP_STDERR=`mktemp` + +on_exit() { + if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" + fi + rm -f $TMP_STDERR $EXCL_FROM $FILES_FROM +} + +trap "on_exit" EXIT + +exec 2> "$TMP_STDERR" +exec < /dev/null + +set -e diff --git a/modules/role/templates/backup/backup_immae_eu.sh.erb b/modules/role/templates/backup/backup_immae_eu.sh.erb new file mode 100644 index 0000000..4fab30e --- /dev/null +++ b/modules/role/templates/backup/backup_immae_eu.sh.erb @@ -0,0 +1,79 @@ +#!/bin/bash +DEST="<%= @dest %>" +MAILTO="<%= @mailto %>" +BASE="<%= @base %>" +OLD_BAK_BASE=$BASE/older/j +BAK_BASE=${OLD_BAK_BASE}0 +RSYNC_OUTPUT=$BASE/rsync_output +NBR=7 + +TMP=`mktemp` +TMP_STDERR=`mktemp` + +trap "rm -f $TMP $TMP_STDERR" EXIT + +exec 2> "$TMP_STDERR" + +set -e +if ! `ssh -o ClearAllForwardings=yes $DEST backup`; then + echo "Fichier de verrouillage backup sur $DEST" + exit 1 +fi + +rm -rf ${OLD_BAK_BASE}${NBR} +for j in `seq -w $(($NBR-1)) -1 0`; do + [ ! -d ${OLD_BAK_BASE}$j ] && continue + mv ${OLD_BAK_BASE}$j ${OLD_BAK_BASE}$(($j+1)) +done +mkdir $BAK_BASE +mv $RSYNC_OUTPUT $BAK_BASE +mkdir $RSYNC_OUTPUT + +############## +NAME="home" +FOLDER="/home/immae" + +cd $BASE/$NAME +cat > $TMP < $OUT || true + +############## +NAME="system" +FOLDER="/" + +cd $BASE/$NAME +cat > $TMP < $OUT || true + +############## +ssh $DEST sh -c "date > .last_backup" + +if [ -s "$TMP_STDERR" ]; then + cat "$TMP_STDERR" | mail -Ssendwait -s "save_distant rsync error" "$MAILTO" +fi diff --git a/modules/role/templates/backup/backup_tail.sh.erb b/modules/role/templates/backup/backup_tail.sh.erb new file mode 100644 index 0000000..e69de29 diff --git a/modules/role/templates/backup/ssh_host_changed.info.erb b/modules/role/templates/backup/ssh_host_changed.info.erb new file mode 100644 index 0000000..ebf202e --- /dev/null +++ b/modules/role/templates/backup/ssh_host_changed.info.erb @@ -0,0 +1,4 @@ +Host <%= @host %> added, please send <%= @user %> key if necessary. +<%- if File.exist?("/home/#{@user}/.ssh/id_rsa.pub") %> + <%= File.read("/home/#{@user}/.ssh/id_rsa.pub") %> +<% end -%> diff --git a/modules/role/templates/backup/ssh_key_changed.info.erb b/modules/role/templates/backup/ssh_key_changed.info.erb new file mode 100644 index 0000000..43fd2ec --- /dev/null +++ b/modules/role/templates/backup/ssh_key_changed.info.erb @@ -0,0 +1,5 @@ +ssh key of <%= @user %> changed, +please update hosts: +<%- @hosts.each do |host| %> + - <%= host %> +<% end -%> diff --git a/modules/ssh_keygen b/modules/ssh_keygen new file mode 160000 index 0000000..ca53363 --- /dev/null +++ b/modules/ssh_keygen @@ -0,0 +1 @@ +Subproject commit ca53363249b58af96f90cb810c7c51dda8ba803b -- 2.41.0