modules/private/databases/mariadb_replication.nix

   1 { pkgs, config, lib, ... }:
   2 let
   3   cfg = config.myServices.databasesReplication.mariadb;
   4 in
   5 {
   6   options.myServices.databasesReplication.mariadb = {
   7     enable = lib.mkEnableOption "Enable mariadb replication";
   8     base = lib.mkOption {
   9       type = lib.types.path;
  10       description = ''
  11         Base path to put the replications
  12         '';
  13     };
  14     hosts = lib.mkOption {
  15       default = {};
  16       description = ''
  17         Hosts to backup
  18         '';
  19       type = lib.types.attrsOf (lib.types.submodule {
  20         options = {
  21           package = lib.mkOption {
  22             type = lib.types.package;
  23             default = pkgs.mariadb;
  24             description = ''
  25               Mariadb package for this host
  26             '';
  27           };
  28           serverId = lib.mkOption {
  29             type = lib.types.int;
  30             description = ''
  31               Server id to use for replication cluster (must be unique among the cluster!)
  32             '';
  33           };
  34           host = lib.mkOption {
  35             type = lib.types.str;
  36             description = ''
  37               Host to connect to
  38               '';
  39           };
  40           port = lib.mkOption {
  41             type = lib.types.str;
  42             description = ''
  43               Port to connect to
  44               '';
  45           };
  46           user = lib.mkOption {
  47             type = lib.types.str;
  48             description = ''
  49               User to connect as
  50               '';
  51           };
  52           password = lib.mkOption {
  53             type = lib.types.str;
  54             description = ''
  55               Password to use
  56               '';
  57           };
  58           dumpUser = lib.mkOption {
  59             type = lib.types.str;
  60             description = ''
  61               User who can do a dump
  62               '';
  63           };
  64           dumpPassword = lib.mkOption {
  65             type = lib.types.str;
  66             description = ''
  67               Password for the dump user
  68               '';
  69           };
  70         };
  71       });
  72     };
  73   };
  74
  75   config = lib.mkIf cfg.enable {
  76     users.users.mysql = {
  77       description = "MySQL server user";
  78       group = "mysql";
  79       uid = config.ids.uids.mysql;
  80       extraGroups = [ "keys" ];
  81     };
  82     users.groups.mysql.gid = config.ids.gids.mysql;
  83
  84     secrets.keys = lib.flatten (lib.mapAttrsToList (name: hcfg: [
  85       {
  86         dest = "mysql_replication/${name}/slave_init_commands";
  87         user = "mysql";
  88         group = "mysql";
  89         permissions = "0400";
  90         text = ''
  91           CHANGE MASTER TO master_host="${hcfg.host}", master_port=${hcfg.port}, master_user="${hcfg.user}", master_password="${hcfg.password}", master_ssl=1, master_use_gtid=slave_pos;
  92           START SLAVE;
  93           '';
  94       }
  95       {
  96         dest = "mysql_replication/${name}/mysqldump_remote";
  97         permissions = "0400";
  98         user = "root";
  99         group = "root";
 100         text = ''
 101           [mysqldump]
 102           user = ${hcfg.user}
 103           password = ${hcfg.password}
 104         '';
 105       }
 106       {
 107         dest = "mysql_replication/${name}/mysqldump";
 108         permissions = "0400";
 109         user = "root";
 110         group = "root";
 111         text = ''
 112           [mysqldump]
 113           user = ${hcfg.dumpUser}
 114           password = ${hcfg.dumpPassword}
 115         '';
 116       }
 117     ]) cfg.hosts);
 118
 119     services.cron = {
 120       enable = true;
 121       systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg:
 122         let
 123           dataDir = "${cfg.base}/${name}/mysql";
 124           backupDir = "${cfg.base}/${name}/mysql_backup";
 125           backup_script = pkgs.writeScript "backup_mysql_${name}" ''
 126               #!${pkgs.stdenv.shell}
 127
 128               set -euo pipefail
 129
 130               ${hcfg.package}/bin/mysqldump \
 131                 --defaults-file=${config.secrets.location}/mysql_replication/${name}/mysqldump \
 132                 -S /run/mysqld_${name}/mysqld.sock \
 133                 --gtid \
 134                 --master-data \
 135                 --flush-privileges \
 136                 --all-databases > ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).sql
 137             '';
 138           u = pkgs.callPackage ./utils.nix {};
 139           cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps "sql" backupDir);
 140         in [
 141           "0 22,4,10,16 * * * root ${backup_script}"
 142           "0 3 * * * root ${cleanup_script}"
 143         ]) cfg.hosts);
 144     };
 145
 146     system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg:
 147       lib.attrsets.nameValuePair "mysql_replication_${name}" {
 148         deps = [ "users" "groups" ];
 149         text = ''
 150           install -m 0700 -o mysql -g mysql -d ${cfg.base}/${name}/mysql
 151           install -m 0700 -o mysql -g mysql -d ${cfg.base}/${name}/mysql_backup
 152           '';
 153       }) cfg.hosts;
 154
 155     environment.etc = lib.attrsets.mapAttrs' (name: hcfg:
 156       lib.attrsets.nameValuePair "mysql/${name}_my.cnf" {
 157         text = ''
 158           [mysqld]
 159           socket = /run/mysqld_${name}/mysqld.sock
 160           datadir = ${cfg.base}/${name}/mysql/
 161           log-bin = mariadb-bin
 162           server-id = ${builtins.toString hcfg.serverId}
 163           '';
 164       }
 165     ) cfg.hosts;
 166
 167     systemd.services = lib.attrsets.mapAttrs' (name: hcfg:
 168       let
 169         dataDir = "${cfg.base}/${name}/mysql";
 170       in
 171       lib.attrsets.nameValuePair "mysql_backup_${name}" {
 172         description = "Mysql replication for ${name}";
 173         wantedBy = [ "multi-user.target" ];
 174         after = [ "network.target" ];
 175         restartTriggers = [ config.environment.etc."mysql/${name}_my.cnf".source ];
 176         unitConfig.RequiresMountsFor = dataDir;
 177
 178         preStart = ''
 179           if ! test -e ${dataDir}/mysql; then
 180             ${hcfg.package}/bin/mysqldump \
 181               --defaults-file=${config.secrets.location}/mysql_replication/${name}/mysqldump_remote \
 182               -h ${hcfg.host} \
 183               -P ${hcfg.port} \
 184               --ssl \
 185               --gtid \
 186               --flush-privileges \
 187               --master-data \
 188               --all-databases > ${dataDir}/initial.sql
 189
 190             ${hcfg.package}/bin/mysql_install_db \
 191               --defaults-file=/etc/mysql/${name}_my.cnf \
 192               --user=mysql \
 193               --datadir=${dataDir} \
 194               --basedir=${hcfg.package}
 195           fi
 196           '';
 197
 198         serviceConfig = {
 199           User = "mysql";
 200           Group = "mysql";
 201           RuntimeDirectory = "mysqld_${name}";
 202           RuntimeDirectoryMode = "0755";
 203           SupplementaryGroups = "keys";
 204           PermissionsStartOnly = true;
 205           Type = "notify";
 206
 207           ExecStart = "${hcfg.package}/bin/mysqld --defaults-file=/etc/mysql/${name}_my.cnf --user=mysql --datadir=${dataDir} --basedir=${hcfg.package}";
 208           ExecStartPost =
 209             let
 210               sql_before = pkgs.writeText "mysql-initial-before" ''
 211                 DROP DATABASE test;
 212                 '';
 213               setupScript = pkgs.writeScript "mysql-setup" ''
 214                 #!${pkgs.runtimeShell} -e
 215
 216                 if test -e ${dataDir}/initial.sql; then
 217                   cat \
 218                     ${sql_before} \
 219                     ${dataDir}/initial.sql \
 220                     ${config.secrets.location}/mysql_replication/${name}/slave_init_commands \
 221                     | ${hcfg.package}/bin/mysql \
 222                     --defaults-file=/etc/mysql/${name}_my.cnf \
 223                     -S /run/mysqld_${name}/mysqld.sock \
 224                     --user=root
 225                   rm -f ${dataDir}/initial.sql
 226                 fi
 227                 '';
 228             in
 229               "+${setupScript}";
 230           # initial dump can take a long time
 231           TimeoutStartSec="infinity";
 232           TimeoutStopSec = 120;
 233         };
 234     }) cfg.hosts;
 235   };
 236 }
 237