modules/private/databases/mariadb_replication.nix

   1 { pkgs, config, lib, ... }:
   2 let
   3   cfg = config.myServices.databasesReplication.mariadb;
   4 in
   5 {
   6   options.myServices.databasesReplication.mariadb = {
   7     enable = lib.mkEnableOption "Enable mariadb replication";
   8     base = lib.mkOption {
   9       type = lib.types.path;
  10       description = ''
  11         Base path to put the replications
  12         '';
  13     };
  14     hosts = lib.mkOption {
  15       default = {};
  16       description = ''
  17         Hosts to backup
  18         '';
  19       type = lib.types.attrsOf (lib.types.submodule {
  20         options = {
  21           package = lib.mkOption {
  22             type = lib.types.package;
  23             default = pkgs.mariadb;
  24             description = ''
  25               Mariadb package for this host
  26             '';
  27           };
  28           serverId = lib.mkOption {
  29             type = lib.types.int;
  30             description = ''
  31               Server id to use for replication cluster (must be unique among the cluster!)
  32             '';
  33           };
  34           host = lib.mkOption {
  35             type = lib.types.str;
  36             description = ''
  37               Host to connect to
  38               '';
  39           };
  40           port = lib.mkOption {
  41             type = lib.types.str;
  42             description = ''
  43               Port to connect to
  44               '';
  45           };
  46           user = lib.mkOption {
  47             type = lib.types.str;
  48             description = ''
  49               User to connect as
  50               '';
  51           };
  52           password = lib.mkOption {
  53             type = lib.types.str;
  54             description = ''
  55               Password to use
  56               '';
  57           };
  58           dumpUser = lib.mkOption {
  59             type = lib.types.str;
  60             description = ''
  61               User who can do a dump
  62               '';
  63           };
  64           dumpPassword = lib.mkOption {
  65             type = lib.types.str;
  66             description = ''
  67               Password for the dump user
  68               '';
  69           };
  70         };
  71       });
  72     };
  73   };
  74
  75   config = lib.mkIf cfg.enable {
  76     users.users.mysql = {
  77       description = "MySQL server user";
  78       group = "mysql";
  79       uid = config.ids.uids.mysql;
  80       extraGroups = [ "keys" ];
  81     };
  82     users.groups.mysql.gid = config.ids.gids.mysql;
  83
  84     secrets.keys = lib.flatten (lib.mapAttrsToList (name: hcfg: [
  85       {
  86         dest = "mysql_replication/${name}/slave_init_commands";
  87         user = "mysql";
  88         group = "mysql";
  89         permissions = "0400";
  90         text = ''
  91           CHANGE MASTER TO master_host="${hcfg.host}", master_port=${hcfg.port}, master_user="${hcfg.user}", master_password="${hcfg.password}", master_ssl=1, master_use_gtid=slave_pos;
  92           START SLAVE;
  93           '';
  94       }
  95       {
  96         dest = "mysql_replication/${name}/mysqldump_remote";
  97         permissions = "0400";
  98         user = "root";
  99         group = "root";
 100         text = ''
 101           [mysqldump]
 102           user = ${hcfg.user}
 103           password = ${hcfg.password}
 104         '';
 105       }
 106       {
 107         dest = "mysql_replication/${name}/mysqldump";
 108         permissions = "0400";
 109         user = "root";
 110         group = "root";
 111         text = ''
 112           [mysqldump]
 113           user = ${hcfg.dumpUser}
 114           password = ${hcfg.dumpPassword}
 115         '';
 116       }
 117       {
 118         dest = "mysql_replication/${name}/client";
 119         permissions = "0400";
 120         user = "mysql";
 121         group = "mysql";
 122         text = ''
 123           [client]
 124           user = ${hcfg.dumpUser}
 125           password = ${hcfg.dumpPassword}
 126         '';
 127       }
 128     ]) cfg.hosts);
 129
 130     services.cron = {
 131       enable = true;
 132       systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg:
 133         let
 134           dataDir = "${cfg.base}/${name}/mysql";
 135           backupDir = "${cfg.base}/${name}/mysql_backup";
 136           backup_script = pkgs.writeScript "backup_mysql_${name}" ''
 137               #!${pkgs.stdenv.shell}
 138
 139               set -euo pipefail
 140
 141               ${hcfg.package}/bin/mysqldump \
 142                 --defaults-file=${config.secrets.location}/mysql_replication/${name}/mysqldump \
 143                 -S /run/mysqld_${name}/mysqld.sock \
 144                 --gtid \
 145                 --master-data \
 146                 --flush-privileges \
 147                 --all-databases > ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).sql
 148             '';
 149           u = pkgs.callPackage ./utils.nix {};
 150           cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps "sql" backupDir);
 151         in [
 152           "0 22,4,10,16 * * * root ${backup_script}"
 153           "0 3 * * * root ${cleanup_script}"
 154         ]) cfg.hosts);
 155     };
 156
 157     system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg:
 158       lib.attrsets.nameValuePair "mysql_replication_${name}" {
 159         deps = [ "users" "groups" ];
 160         text = ''
 161           install -m 0700 -o mysql -g mysql -d ${cfg.base}/${name}/mysql
 162           install -m 0700 -o mysql -g mysql -d ${cfg.base}/${name}/mysql_backup
 163           '';
 164       }) cfg.hosts;
 165
 166     environment.etc = lib.attrsets.mapAttrs' (name: hcfg:
 167       lib.attrsets.nameValuePair "mysql/${name}_my.cnf" {
 168         text = ''
 169           [mysqld]
 170           skip-networking
 171           socket = /run/mysqld_${name}/mysqld.sock
 172           datadir = ${cfg.base}/${name}/mysql/
 173           log-bin = mariadb-bin
 174           server-id = ${builtins.toString hcfg.serverId}
 175           '';
 176       }
 177     ) cfg.hosts;
 178
 179     systemd.services = lib.attrsets.mapAttrs' (name: hcfg:
 180       let
 181         dataDir = "${cfg.base}/${name}/mysql";
 182       in
 183       lib.attrsets.nameValuePair "mysql_backup_${name}" {
 184         description = "Mysql replication for ${name}";
 185         wantedBy = [ "multi-user.target" ];
 186         after = [ "network.target" ];
 187         restartTriggers = [ config.environment.etc."mysql/${name}_my.cnf".source ];
 188         unitConfig.RequiresMountsFor = dataDir;
 189
 190         preStart = ''
 191           if ! test -e ${dataDir}/mysql; then
 192             ${hcfg.package}/bin/mysqldump \
 193               --defaults-file=${config.secrets.location}/mysql_replication/${name}/mysqldump_remote \
 194               -h ${hcfg.host} \
 195               -P ${hcfg.port} \
 196               --ssl \
 197               --gtid \
 198               --flush-privileges \
 199               --master-data \
 200               --all-databases > ${dataDir}/initial.sql
 201
 202             ${hcfg.package}/bin/mysql_install_db \
 203               --defaults-file=/etc/mysql/${name}_my.cnf \
 204               --user=mysql \
 205               --datadir=${dataDir} \
 206               --basedir=${hcfg.package}
 207           fi
 208           '';
 209
 210         serviceConfig = {
 211           User = "mysql";
 212           Group = "mysql";
 213           RuntimeDirectory = "mysqld_${name}";
 214           RuntimeDirectoryMode = "0755";
 215           SupplementaryGroups = "keys";
 216           PermissionsStartOnly = true;
 217           Type = "notify";
 218
 219           ExecStart = "${hcfg.package}/bin/mysqld --defaults-file=/etc/mysql/${name}_my.cnf --user=mysql --datadir=${dataDir} --basedir=${hcfg.package}";
 220           ExecStartPost =
 221             let
 222               sql_before = pkgs.writeText "mysql-initial-before" ''
 223                 DROP DATABASE test;
 224                 '';
 225               setupScript = pkgs.writeScript "mysql-setup" ''
 226                 #!${pkgs.runtimeShell} -e
 227
 228                 if test -e ${dataDir}/initial.sql; then
 229                   cat \
 230                     ${sql_before} \
 231                     ${dataDir}/initial.sql \
 232                     ${config.secrets.location}/mysql_replication/${name}/slave_init_commands \
 233                     | ${hcfg.package}/bin/mysql \
 234                     --defaults-file=/etc/mysql/${name}_my.cnf \
 235                     -S /run/mysqld_${name}/mysqld.sock \
 236                     --user=root
 237                   rm -f ${dataDir}/initial.sql
 238                 fi
 239                 '';
 240             in
 241               "+${setupScript}";
 242           # initial dump can take a long time
 243           TimeoutStartSec="infinity";
 244           TimeoutStopSec = 120;
 245         };
 246     }) cfg.hosts;
 247   };
 248 }
 249