From: Ismaël Bouya <ismael.bouya@normalesup.org>
Date: Sat, 7 Dec 2019 07:05:53 +0000 (+0100)
Subject: Add replication for redis
X-Git-Url: https://git.immae.eu/?p=perso%2FImmae%2FConfig%2FNix.git;a=commitdiff_plain;h=dded66995529a0419cc56778f4ebb4247c2ab765

Add replication for redis
---

diff --git a/modules/private/databases/mariadb_replication.nix b/modules/private/databases/mariadb_replication.nix
index 23648bb..5f97e84 100644
--- a/modules/private/databases/mariadb_replication.nix
+++ b/modules/private/databases/mariadb_replication.nix
@@ -136,7 +136,7 @@ in
                 --all-databases > ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).sql
             '';
           u = pkgs.callPackage ./utils.nix {};
-          cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps backupDir);
+          cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps "sql" backupDir);
         in [
           "0 22,4,10,16 * * * root ${backup_script}"
           "0 3 * * * root ${cleanup_script}"
diff --git a/modules/private/databases/postgresql_replication.nix b/modules/private/databases/postgresql_replication.nix
index cc32c2b..19ec168 100644
--- a/modules/private/databases/postgresql_replication.nix
+++ b/modules/private/databases/postgresql_replication.nix
@@ -116,7 +116,7 @@ in
               ${hcfg.package}/bin/pg_dumpall -h ${dataDir} -f ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).sql
             '';
           u = pkgs.callPackage ./utils.nix {};
-          cleanup_script = pkgs.writeScript "cleanup_postgresql_${name}" (u.keepLastNDumps backupDir 12);
+          cleanup_script = pkgs.writeScript "cleanup_postgresql_${name}" (u.keepLastNDumps "sql" backupDir 12);
         in [
           "0 22,4,10,16 * * * postgres ${backup_script}"
           "0 3 * * * postgres ${cleanup_script}"
diff --git a/modules/private/databases/redis.nix b/modules/private/databases/redis.nix
index c23ffec..693f402 100644
--- a/modules/private/databases/redis.nix
+++ b/modules/private/databases/redis.nix
@@ -1,4 +1,4 @@
-{ lib, config, ... }:
+{ lib, config, pkgs, myconfig, ... }:
 let
     cfg = config.myServices.databases.redis;
 in {
@@ -52,6 +52,88 @@ in {
         '';
     };
     systemd.services.redis.serviceConfig.RuntimeDirectory = cfg.systemdRuntimeDirectory;
+
+    services.spiped = {
+      enable = true;
+      config.redis = {
+        decrypt = true;
+        source = "0.0.0.0:16379";
+        target = "/run/redis/redis.sock";
+        keyfile = "${config.secrets.location}/redis/spiped_keyfile";
+      };
+    };
+    systemd.services.spiped_redis = {
+      description = "Secure pipe 'redis'";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        Restart   = "always";
+        User      = "spiped";
+        PermissionsStartOnly = true;
+        SupplementaryGroups = "keys";
+      };
+
+      script = "exec ${pkgs.spiped}/bin/spiped -F `cat /etc/spiped/redis.spec`";
+    };
+
+    services.filesWatcher.predixy = {
+      restart = true;
+      paths = [ "${config.secrets.location}/redis/predixy.conf" ];
+    };
+
+    networking.firewall.allowedTCPPorts = [ 7617 16379 ];
+    secrets.keys = [
+      {
+        dest = "redis/predixy.conf";
+        user = "redis";
+        group = "redis";
+        permissions = "0400";
+        text = ''
+          Name Predixy
+          Bind 127.0.0.1:7617
+          ClientTimeout 300
+          WorkerThreads 1
+
+          Authority {
+              Auth "${myconfig.env.databases.redis.predixy.read}" {
+                  Mode read
+              }
+          }
+
+          StandaloneServerPool {
+            Databases 16
+            RefreshMethod fixed
+            Group shard001 {
+              + ${myconfig.env.databases.redis.socket}
+            }
+          }
+          '';
+      }
+      {
+        dest = "redis/spiped_keyfile";
+        user = "spiped";
+        group = "spiped";
+        permissions = "0400";
+        text = myconfig.env.databases.redis.spiped_key;
+      }
+    ];
+
+    systemd.services.predixy = {
+      description = "Redis proxy";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "redis.service" ];
+
+      serviceConfig = {
+        User = "redis";
+        Group = "redis";
+        SupplementaryGroups = "keys";
+        Type = "simple";
+
+        ExecStart = "${pkgs.predixy}/bin/predixy ${config.secrets.location}/redis/predixy.conf";
+      };
+
+    };
   };
 }
 
diff --git a/modules/private/databases/redis_replication.nix b/modules/private/databases/redis_replication.nix
new file mode 100644
index 0000000..cc626f5
--- /dev/null
+++ b/modules/private/databases/redis_replication.nix
@@ -0,0 +1,173 @@
+{ pkgs, config, myconfig, lib, ... }:
+let
+  cfg = config.myServices.databasesReplication.redis;
+in
+{
+  options.myServices.databasesReplication.redis = {
+    enable = lib.mkEnableOption "Enable redis replication";
+    base = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        Base path to put the replications
+        '';
+    };
+    hosts = lib.mkOption {
+      default = {};
+      description = ''
+        Hosts to backup
+        '';
+      type = lib.types.attrsOf (lib.types.submodule {
+        options = {
+          package = lib.mkOption {
+            type = lib.types.package;
+            default = pkgs.redis;
+            description = ''
+              Redis package for this host
+            '';
+          };
+          host = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Host to connect to
+              '';
+          };
+          port = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Port to connect to
+              '';
+          };
+          password = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            default = null;
+            description = ''
+              Password to use
+              '';
+          };
+        };
+      });
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users.redis = {
+      description = "Redis database user";
+      group = "redis";
+      uid = config.ids.uids.redis;
+      extraGroups = [ "keys" ];
+    };
+    users.groups.redis.gid = config.ids.gids.redis;
+
+    services.spiped = { # sync from eldiron
+      enable = true;
+      config.redis = {
+        encrypt = true;
+        source = "127.0.0.1:16379";
+        target = "${myconfig.env.servers.eldiron.ips.main.ip4}:16379";
+        keyfile = "${config.secrets.location}/redis/spiped_eldiron_keyfile";
+      };
+    };
+
+    secrets.keys = lib.flatten (lib.mapAttrsToList (name: hcfg: [
+      {
+        dest = "redis_replication/${name}/config";
+        user = "redis";
+        group = "redis";
+        permissions = "0400";
+        text = ''
+          pidfile ${cfg.base}/${name}/redis/redis.pid
+          port 0
+          unixsocket /run/redis_${name}/redis.sock
+          loglevel notice
+          logfile /dev/null
+          syslog-enabled yes
+          databases 16
+          save 900 1
+          save 300 10
+          save 60 10000
+          dbfilename dump.rdb
+          dir ${cfg.base}/${name}/redis/
+          slaveof ${hcfg.host} ${hcfg.port}
+          ${if hcfg.password != null then "masterauth ${hcfg.password}" else ""}
+          appendOnly no
+          appendfsync everysec
+          slowlog-log-slower-than 10000
+          slowlog-max-len 128
+          unixsocketperm 777
+          maxclients 1024
+          '';
+      }
+    ]) cfg.hosts) ++ [
+      { # For eldiron only
+        dest = "redis/spiped_eldiron_keyfile";
+        user = "spiped";
+        group = "spiped";
+        permissions = "0400";
+        text = myconfig.env.databases.redis.spiped_key;
+      }
+    ];
+
+    services.cron = {
+      enable = true;
+      systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg:
+        let
+          dataDir = "${cfg.base}/${name}/redis";
+          backupDir = "${cfg.base}/${name}/redis_backup";
+          backup_script = pkgs.writeScript "backup_redis_${name}" ''
+              #!${pkgs.stdenv.shell}
+
+              ${pkgs.coreutils}/bin/cp ${cfg.base}/${name}/redis/dump.rdb \
+                ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).rdb
+            '';
+          u = pkgs.callPackage ./utils.nix {};
+          cleanup_script = pkgs.writeScript "cleanup_redis_${name}" (u.exponentialDumps "rdb" backupDir);
+        in [
+          "0 22,4,10,16 * * * root ${backup_script}"
+          "0 3 * * * root ${cleanup_script}"
+        ]) cfg.hosts);
+    };
+
+    system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg:
+      lib.attrsets.nameValuePair "redis_replication_${name}" {
+        deps = [ "users" "groups" ];
+        text = ''
+          install -m 0700 -o redis -g redis -d ${cfg.base}/${name}/redis
+          install -m 0700 -o redis -g redis -d ${cfg.base}/${name}/redis_backup
+          '';
+      }) cfg.hosts;
+
+    systemd.services = {
+      spiped_redis = { # For eldiron
+        description = "Secure pipe 'redis'";
+        after = [ "network.target" ];
+        wantedBy = [ "multi-user.target" ];
+
+        serviceConfig = {
+          Restart   = "always";
+          User      = "spiped";
+          PermissionsStartOnly = true;
+          SupplementaryGroups = "keys";
+        };
+
+        script = "exec ${pkgs.spiped}/bin/spiped -F `cat /etc/spiped/redis.spec`";
+      };
+    } // lib.attrsets.mapAttrs' (name: hcfg:
+      let
+        dataDir = "${cfg.base}/${name}/redis";
+      in
+      lib.attrsets.nameValuePair "redis_backup_${name}" {
+        description = "Redis replication for ${name}";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        unitConfig.RequiresMountsFor = dataDir;
+
+        serviceConfig = {
+          ExecStart = "${hcfg.package}/bin/redis-server ${config.secrets.location}/redis_replication/${name}/config";
+          User = "redis";
+          RuntimeDirectory = "redis_${name}";
+        };
+    }) cfg.hosts;
+  };
+}
+
+
diff --git a/modules/private/databases/utils.nix b/modules/private/databases/utils.nix
index c296f49..06ce2cc 100644
--- a/modules/private/databases/utils.nix
+++ b/modules/private/databases/utils.nix
@@ -1,15 +1,15 @@
 { pkgs }:
 {
-  keepLastNDumps = backupDir: n: ''
+  keepLastNDumps = ext: backupDir: n: ''
     #!${pkgs.stdenv.shell}
 
     cd ${backupDir}
     ${pkgs.coreutils}/bin/rm -f \
-      $(${pkgs.coreutils}/bin/ls -1 *.sql \
+      $(${pkgs.coreutils}/bin/ls -1 *.${ext} \
         | ${pkgs.coreutils}/bin/sort -r \
         | ${pkgs.gnused}/bin/sed -e '1,${builtins.toString n}d')
     '';
-  exponentialDumps = backupDir: let
+  exponentialDumps = ext: backupDir: let
     log2rotateSrc = builtins.fetchGit {
       url = "https://github.com/avian2/pylog2rotate";
       ref = "master";
@@ -24,7 +24,7 @@
     #!${pkgs.stdenv.shell}
 
     cd ${backupDir}
-    ${pkgs.coreutils}/bin/rm -f $(ls -1 *.sql | grep -v 'T22:' | sort -r | sed -e '1,12d')
-    ${pkgs.coreutils}/bin/rm -f $(ls -1 *T22*.sql | ${log2rotate} --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M:%S+00:00.sql')
+    ${pkgs.coreutils}/bin/rm -f $(ls -1 *.${ext} | grep -v 'T22:' | sort -r | sed -e '1,12d')
+    ${pkgs.coreutils}/bin/rm -f $(ls -1 *T22*.${ext} | ${log2rotate} --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M:%S+00:00.${ext}')
     '';
 }
diff --git a/modules/private/default.nix b/modules/private/default.nix
index 51052ed..1d0f1a9 100644
--- a/modules/private/default.nix
+++ b/modules/private/default.nix
@@ -12,6 +12,7 @@ set = {
   redis      = ./databases/redis.nix;
   postgresqlReplication = ./databases/postgresql_replication.nix;
   mariadbReplication = ./databases/mariadb_replication.nix;
+  redisReplication = ./databases/redis_replication.nix;
 
   websites = ./websites;
   atenInte = ./websites/aten/integration.nix;
diff --git a/modules/private/system/backup-2.nix b/modules/private/system/backup-2.nix
index 0e94a1a..3d51fa3 100644
--- a/modules/private/system/backup-2.nix
+++ b/modules/private/system/backup-2.nix
@@ -77,6 +77,16 @@
         };
       };
     };
+    redis = {
+      enable = true;
+      base = "/backup2";
+      hosts = {
+        eldiron = {
+          host = "127.0.0.1";
+          port = "16379";
+        };
+      };
+    };
   };
 
   # This value determines the NixOS release with which your system is
diff --git a/pkgs/default.nix b/pkgs/default.nix
index 07b2700..82be20e 100644
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@@ -22,6 +22,7 @@ rec {
   opendmarc = callPackage ../pkgs/opendmarc { libspf2 = callPackage ../pkgs/opendmarc/libspf2.nix {}; };
   pg_activity = callPackage ../pkgs/pg_activity { inherit mylibs; };
   pgloader = callPackage ../pkgs/pgloader {};
+  predixy = callPackage ../pkgs/predixy { inherit mylibs; };
   telegram-cli = callPackage ../pkgs/telegram-cli { inherit mylibs; };
   telegram-history-dump = callPackage ../pkgs/telegram-history-dump { inherit mylibs; };
   telegramircd = callPackage ../pkgs/telegramircd { inherit mylibs; telethon = callPackage ../pkgs/telethon_sync {}; };
diff --git a/pkgs/predixy/default.nix b/pkgs/predixy/default.nix
new file mode 100644
index 0000000..7e7a9ba
--- /dev/null
+++ b/pkgs/predixy/default.nix
@@ -0,0 +1,10 @@
+{ stdenv, mylibs }:
+stdenv.mkDerivation (mylibs.fetchedGithub ./predixy.json // {
+  installPhase = ''
+    mkdir -p $out/bin
+    cp src/predixy $out/bin
+    mkdir -p $out/share
+    cp -r doc $out/share
+    cp -r conf $out/share/doc
+    '';
+})
diff --git a/pkgs/predixy/predixy.json b/pkgs/predixy/predixy.json
new file mode 100644
index 0000000..9f9759c
--- /dev/null
+++ b/pkgs/predixy/predixy.json
@@ -0,0 +1,15 @@
+{
+  "tag": "dacf3fb-master",
+  "meta": {
+    "name": "predixy",
+    "url": "https://github.com/joyieldInc/predixy",
+    "branch": "master"
+  },
+  "github": {
+    "owner": "joyieldInc",
+    "repo": "predixy",
+    "rev": "dacf3fb30c2602dc044040df04e194d44b49c1be",
+    "sha256": "0sbvy0jg551lwkfq8qh0a49cl9mhfnkhi3cnk25l8pz4jcdrr9k9",
+    "fetchSubmodules": true
+  }
+}