--all-databases > ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).sql
'';
u = pkgs.callPackage ./utils.nix {};
- cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps backupDir);
+ cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps "sql" backupDir);
in [
"0 22,4,10,16 * * * root ${backup_script}"
"0 3 * * * root ${cleanup_script}"
${hcfg.package}/bin/pg_dumpall -h ${dataDir} -f ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).sql
'';
u = pkgs.callPackage ./utils.nix {};
- cleanup_script = pkgs.writeScript "cleanup_postgresql_${name}" (u.keepLastNDumps backupDir 12);
+ cleanup_script = pkgs.writeScript "cleanup_postgresql_${name}" (u.keepLastNDumps "sql" backupDir 12);
in [
"0 22,4,10,16 * * * postgres ${backup_script}"
"0 3 * * * postgres ${cleanup_script}"
-{ lib, config, ... }:
+{ lib, config, pkgs, myconfig, ... }:
let
cfg = config.myServices.databases.redis;
in {
'';
};
systemd.services.redis.serviceConfig.RuntimeDirectory = cfg.systemdRuntimeDirectory;
+
+ services.spiped = {
+ enable = true;
+ config.redis = {
+ decrypt = true;
+ source = "0.0.0.0:16379";
+ target = "/run/redis/redis.sock";
+ keyfile = "${config.secrets.location}/redis/spiped_keyfile";
+ };
+ };
+ systemd.services.spiped_redis = {
+ description = "Secure pipe 'redis'";
+ after = [ "network.target" ];
+ wantedBy = [ "multi-user.target" ];
+
+ serviceConfig = {
+ Restart = "always";
+ User = "spiped";
+ PermissionsStartOnly = true;
+ SupplementaryGroups = "keys";
+ };
+
+ script = "exec ${pkgs.spiped}/bin/spiped -F `cat /etc/spiped/redis.spec`";
+ };
+
+ services.filesWatcher.predixy = {
+ restart = true;
+ paths = [ "${config.secrets.location}/redis/predixy.conf" ];
+ };
+
+ networking.firewall.allowedTCPPorts = [ 7617 16379 ];
+ secrets.keys = [
+ {
+ dest = "redis/predixy.conf";
+ user = "redis";
+ group = "redis";
+ permissions = "0400";
+ text = ''
+ Name Predixy
+ Bind 127.0.0.1:7617
+ ClientTimeout 300
+ WorkerThreads 1
+
+ Authority {
+ Auth "${myconfig.env.databases.redis.predixy.read}" {
+ Mode read
+ }
+ }
+
+ StandaloneServerPool {
+ Databases 16
+ RefreshMethod fixed
+ Group shard001 {
+ + ${myconfig.env.databases.redis.socket}
+ }
+ }
+ '';
+ }
+ {
+ dest = "redis/spiped_keyfile";
+ user = "spiped";
+ group = "spiped";
+ permissions = "0400";
+ text = myconfig.env.databases.redis.spiped_key;
+ }
+ ];
+
+ systemd.services.predixy = {
+ description = "Redis proxy";
+ wantedBy = [ "multi-user.target" ];
+ after = [ "redis.service" ];
+
+ serviceConfig = {
+ User = "redis";
+ Group = "redis";
+ SupplementaryGroups = "keys";
+ Type = "simple";
+
+ ExecStart = "${pkgs.predixy}/bin/predixy ${config.secrets.location}/redis/predixy.conf";
+ };
+
+ };
};
}
--- /dev/null
+{ pkgs, config, myconfig, lib, ... }:
+let
+ cfg = config.myServices.databasesReplication.redis;
+in
+{
+ options.myServices.databasesReplication.redis = {
+ enable = lib.mkEnableOption "Enable redis replication";
+ base = lib.mkOption {
+ type = lib.types.path;
+ description = ''
+ Base path to put the replications
+ '';
+ };
+ hosts = lib.mkOption {
+ default = {};
+ description = ''
+ Hosts to backup
+ '';
+ type = lib.types.attrsOf (lib.types.submodule {
+ options = {
+ package = lib.mkOption {
+ type = lib.types.package;
+ default = pkgs.redis;
+ description = ''
+ Redis package for this host
+ '';
+ };
+ host = lib.mkOption {
+ type = lib.types.str;
+ description = ''
+ Host to connect to
+ '';
+ };
+ port = lib.mkOption {
+ type = lib.types.str;
+ description = ''
+ Port to connect to
+ '';
+ };
+ password = lib.mkOption {
+ type = lib.types.nullOr lib.types.str;
+ default = null;
+ description = ''
+ Password to use
+ '';
+ };
+ };
+ });
+ };
+ };
+
+ config = lib.mkIf cfg.enable {
+ users.users.redis = {
+ description = "Redis database user";
+ group = "redis";
+ uid = config.ids.uids.redis;
+ extraGroups = [ "keys" ];
+ };
+ users.groups.redis.gid = config.ids.gids.redis;
+
+ services.spiped = { # sync from eldiron
+ enable = true;
+ config.redis = {
+ encrypt = true;
+ source = "127.0.0.1:16379";
+ target = "${myconfig.env.servers.eldiron.ips.main.ip4}:16379";
+ keyfile = "${config.secrets.location}/redis/spiped_eldiron_keyfile";
+ };
+ };
+
+ secrets.keys = lib.flatten (lib.mapAttrsToList (name: hcfg: [
+ {
+ dest = "redis_replication/${name}/config";
+ user = "redis";
+ group = "redis";
+ permissions = "0400";
+ text = ''
+ pidfile ${cfg.base}/${name}/redis/redis.pid
+ port 0
+ unixsocket /run/redis_${name}/redis.sock
+ loglevel notice
+ logfile /dev/null
+ syslog-enabled yes
+ databases 16
+ save 900 1
+ save 300 10
+ save 60 10000
+ dbfilename dump.rdb
+ dir ${cfg.base}/${name}/redis/
+ slaveof ${hcfg.host} ${hcfg.port}
+ ${if hcfg.password != null then "masterauth ${hcfg.password}" else ""}
+ appendOnly no
+ appendfsync everysec
+ slowlog-log-slower-than 10000
+ slowlog-max-len 128
+ unixsocketperm 777
+ maxclients 1024
+ '';
+ }
+ ]) cfg.hosts) ++ [
+ { # For eldiron only
+ dest = "redis/spiped_eldiron_keyfile";
+ user = "spiped";
+ group = "spiped";
+ permissions = "0400";
+ text = myconfig.env.databases.redis.spiped_key;
+ }
+ ];
+
+ services.cron = {
+ enable = true;
+ systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg:
+ let
+ dataDir = "${cfg.base}/${name}/redis";
+ backupDir = "${cfg.base}/${name}/redis_backup";
+ backup_script = pkgs.writeScript "backup_redis_${name}" ''
+ #!${pkgs.stdenv.shell}
+
+ ${pkgs.coreutils}/bin/cp ${cfg.base}/${name}/redis/dump.rdb \
+ ${backupDir}/$(${pkgs.coreutils}/bin/date -Iseconds).rdb
+ '';
+ u = pkgs.callPackage ./utils.nix {};
+ cleanup_script = pkgs.writeScript "cleanup_redis_${name}" (u.exponentialDumps "rdb" backupDir);
+ in [
+ "0 22,4,10,16 * * * root ${backup_script}"
+ "0 3 * * * root ${cleanup_script}"
+ ]) cfg.hosts);
+ };
+
+ system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg:
+ lib.attrsets.nameValuePair "redis_replication_${name}" {
+ deps = [ "users" "groups" ];
+ text = ''
+ install -m 0700 -o redis -g redis -d ${cfg.base}/${name}/redis
+ install -m 0700 -o redis -g redis -d ${cfg.base}/${name}/redis_backup
+ '';
+ }) cfg.hosts;
+
+ systemd.services = {
+ spiped_redis = { # For eldiron
+ description = "Secure pipe 'redis'";
+ after = [ "network.target" ];
+ wantedBy = [ "multi-user.target" ];
+
+ serviceConfig = {
+ Restart = "always";
+ User = "spiped";
+ PermissionsStartOnly = true;
+ SupplementaryGroups = "keys";
+ };
+
+ script = "exec ${pkgs.spiped}/bin/spiped -F `cat /etc/spiped/redis.spec`";
+ };
+ } // lib.attrsets.mapAttrs' (name: hcfg:
+ let
+ dataDir = "${cfg.base}/${name}/redis";
+ in
+ lib.attrsets.nameValuePair "redis_backup_${name}" {
+ description = "Redis replication for ${name}";
+ wantedBy = [ "multi-user.target" ];
+ after = [ "network.target" ];
+ unitConfig.RequiresMountsFor = dataDir;
+
+ serviceConfig = {
+ ExecStart = "${hcfg.package}/bin/redis-server ${config.secrets.location}/redis_replication/${name}/config";
+ User = "redis";
+ RuntimeDirectory = "redis_${name}";
+ };
+ }) cfg.hosts;
+ };
+}
+
+
{ pkgs }:
{
- keepLastNDumps = backupDir: n: ''
+ keepLastNDumps = ext: backupDir: n: ''
#!${pkgs.stdenv.shell}
cd ${backupDir}
${pkgs.coreutils}/bin/rm -f \
- $(${pkgs.coreutils}/bin/ls -1 *.sql \
+ $(${pkgs.coreutils}/bin/ls -1 *.${ext} \
| ${pkgs.coreutils}/bin/sort -r \
| ${pkgs.gnused}/bin/sed -e '1,${builtins.toString n}d')
'';
- exponentialDumps = backupDir: let
+ exponentialDumps = ext: backupDir: let
log2rotateSrc = builtins.fetchGit {
url = "https://github.com/avian2/pylog2rotate";
ref = "master";
#!${pkgs.stdenv.shell}
cd ${backupDir}
- ${pkgs.coreutils}/bin/rm -f $(ls -1 *.sql | grep -v 'T22:' | sort -r | sed -e '1,12d')
- ${pkgs.coreutils}/bin/rm -f $(ls -1 *T22*.sql | ${log2rotate} --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M:%S+00:00.sql')
+ ${pkgs.coreutils}/bin/rm -f $(ls -1 *.${ext} | grep -v 'T22:' | sort -r | sed -e '1,12d')
+ ${pkgs.coreutils}/bin/rm -f $(ls -1 *T22*.${ext} | ${log2rotate} --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M:%S+00:00.${ext}')
'';
}
redis = ./databases/redis.nix;
postgresqlReplication = ./databases/postgresql_replication.nix;
mariadbReplication = ./databases/mariadb_replication.nix;
+ redisReplication = ./databases/redis_replication.nix;
websites = ./websites;
atenInte = ./websites/aten/integration.nix;
};
};
};
+ redis = {
+ enable = true;
+ base = "/backup2";
+ hosts = {
+ eldiron = {
+ host = "127.0.0.1";
+ port = "16379";
+ };
+ };
+ };
};
# This value determines the NixOS release with which your system is
opendmarc = callPackage ../pkgs/opendmarc { libspf2 = callPackage ../pkgs/opendmarc/libspf2.nix {}; };
pg_activity = callPackage ../pkgs/pg_activity { inherit mylibs; };
pgloader = callPackage ../pkgs/pgloader {};
+ predixy = callPackage ../pkgs/predixy { inherit mylibs; };
telegram-cli = callPackage ../pkgs/telegram-cli { inherit mylibs; };
telegram-history-dump = callPackage ../pkgs/telegram-history-dump { inherit mylibs; };
telegramircd = callPackage ../pkgs/telegramircd { inherit mylibs; telethon = callPackage ../pkgs/telethon_sync {}; };
--- /dev/null
+{ stdenv, mylibs }:
+stdenv.mkDerivation (mylibs.fetchedGithub ./predixy.json // {
+ installPhase = ''
+ mkdir -p $out/bin
+ cp src/predixy $out/bin
+ mkdir -p $out/share
+ cp -r doc $out/share
+ cp -r conf $out/share/doc
+ '';
+})
--- /dev/null
+{
+ "tag": "dacf3fb-master",
+ "meta": {
+ "name": "predixy",
+ "url": "https://github.com/joyieldInc/predixy",
+ "branch": "master"
+ },
+ "github": {
+ "owner": "joyieldInc",
+ "repo": "predixy",
+ "rev": "dacf3fb30c2602dc044040df04e194d44b49c1be",
+ "sha256": "0sbvy0jg551lwkfq8qh0a49cl9mhfnkhi3cnk25l8pz4jcdrr9k9",
+ "fetchSubmodules": true
+ }
+}
projects / perso / Immae / Config / Nix.git / commitdiff
