From 1a64deeb894dc95e2645a75771732c6cc53a79ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Wed, 4 Oct 2023 01:35:06 +0200 Subject: Squash changes containing private information There were a lot of changes since the previous commit, but a lot of them contained personnal information about users. All thos changes got stashed into a single commit (history is kept in a different place) and private information was moved in a separate private repository --- modules/private/databases/default.nix | 57 ----- modules/private/databases/mariadb.nix | 182 --------------- modules/private/databases/mariadb_replication.nix | 251 --------------------- modules/private/databases/openldap/default.nix | 147 ------------ .../private/databases/openldap/eldiron_schemas.nix | 21 -- modules/private/databases/openldap/immae.schema | 179 --------------- modules/private/databases/openldap_replication.nix | 166 -------------- modules/private/databases/postgresql.nix | 228 ------------------- .../private/databases/postgresql_replication.nix | 182 --------------- modules/private/databases/redis.nix | 133 ----------- modules/private/databases/redis_replication.nix | 171 -------------- modules/private/databases/utils.nix | 30 --- 12 files changed, 1747 deletions(-) delete mode 100644 modules/private/databases/default.nix delete mode 100644 modules/private/databases/mariadb.nix delete mode 100644 modules/private/databases/mariadb_replication.nix delete mode 100644 modules/private/databases/openldap/default.nix delete mode 100644 modules/private/databases/openldap/eldiron_schemas.nix delete mode 100644 modules/private/databases/openldap/immae.schema delete mode 100644 modules/private/databases/openldap_replication.nix delete mode 100644 modules/private/databases/postgresql.nix delete mode 100644 modules/private/databases/postgresql_replication.nix delete mode 100644 modules/private/databases/redis.nix delete mode 100644 modules/private/databases/redis_replication.nix delete mode 100644 modules/private/databases/utils.nix (limited to 'modules/private/databases') diff --git a/modules/private/databases/default.nix b/modules/private/databases/default.nix deleted file mode 100644 index 1241658..0000000 --- a/modules/private/databases/default.nix +++ /dev/null @@ -1,57 +0,0 @@ -{ lib, config, nodes, ... }: -let - cfg = config.myServices.databases; -in -{ - options.myServices = { - databases.enable = lib.mkEnableOption "my databases service"; - databasesCerts = lib.mkOption { - description = "Default databases configurations for certificates as accepted by acme"; - }; - }; - - config.myServices.databases = lib.mkIf cfg.enable { - mariadb = { - enable = true; - ldapConfig = { - inherit (config.myEnv.ldap) host base; - inherit (config.myEnv.databases.mysql.pam) dn filter password; - }; - replicationLdapConfig = { - inherit (config.myEnv.ldap) host base; - inherit (config.myEnv.servers.eldiron.ldap) dn password; - }; - credentials.root = config.myEnv.databases.mysql.systemUsers.root; - }; - - openldap = { - accessFile = ../../../nixops/secrets/ldap.conf; - baseDn = config.myEnv.ldap.base; - rootDn = config.myEnv.ldap.root_dn; - rootPw = config.myEnv.ldap.root_pw; - enable = true; - }; - - postgresql = { - ldapConfig = { - inherit (config.myEnv.ldap) host base; - inherit (config.myEnv.databases.postgresql.pam) dn filter password; - }; - replicationLdapConfig = { - inherit (config.myEnv.ldap) host base; - inherit (config.myEnv.servers.eldiron.ldap) dn password; - }; - authorizedHosts = { - }; - replicationHosts = { - backup-2 = { - ip4 = [config.myEnv.servers.backup-2.ips.main.ip4]; - ip6 = config.myEnv.servers.backup-2.ips.main.ip6; - }; - }; - enable = true; - }; - - redis.enable = true; - }; -} diff --git a/modules/private/databases/mariadb.nix b/modules/private/databases/mariadb.nix deleted file mode 100644 index 101eb3f..0000000 --- a/modules/private/databases/mariadb.nix +++ /dev/null @@ -1,182 +0,0 @@ -{ lib, pkgs, config, ... }: -let - cfg = config.myServices.databases.mariadb; -in { - options.myServices.databases = { - mariadb = { - enable = lib.mkOption { - default = false; - example = true; - description = "Whether to enable mariadb database"; - type = lib.types.bool; - }; - package = lib.mkOption { - type = lib.types.package; - default = pkgs.mariadb; - description = '' - Mariadb package to use. - ''; - }; - credentials = lib.mkOption { - default = {}; - description = "Credentials"; - type = lib.types.attrsOf lib.types.str; - }; - ldapConfig = lib.mkOption { - description = "LDAP configuration to allow PAM identification via LDAP"; - type = lib.types.submodule { - options = { - host = lib.mkOption { type = lib.types.str; }; - base = lib.mkOption { type = lib.types.str; }; - dn = lib.mkOption { type = lib.types.str; }; - password = lib.mkOption { type = lib.types.str; }; - filter = lib.mkOption { type = lib.types.str; }; - }; - }; - }; - replicationLdapConfig = lib.mkOption { - description = "LDAP configuration to allow replication"; - type = lib.types.submodule { - options = { - host = lib.mkOption { type = lib.types.str; }; - base = lib.mkOption { type = lib.types.str; }; - dn = lib.mkOption { type = lib.types.str; }; - password = lib.mkOption { type = lib.types.str; }; - }; - }; - }; - dataDir = lib.mkOption { - type = lib.types.path; - default = "/var/lib/mysql"; - description = '' - The directory where Mariadb stores its data. - ''; - }; - # Output variables - socketsDir = lib.mkOption { - type = lib.types.path; - default = "/run/mysqld"; - description = '' - The directory where Mariadb puts sockets. - ''; - }; - sockets = lib.mkOption { - type = lib.types.attrsOf lib.types.path; - default = { - mysqld = "${cfg.socketsDir}/mysqld.sock"; - }; - readOnly = true; - description = '' - Mariadb sockets - ''; - }; - }; - }; - - config = lib.mkIf cfg.enable { - networking.firewall.allowedTCPPorts = [ 3306 ]; - - # for adminer, ssl is implemented with mysqli only, which is - # currently disabled because it’s not compatible with pam. - # Thus we need to generate two users for each 'remote': one remote - # with SSL, and one localhost without SSL. - # User identified by LDAP: - # CREATE USER foo@% IDENTIFIED VIA pam USING 'mysql' REQUIRE SSL; - # CREATE USER foo@localhost IDENTIFIED VIA pam USING 'mysql'; - - # To create a user (host) for replication: - # CREATE USER 'host'@'%' IDENTIFIED VIA pam USING 'mysql_replication' REQUIRE SSL; - # GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD, LOCK TABLES, SELECT, SHOW VIEW ON *.* TO 'host'@'%'; - # (the lock/select grant permits to let the replication host handle - # the initial fetch of the database) - # % should be valid for both localhost (for cron dumps) and the origin host. - services.mysql = { - enable = true; - package = cfg.package; - dataDir = cfg.dataDir; - settings = { - mysqld = { - ssl_ca = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; - ssl_key = "${config.security.acme.certs.mysql.directory}/key.pem"; - ssl_cert = "${config.security.acme.certs.mysql.directory}/fullchain.pem"; - - # for replication - log-bin = "mariadb-bin"; - server-id = "1"; - - # this introduces a small delay before storing on disk, but - # makes it order of magnitudes quicker - innodb_flush_log_at_trx_commit = "0"; - }; - }; - }; - - users.users.mysql.extraGroups = [ "keys" ]; - security.acme.certs."mysql" = config.myServices.databasesCerts // { - user = "mysql"; - group = "mysql"; - domain = "db-1.immae.eu"; - postRun = '' - systemctl restart mysql.service - ''; - }; - - secrets.keys = { - "mysql/mysqldump" = { - permissions = "0400"; - user = "root"; - group = "root"; - text = '' - [mysqldump] - user = root - password = ${cfg.credentials.root} - ''; - }; - "mysql/pam" = { - permissions = "0400"; - user = "mysql"; - group = "mysql"; - text = with cfg.ldapConfig; '' - host ${host} - base ${base} - binddn ${dn} - bindpw ${password} - pam_filter ${filter} - ssl start_tls - ''; - }; - "mysql/pam_replication" = { - permissions = "0400"; - user = "mysql"; - group = "mysql"; - text = with cfg.replicationLdapConfig; '' - host ${host} - base ${base} - binddn ${dn} - bindpw ${password} - pam_login_attribute cn - ssl start_tls - ''; - }; - }; - - security.pam.services = let - pam_ldap = "${pkgs.pam_ldap}/lib/security/pam_ldap.so"; - in { - mysql = { - text = '' - # https://mariadb.com/kb/en/mariadb/pam-authentication-plugin/ - auth required ${pam_ldap} config=${config.secrets.fullPaths."mysql/pam"} - account required ${pam_ldap} config=${config.secrets.fullPaths."mysql/pam"} - ''; - }; - mysql_replication = { - text = '' - auth required ${pam_ldap} config=${config.secrets.fullPaths."mysql/pam_replication"} - account required ${pam_ldap} config=${config.secrets.fullPaths."mysql/pam_replication"} - ''; - }; - }; - - }; -} diff --git a/modules/private/databases/mariadb_replication.nix b/modules/private/databases/mariadb_replication.nix deleted file mode 100644 index 68e6f7f..0000000 --- a/modules/private/databases/mariadb_replication.nix +++ /dev/null @@ -1,251 +0,0 @@ -{ pkgs, config, lib, ... }: -let - cfg = config.myServices.databasesReplication.mariadb; -in -{ - options.myServices.databasesReplication.mariadb = { - enable = lib.mkEnableOption "Enable mariadb replication"; - base = lib.mkOption { - type = lib.types.path; - description = '' - Base path to put the replications - ''; - }; - hosts = lib.mkOption { - default = {}; - description = '' - Hosts to backup - ''; - type = lib.types.attrsOf (lib.types.submodule { - options = { - package = lib.mkOption { - type = lib.types.package; - default = pkgs.mariadb; - description = '' - Mariadb package for this host - ''; - }; - serverId = lib.mkOption { - type = lib.types.int; - description = '' - Server id to use for replication cluster (must be unique among the cluster!) - ''; - }; - host = lib.mkOption { - type = lib.types.str; - description = '' - Host to connect to - ''; - }; - port = lib.mkOption { - type = lib.types.str; - description = '' - Port to connect to - ''; - }; - user = lib.mkOption { - type = lib.types.str; - description = '' - User to connect as - ''; - }; - password = lib.mkOption { - type = lib.types.str; - description = '' - Password to use - ''; - }; - dumpUser = lib.mkOption { - type = lib.types.str; - description = '' - User who can do a dump - ''; - }; - dumpPassword = lib.mkOption { - type = lib.types.str; - description = '' - Password for the dump user - ''; - }; - }; - }); - }; - }; - - config = lib.mkIf cfg.enable { - users.users.mysql = { - description = "MySQL server user"; - group = "mysql"; - uid = config.ids.uids.mysql; - extraGroups = [ "keys" ]; - }; - users.groups.mysql.gid = config.ids.gids.mysql; - - secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (name: hcfg: [ - (lib.nameValuePair "mysql_replication/${name}/slave_init_commands" { - user = "mysql"; - group = "mysql"; - permissions = "0400"; - text = '' - CHANGE MASTER TO master_host="${hcfg.host}", master_port=${hcfg.port}, master_user="${hcfg.user}", master_password="${hcfg.password}", master_ssl=1, master_use_gtid=slave_pos; - START SLAVE; - ''; - }) - (lib.nameValuePair "mysql_replication/${name}/mysqldump_remote" { - permissions = "0400"; - user = "root"; - group = "root"; - text = '' - [mysqldump] - user = ${hcfg.user} - password = ${hcfg.password} - ''; - }) - (lib.nameValuePair "mysql_replication/${name}/mysqldump" { - permissions = "0400"; - user = "root"; - group = "root"; - text = '' - [mysqldump] - user = ${hcfg.dumpUser} - password = ${hcfg.dumpPassword} - ''; - }) - (lib.nameValuePair "mysql_replication/${name}/client" { - permissions = "0400"; - user = "mysql"; - group = "mysql"; - text = '' - [client] - user = ${hcfg.dumpUser} - password = ${hcfg.dumpPassword} - ''; - }) - ]) cfg.hosts)); - - services.cron = { - enable = true; - systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg: - let - dataDir = "${cfg.base}/${name}/mysql"; - backupDir = "${cfg.base}/${name}/mysql_backup"; - backup_script = pkgs.writeScript "backup_mysql_${name}" '' - #!${pkgs.stdenv.shell} - - set -euo pipefail - - filename=${backupDir}/$(${pkgs.coreutils}/bin/date -Iminutes).sql - ${hcfg.package}/bin/mysqldump \ - --defaults-file=${config.secrets.fullPaths."mysql_replication/${name}/mysqldump"} \ - -S /run/mysqld_${name}/mysqld.sock \ - --gtid \ - --master-data \ - --flush-privileges \ - --ignore-database=netdata \ - --all-databases > $filename - ${pkgs.gzip}/bin/gzip $filename - ''; - u = pkgs.callPackage ./utils.nix {}; - cleanup_script = pkgs.writeScript "cleanup_mysql_${name}" (u.exponentialDumps "sql.gz" backupDir); - in [ - "0 22,4,10,16 * * * root ${backup_script}" - "0 3 * * * root ${cleanup_script}" - ]) cfg.hosts); - }; - - system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg: - lib.attrsets.nameValuePair "mysql_replication_${name}" { - deps = [ "users" "groups" ]; - text = '' - install -m 0700 -o mysql -g mysql -d ${cfg.base}/${name}/mysql - install -m 0700 -o mysql -g mysql -d ${cfg.base}/${name}/mysql_backup - ''; - }) cfg.hosts; - - environment.etc = lib.attrsets.mapAttrs' (name: hcfg: - lib.attrsets.nameValuePair "mysql/${name}_my.cnf" { - text = '' - [mysqld] - skip-networking - socket = /run/mysqld_${name}/mysqld.sock - datadir = ${cfg.base}/${name}/mysql/ - log-bin = mariadb-bin - server-id = ${builtins.toString hcfg.serverId} - ''; - } - ) cfg.hosts; - - systemd.services = lib.attrsets.mapAttrs' (name: hcfg: - let - dataDir = "${cfg.base}/${name}/mysql"; - in - lib.attrsets.nameValuePair "mysql_backup_${name}" { - description = "Mysql replication for ${name}"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - restartTriggers = [ config.environment.etc."mysql/${name}_my.cnf".source ]; - unitConfig.RequiresMountsFor = dataDir; - - preStart = '' - if ! test -e ${dataDir}/mysql; then - if ! test -e ${dataDir}/initial.sql; then - ${hcfg.package}/bin/mysqldump \ - --defaults-file=${config.secrets.fullPaths."mysql_replication/${name}/mysqldump_remote"} \ - -h ${hcfg.host} \ - -P ${hcfg.port} \ - --ssl \ - --gtid \ - --flush-privileges \ - --master-data \ - --all-databases > ${dataDir}/initial.sql - fi - - ${hcfg.package}/bin/mysql_install_db \ - --defaults-file=/etc/mysql/${name}_my.cnf \ - --user=mysql \ - --datadir=${dataDir} \ - --basedir=${hcfg.package} - fi - ''; - - serviceConfig = { - User = "mysql"; - Group = "mysql"; - RuntimeDirectory = "mysqld_${name}"; - RuntimeDirectoryMode = "0755"; - SupplementaryGroups = "keys"; - PermissionsStartOnly = true; - Type = "notify"; - - ExecStart = "${hcfg.package}/bin/mysqld --defaults-file=/etc/mysql/${name}_my.cnf --user=mysql --datadir=${dataDir} --basedir=${hcfg.package}"; - ExecStartPost = - let - sql_before = pkgs.writeText "mysql-initial-before" '' - DROP DATABASE test; - INSTALL SONAME 'auth_pam'; - ''; - setupScript = pkgs.writeScript "mysql-setup" '' - #!${pkgs.runtimeShell} -e - - if test -e ${dataDir}/initial.sql; then - cat \ - ${sql_before} \ - ${dataDir}/initial.sql \ - ${config.secrets.fullPaths."mysql_replication/${name}/slave_init_commands"} \ - | ${hcfg.package}/bin/mysql \ - --defaults-file=/etc/mysql/${name}_my.cnf \ - -S /run/mysqld_${name}/mysqld.sock \ - --user=root - rm -f ${dataDir}/initial.sql - fi - ''; - in - "+${setupScript}"; - # initial dump can take a long time - TimeoutStartSec="infinity"; - TimeoutStopSec = 120; - }; - }) cfg.hosts; - }; -} - diff --git a/modules/private/databases/openldap/default.nix b/modules/private/databases/openldap/default.nix deleted file mode 100644 index d35aca0..0000000 --- a/modules/private/databases/openldap/default.nix +++ /dev/null @@ -1,147 +0,0 @@ -{ lib, pkgs, config, ... }: -let - cfg = config.myServices.databases.openldap; - ldapConfig = let - eldiron_schemas = pkgs.callPackage ./eldiron_schemas.nix {}; - in '' - ${eldiron_schemas} - - pidfile ${cfg.pids.pid} - argsfile ${cfg.pids.args} - - moduleload back_hdb - backend hdb - - TLSCertificateFile ${config.security.acme.certs.ldap.directory}/cert.pem - TLSCertificateKeyFile ${config.security.acme.certs.ldap.directory}/key.pem - TLSCACertificateFile ${config.security.acme.certs.ldap.directory}/fullchain.pem - TLSCACertificatePath ${pkgs.cacert.unbundled}/etc/ssl/certs/ - #This makes openldap crash - #TLSCipherSuite DEFAULT - - sasl-host kerberos.immae.eu - ''; -in -{ - options.myServices.databases = { - openldap = { - enable = lib.mkOption { - default = false; - example = true; - description = "Whether to enable ldap"; - type = lib.types.bool; - }; - baseDn = lib.mkOption { - type = lib.types.str; - description = '' - Base DN for LDAP - ''; - }; - rootDn = lib.mkOption { - type = lib.types.str; - description = '' - Root DN - ''; - }; - rootPw = lib.mkOption { - type = lib.types.str; - description = '' - Root (Hashed) password - ''; - }; - accessFile = lib.mkOption { - type = lib.types.path; - description = '' - The file path that defines the access - ''; - }; - dataDir = lib.mkOption { - type = lib.types.path; - default = "/var/lib/openldap"; - description = '' - The directory where Openldap stores its data. - ''; - }; - socketsDir = lib.mkOption { - type = lib.types.path; - default = "/run/slapd"; - description = '' - The directory where Openldap puts sockets and pid files. - ''; - }; - # Output variables - pids = lib.mkOption { - type = lib.types.attrsOf lib.types.path; - default = { - pid = "${cfg.socketsDir}/slapd.pid"; - args = "${cfg.socketsDir}/slapd.args"; - }; - readOnly = true; - description = '' - Slapd pid files - ''; - }; - }; - }; - - config = lib.mkIf cfg.enable { - secrets.keys = { - "ldap/password" = { - permissions = "0400"; - user = "openldap"; - group = "openldap"; - text = "rootpw ${cfg.rootPw}"; - }; - "ldap/access" = { - permissions = "0400"; - user = "openldap"; - group = "openldap"; - text = builtins.readFile cfg.accessFile; - }; - "ldap" = { - permissions = "0500"; - user = "openldap"; - group = "openldap"; - isDir = true; - }; - }; - users.users.openldap.extraGroups = [ "keys" ]; - networking.firewall.allowedTCPPorts = [ 636 389 ]; - - security.acme.certs."ldap" = config.myServices.databasesCerts // { - user = "openldap"; - group = "openldap"; - domain = "ldap.immae.eu"; - postRun = '' - systemctl restart openldap.service - ''; - }; - - services.filesWatcher.openldap = { - restart = true; - paths = [ config.secrets.fullPaths."ldap" ]; - }; - - services.openldap = { - enable = true; - dataDir = cfg.dataDir; - urlList = [ "ldap://" "ldaps://" ]; - logLevel = "none"; - extraConfig = ldapConfig; - extraDatabaseConfig = '' - moduleload memberof - overlay memberof - - moduleload syncprov - overlay syncprov - syncprov-checkpoint 100 10 - - include ${config.secrets.fullPaths."ldap/access"} - ''; - rootpwFile = config.secrets.fullPaths."ldap/password"; - suffix = cfg.baseDn; - rootdn = cfg.rootDn; - database = "hdb"; - }; - }; -} diff --git a/modules/private/databases/openldap/eldiron_schemas.nix b/modules/private/databases/openldap/eldiron_schemas.nix deleted file mode 100644 index cf45ebe..0000000 --- a/modules/private/databases/openldap/eldiron_schemas.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ fetchurl, openldap }: -let - kerberosSchema = fetchurl { - url = "https://raw.githubusercontent.com/krb5/krb5/0bdd3b8058ed4ec9acc050e316bea86f6830b15f/src/plugins/kdb/ldap/libkdb_ldap/kerberos.schema"; - sha256 = "17fnkkf6s3lznsl7wp6914pqsc78d038rh38l638big8z608ksww"; - }; - puppetSchema = fetchurl { - url = "https://raw.githubusercontent.com/puppetlabs/puppet/bf7c108825ffdb5ea89cf3e500d55d27ab64b8d2/ext/ldap/puppet.schema"; - sha256 = "11bjf5zfvqlim7p9vddcafs0wiq3v8ys77x8h6fbp9c6bdfh0awh"; - }; - schemas = [ - #"${openldap}/etc/schema/core.schema" - #"${openldap}/etc/schema/cosine.schema" - #"${openldap}/etc/schema/inetorgperson.schema" - #"${openldap}/etc/schema/nis.schema" - puppetSchema - kerberosSchema - ./immae.schema - ]; -in - builtins.concatStringsSep "\n" (map (v: "include ${v}") schemas) diff --git a/modules/private/databases/openldap/immae.schema b/modules/private/databases/openldap/immae.schema deleted file mode 100644 index d2ef972..0000000 --- a/modules/private/databases/openldap/immae.schema +++ /dev/null @@ -1,179 +0,0 @@ -# vim: set filetype=slapd: -objectIdentifier Immaeroot 1.3.6.1.4.1.50071 - -objectIdentifier Immae Immaeroot:2 -objectIdentifier ImmaeattributeType Immae:3 -objectIdentifier ImmaeobjectClass Immae:4 - -# TT-RSS -attributetype ( ImmaeattributeType:1 NAME 'immaeTtrssLogin' - DESC 'login for TTRSS' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{256} ) - -objectclass ( ImmaeobjectClass:1 NAME 'immaeTtrssClass' - DESC 'Expansion of the existing object classes for ttrss' - SUP top AUXILIARY - MUST ( immaeTtrssLogin ) ) - -# FTP -attributetype ( ImmaeattributeType:2 NAME 'immaeFtpDirectory' - DESC 'home directory for ftp' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( ImmaeattributeType:3 NAME 'immaeFtpUid' - DESC 'user id for ftp' - EQUALITY integerMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 ) - -attributetype ( ImmaeattributeType:4 NAME 'immaeFtpGid' - DESC 'group id for ftp' - EQUALITY integerMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 ) - -objectclass ( ImmaeobjectClass:2 NAME 'immaeFtpClass' - DESC 'Expansion of the existing object classes for ftp' - SUP top AUXILIARY - MUST ( immaeFtpDirectory $ immaeFtpGid $ immaeFtpUid ) ) - - -# SSH keys -attributetype ( ImmaeattributeType:5 NAME 'immaeSshKey' - DESC 'OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -objectClass ( ImmaeobjectClass:3 NAME 'immaeSshClass' - DESC 'OpenSSH class' - SUP top AUXILIARY - MAy ( immaeSSHKey ) ) - -# Specific access -attributetype (ImmaeattributeType:6 NAME 'immaeAccessDn' - EQUALITY distinguishedNameMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) - -attributetype (ImmaeattributeType:17 NAME 'immaeAccessWriteDn' - EQUALITY distinguishedNameMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) - -attributetype (ImmaeattributeType:18 NAME 'immaeAccessReadSubtree' - EQUALITY distinguishedNameMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) - -objectClass ( ImmaeobjectClass:4 NAME 'immaeAccessClass' - DESC 'Access class' - SUP top AUXILIARY - MAY ( immaeAccessDn $ immaeAccessWriteDn $ immaeAccessReadSubtree ) ) - -# Xmpp uid -attributetype ( ImmaeattributeType:7 NAME 'immaeXmppUid' - DESC 'user part for Xmpp' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{256} ) - -objectclass ( ImmaeobjectClass:5 NAME 'immaeXmppClass' - DESC 'Expansion of the existing object classes for XMPP' - SUP top AUXILIARY - MUST ( immaeXmppUid ) ) - -# Postfix accounts -attributetype ( ImmaeattributeType:8 NAME 'immaePostfixAddress' - DESC 'the dovecot address to match as username' - EQUALITY caseIgnoreIA5Match - SUBSTR caseIgnoreIA5SubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) - -attributetype ( ImmaeattributeType:9 NAME 'immaePostfixHome' - DESC 'the postfix home directory' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - -attributetype ( ImmaeattributeType:10 NAME 'immaePostfixMail' - DESC 'the dovecot mail location' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - -attributetype ( ImmaeattributeType:11 NAME 'immaePostfixUid' - DESC 'the dovecot uid' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - -attributetype ( ImmaeattributeType:12 NAME 'immaePostfixGid' - DESC 'the dovecot gid' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - -objectclass ( ImmaeobjectClass:6 NAME 'immaePostfixClass' - DESC 'Expansion of the existing object classes for Postfix' - SUP top AUXILIARY - MUST ( immaePostfixAddress $ immaePostfixHome $ - immaePostfixMail $ immaePostfixUid $ immaePostfixGid ) - ) - -# Tinc informations -# Domaine = une classe a part ou une partie du dn ? -# attributetype ( ImmaeattributeType:13 NAME 'immaeTincIpSegment' -# DESC 'the internal ip segment in tinc' -# EQUALITY caseIgnoreIA5Match -# SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) -# -# attributetype ( ImmaeattributeType:14 NAME 'immaeTincSubdomain' -# DESC 'the host subdomain' -# EQUALITY caseIgnoreIA5Match -# SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) -# -# attributetype ( ImmaeattributeType:15 NAME 'immaeTincHostname' -# DESC 'the host name' -# EQUALITY caseIgnoreIA5Match -# SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) -# -# objectclass ( ImmaeobjectClass:7 NAME 'immaeTincHostClass' -# DESC 'Expansion of the existing object classes for Tinc' -# SUP top AUXILIARY -# MUST ( immaeTincInternalIp $ immaeTincSubdomain $ -# immaeTincHostname ) -# ) - -attributetype (ImmaeattributeType:16 NAME 'immaePuppetJson' - DESC 'Puppet hiera json' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -objectclass ( ImmaeobjectClass:8 NAME 'immaePuppetClass' - DESC 'Expansion of the existing object classes for Puppet' - SUP top AUXILIARY - MUST ( immaePuppetJson ) - ) - -attributetype (ImmaeattributeType:19 NAME 'immaeTaskId' - DESC 'Taskwarrior server Org:Name:Key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{256} ) - -objectclass ( ImmaeobjectClass:9 NAME 'immaeTaskClass' - DESC 'Expansion of the existing object classes for Task' - SUP top AUXILIARY - MUST ( immaeTaskId ) - ) - -# Peertube uid -attributetype ( ImmaeattributeType:20 NAME 'immaePeertubeId' - DESC 'login for Peertube' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{256} ) - -objectclass ( ImmaeobjectClass:10 NAME 'immaePeertubeClass' - DESC 'Expansion of the existing object classes for peertube' - SUP top AUXILIARY - MUST ( immaePeertubeId ) ) - - -# Last: -# attributetype ( ImmaeattributeType:20 NAME 'immaePeertubeId' -# objectclass ( ImmaeobjectClass:10 NAME 'immaePeertubeClass' diff --git a/modules/private/databases/openldap_replication.nix b/modules/private/databases/openldap_replication.nix deleted file mode 100644 index b456323..0000000 --- a/modules/private/databases/openldap_replication.nix +++ /dev/null @@ -1,166 +0,0 @@ -{ pkgs, config, lib, ... }: -let - cfg = config.myServices.databasesReplication.openldap; - eldiron_schemas = pkgs.callPackage ./openldap/eldiron_schemas.nix {}; - ldapConfig = hcfg: name: pkgs.writeText "slapd.conf" '' - include ${pkgs.openldap}/etc/schema/core.schema - include ${pkgs.openldap}/etc/schema/cosine.schema - include ${pkgs.openldap}/etc/schema/inetorgperson.schema - include ${pkgs.openldap}/etc/schema/nis.schema - ${eldiron_schemas} - pidfile /run/slapd_${name}/slapd.pid - argsfile /run/slapd_${name}/slapd.args - - moduleload back_hdb - backend hdb - database hdb - - suffix "${hcfg.base}" - rootdn "cn=root,${hcfg.base}" - directory ${cfg.base}/${name}/openldap - - index objectClass eq - index uid pres,eq - index entryUUID eq - - include ${config.secrets.fullPaths."openldap_replication/${name}/replication_config"} - ''; -in -{ - options.myServices.databasesReplication.openldap = { - enable = lib.mkEnableOption "Enable openldap replication"; - base = lib.mkOption { - type = lib.types.path; - description = '' - Base path to put the replications - ''; - }; - hosts = lib.mkOption { - default = {}; - description = '' - Hosts to backup - ''; - type = lib.types.attrsOf (lib.types.submodule { - options = { - package = lib.mkOption { - type = lib.types.package; - default = pkgs.openldap; - description = '' - Openldap package for this host - ''; - }; - url = lib.mkOption { - type = lib.types.str; - description = '' - Host to connect to - ''; - }; - base = lib.mkOption { - type = lib.types.str; - description = '' - Base DN to replicate - ''; - }; - dn = lib.mkOption { - type = lib.types.str; - description = '' - DN to use - ''; - }; - password = lib.mkOption { - type = lib.types.str; - description = '' - Password to use - ''; - }; - }; - }); - }; - }; - - config = lib.mkIf cfg.enable { - users.users.openldap = { - description = "Openldap database user"; - group = "openldap"; - uid = config.ids.uids.openldap; - extraGroups = [ "keys" ]; - }; - users.groups.openldap.gid = config.ids.gids.openldap; - - secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (name: hcfg: [ - (lib.nameValuePair "openldap_replication/${name}/replication_config" { - user = "openldap"; - group = "openldap"; - permissions = "0400"; - text = '' - syncrepl rid=000 - provider=${hcfg.url} - type=refreshAndPersist - searchbase="${hcfg.base}" - retry="5 10 300 +" - attrs="*,+" - schemachecking=off - bindmethod=simple - binddn="${hcfg.dn}" - credentials="${hcfg.password}" - ''; - }) - (lib.nameValuePair "openldap_replication/${name}/replication_password" { - user = "openldap"; - group = "openldap"; - permissions = "0400"; - text = hcfg.password; - }) - ]) cfg.hosts)); - - services.cron = { - enable = true; - systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg: - let - dataDir = "${cfg.base}/${name}/openldap"; - backupDir = "${cfg.base}/${name}/openldap_backup"; - backup_script = pkgs.writeScript "backup_openldap_${name}" '' - #!${pkgs.stdenv.shell} - - ${hcfg.package}/bin/slapcat -b "${hcfg.base}" -f ${ldapConfig hcfg name} -l ${backupDir}/$(${pkgs.coreutils}/bin/date -Iminutes).ldif - ''; - u = pkgs.callPackage ./utils.nix {}; - cleanup_script = pkgs.writeScript "cleanup_openldap_${name}" (u.exponentialDumps "ldif" backupDir); - in [ - "0 22,4,10,16 * * * root ${backup_script}" - "0 3 * * * root ${cleanup_script}" - ]) cfg.hosts); - }; - - system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg: - lib.attrsets.nameValuePair "openldap_replication_${name}" { - deps = [ "users" "groups" ]; - text = '' - install -m 0700 -o openldap -g openldap -d ${cfg.base}/${name}/openldap - install -m 0700 -o openldap -g openldap -d ${cfg.base}/${name}/openldap_backup - ''; - }) cfg.hosts; - - systemd.services = lib.attrsets.mapAttrs' (name: hcfg: - let - dataDir = "${cfg.base}/${name}/openldap"; - in - lib.attrsets.nameValuePair "openldap_backup_${name}" { - description = "Openldap replication for ${name}"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - unitConfig.RequiresMountsFor = dataDir; - - preStart = '' - mkdir -p /run/slapd_${name} - chown -R "openldap:openldap" /run/slapd_${name} - ''; - - serviceConfig = { - ExecStart = "${hcfg.package}/libexec/slapd -d 0 -u openldap -g openldap -f ${ldapConfig hcfg name}"; - }; - }) cfg.hosts; - }; -} - - diff --git a/modules/private/databases/postgresql.nix b/modules/private/databases/postgresql.nix deleted file mode 100644 index a6c4cc9..0000000 --- a/modules/private/databases/postgresql.nix +++ /dev/null @@ -1,228 +0,0 @@ -{ lib, pkgs, config, ... }: -let - cfg = config.myServices.databases.postgresql; -in { - options.myServices.databases = { - postgresql = { - enable = lib.mkOption { - default = false; - example = true; - description = "Whether to enable postgresql database"; - type = lib.types.bool; - }; - package = lib.mkOption { - type = lib.types.package; - default = pkgs.postgresql; - description = '' - Postgresql package to use. - ''; - }; - ldapConfig = lib.mkOption { - description = "LDAP configuration to allow PAM identification via LDAP"; - type = lib.types.submodule { - options = { - host = lib.mkOption { type = lib.types.str; }; - base = lib.mkOption { type = lib.types.str; }; - dn = lib.mkOption { type = lib.types.str; }; - password = lib.mkOption { type = lib.types.str; }; - filter = lib.mkOption { type = lib.types.str; }; - }; - }; - }; - replicationLdapConfig = lib.mkOption { - description = "LDAP configuration to allow replication"; - type = lib.types.submodule { - options = { - host = lib.mkOption { type = lib.types.str; }; - base = lib.mkOption { type = lib.types.str; }; - dn = lib.mkOption { type = lib.types.str; }; - password = lib.mkOption { type = lib.types.str; }; - }; - }; - }; - authorizedHosts = lib.mkOption { - default = {}; - description = "Hosts to allow connections from"; - type = lib.types.attrsOf (lib.types.listOf (lib.types.submodule { - options = { - method = lib.mkOption { - default = "md5"; - type = lib.types.str; - }; - username = lib.mkOption { - default = "all"; - type = lib.types.str; - }; - database = lib.mkOption { - default = "all"; - type = lib.types.str; - }; - ip4 = lib.mkOption { - default = []; - type = lib.types.listOf lib.types.str; - }; - ip6 = lib.mkOption { - default = []; - type = lib.types.listOf lib.types.str; - }; - }; - })); - }; - replicationHosts = lib.mkOption { - default = {}; - description = "Hosts to allow replication from"; - type = lib.types.attrsOf (lib.types.submodule { - options = { - ip4 = lib.mkOption { - type = lib.types.listOf lib.types.str; - }; - ip6 = lib.mkOption { - type = lib.types.listOf lib.types.str; - }; - }; - }); - }; - # Output variables - socketsDir = lib.mkOption { - type = lib.types.path; - default = "/run/postgresql"; - description = '' - The directory where Postgresql puts sockets. - ''; - readOnly = true; - }; - }; - }; - - config = lib.mkIf cfg.enable { - networking.firewall.allowedTCPPorts = [ 5432 ]; - - security.acme.certs."postgresql" = config.myServices.databasesCerts // { - user = "postgres"; - group = "postgres"; - domain = "db-1.immae.eu"; - postRun = '' - systemctl reload postgresql.service - ''; - }; - - systemd.services.postgresql.serviceConfig = { - SupplementaryGroups = "keys"; - }; - systemd.services.postgresql.postStart = lib.mkAfter '' - # This line is already defined in 19.09 - PSQL="${pkgs.sudo}/bin/sudo -u postgres psql --port=5432" - - ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (role: _: '' - $PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${role}'" \ - | grep -q 1 \ - || $PSQL -tAc 'CREATE USER "${role}" WITH REPLICATION' - '') cfg.replicationHosts)} - - ${builtins.concatStringsSep "\n" (lib.mapAttrsToList (role: _: - let - sname = builtins.replaceStrings ["-"] ["_"] role; - in - '' - $PSQL -tAc "SELECT 1 FROM pg_replication_slots WHERE slot_name='${sname}'" \ - | grep -q 1 \ - || $PSQL -tAc "SELECT * FROM pg_create_physical_replication_slot('${sname}')" - '') cfg.replicationHosts)} - ''; - - services.postgresql = { - enable = true; - package = cfg.package; - enableTCPIP = true; - extraConfig = '' - max_connections = 100 - wal_level = logical - shared_buffers = 512MB - work_mem = 10MB - max_wal_size = 1GB - min_wal_size = 80MB - log_timezone = 'Europe/Paris' - datestyle = 'iso, mdy' - timezone = 'Europe/Paris' - lc_messages = 'en_US.UTF-8' - lc_monetary = 'en_US.UTF-8' - lc_numeric = 'en_US.UTF-8' - lc_time = 'en_US.UTF-8' - default_text_search_config = 'pg_catalog.english' - # this introduces a small delay before storing on disk, but - # makes it order of magnitudes quicker - synchronous_commit = off - ssl = on - ssl_cert_file = '${config.security.acme.certs.postgresql.directory}/fullchain.pem' - ssl_key_file = '${config.security.acme.certs.postgresql.directory}/key.pem' - ''; - authentication = let - hosts = builtins.concatStringsSep "\n" ( - lib.lists.flatten (lib.mapAttrsToList (k: vs: map (v: - map (ip6: "hostssl ${v.database} ${v.username} ${ip6}/128 ${v.method}") v.ip6 - ++ map (ip4: "hostssl ${v.database} ${v.username} ${ip4}/32 ${v.method}") v.ip4 - ) vs) cfg.authorizedHosts - )); - replication = builtins.concatStringsSep "\n" ( - lib.lists.flatten (lib.mapAttrsToList (k: v: - map (ip6: "hostssl replication ${k} ${ip6}/128 pam pamservice=postgresql_replication") v.ip6 - ++ map (ip4: "hostssl replication ${k} ${ip4}/32 pam pamservice=postgresql_replication") v.ip4 - ) cfg.replicationHosts - )); - in '' - local all postgres ident - local all all md5 - ${hosts} - hostssl all all all pam - ${replication} - ''; - }; - - secrets.keys = { - "postgresql/pam" = { - permissions = "0400"; - group = "postgres"; - user = "postgres"; - text = with cfg.ldapConfig; '' - host ${host} - base ${base} - binddn ${dn} - bindpw ${password} - pam_filter ${filter} - ssl start_tls - ''; - }; - "postgresql/pam_replication" = { - permissions = "0400"; - group = "postgres"; - user = "postgres"; - text = with cfg.replicationLdapConfig; '' - host ${host} - base ${base} - binddn ${dn} - bindpw ${password} - pam_login_attribute cn - ssl start_tls - ''; - }; - }; - - security.pam.services = let - pam_ldap = "${pkgs.pam_ldap}/lib/security/pam_ldap.so"; - in { - postgresql = { - text = '' - auth required ${pam_ldap} config=${config.secrets.fullPaths."postgresql/pam"} - account required ${pam_ldap} config=${config.secrets.fullPaths."postgresql/pam"} - ''; - }; - postgresql_replication = { - text = '' - auth required ${pam_ldap} config=${config.secrets.fullPaths."postgresql/pam_replication"} - account required ${pam_ldap} config=${config.secrets.fullPaths."postgresql/pam_replication"} - ''; - }; - }; - }; -} - diff --git a/modules/private/databases/postgresql_replication.nix b/modules/private/databases/postgresql_replication.nix deleted file mode 100644 index 135bbed..0000000 --- a/modules/private/databases/postgresql_replication.nix +++ /dev/null @@ -1,182 +0,0 @@ -{ pkgs, config, lib, ... }: -let - cfg = config.myServices.databasesReplication.postgresql; -in -{ - options.myServices.databasesReplication.postgresql = { - enable = lib.mkEnableOption "Enable postgresql replication"; - base = lib.mkOption { - type = lib.types.path; - description = '' - Base path to put the replications - ''; - }; - mainPackage = lib.mkOption { - type = lib.types.package; - default = pkgs.postgresql; - description = '' - Postgresql package available in shell - ''; - }; - hosts = lib.mkOption { - default = {}; - description = '' - Hosts to backup - ''; - type = lib.types.attrsOf (lib.types.submodule { - options = { - package = lib.mkOption { - type = lib.types.package; - default = pkgs.postgresql; - description = '' - Postgresql package for this host - ''; - }; - slot = lib.mkOption { - type = lib.types.str; - description = '' - Slot to use for replication - ''; - }; - connection = lib.mkOption { - type = lib.types.str; - description = '' - Connection string to access the psql master - ''; - }; - }; - }); - }; - }; - - config = lib.mkIf cfg.enable { - users.users.postgres = { - name = "postgres"; - uid = config.ids.uids.postgres; - group = "postgres"; - description = "PostgreSQL server user"; - home = "/var/lib/postgresql"; - useDefaultShell = true; - extraGroups = [ "keys" ]; - }; - users.groups.postgres.gid = config.ids.gids.postgres; - environment.systemPackages = [ cfg.mainPackage ]; - - secrets.keys = lib.listToAttrs (lib.flatten (lib.mapAttrsToList (name: hcfg: [ - (lib.nameValuePair "postgresql_replication/${name}/recovery.conf" { - user = "postgres"; - group = "postgres"; - permissions = "0400"; - text = '' - standby_mode = on - primary_conninfo = '${hcfg.connection}?sslmode=require' - primary_slot_name = '${hcfg.slot}' - ''; - }) - (lib.nameValuePair "postgresql_replication/${name}/connection_string" { - user = "postgres"; - group = "postgres"; - permissions = "0400"; - text = hcfg.connection; - }) - (lib.nameValuePair "postgresql_replication/${name}/postgresql.conf" { - user = "postgres"; - group = "postgres"; - permissions = "0400"; - text = let - dataDir = "${cfg.base}/${name}/postgresql"; - in '' - listen_addresses = ''' - unix_socket_directories = '${dataDir}' - data_directory = '${dataDir}' - wal_level = logical - ''; - }) - ]) cfg.hosts)); - - services.cron = { - enable = true; - systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg: - let - dataDir = "${cfg.base}/${name}/postgresql"; - backupDir = "${cfg.base}/${name}/postgresql_backup"; - backup_script = pkgs.writeScript "backup_psql_${name}" '' - #!${pkgs.stdenv.shell} - - set -euo pipefail - - resume_replication() { - ${hcfg.package}/bin/psql -h ${dataDir} -c "SELECT pg_wal_replay_resume();" >/dev/null || echo "impossible to resume replication" - } - - trap resume_replication EXIT - - ${hcfg.package}/bin/psql -h ${dataDir} -c "SELECT pg_wal_replay_pause();" >/dev/null || (echo "impossible to pause replication" && false) - - ${hcfg.package}/bin/pg_dumpall -h ${dataDir} -f ${backupDir}/$(${pkgs.coreutils}/bin/date -Iminutes).sql - ''; - u = pkgs.callPackage ./utils.nix {}; - cleanup_script = pkgs.writeScript "cleanup_postgresql_${name}" (u.keepLastNDumps "sql" backupDir 6); - in [ - "0 22,4,10,16 * * * postgres ${backup_script}" - "0 3 * * * postgres ${cleanup_script}" - ]) cfg.hosts); - }; - - system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg: - lib.attrsets.nameValuePair "psql_replication_${name}" { - deps = [ "users" ]; - text = '' - install -m 0700 -o postgres -g postgres -d ${cfg.base}/${name}/postgresql - install -m 0700 -o postgres -g postgres -d ${cfg.base}/${name}/postgresql_backup - ''; - }) cfg.hosts; - - systemd.services = lib.attrsets.mapAttrs' (name: hcfg: - let - dataDir = "${cfg.base}/${name}/postgresql"; - in - lib.attrsets.nameValuePair "postgresql_backup_${name}" { - description = "Postgresql replication for ${name}"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - - environment.PGDATA = dataDir; - path = [ hcfg.package ]; - - preStart = '' - if ! test -e ${dataDir}/PG_VERSION; then - mkdir -m 0700 -p ${dataDir} - chown -R postgres:postgres ${dataDir} - fi - ''; - script = let - fp = n: config.secrets.fullPaths."postgresql_replication/${name}/${n}"; - in '' - if ! test -e ${dataDir}/PG_VERSION; then - pg_basebackup -d $(cat ${fp "connection_string"}) -D ${dataDir} -S ${hcfg.slot} - fi - ln -sfn ${fp "recovery.conf"} ${dataDir}/recovery.conf - ln -sfn ${fp "postgresql.conf"} ${dataDir}/postgresql.conf - - exec postgres - ''; - - serviceConfig = { - ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; - User = "postgres"; - Group = "postgres"; - PermissionsStartOnly = true; - RuntimeDirectory = "postgresql"; - Type = "notify"; - - KillSignal = "SIGINT"; - KillMode = "mixed"; - # basebackup can take a long time - TimeoutStartSec="infinity"; - TimeoutStopSec = 120; - }; - unitConfig.RequiresMountsFor = dataDir; - }) cfg.hosts; - }; -} diff --git a/modules/private/databases/redis.nix b/modules/private/databases/redis.nix deleted file mode 100644 index 685fa46..0000000 --- a/modules/private/databases/redis.nix +++ /dev/null @@ -1,133 +0,0 @@ -{ lib, config, pkgs, ... }: -let - cfg = config.myServices.databases.redis; -in { - options.myServices.databases.redis = { - enable = lib.mkOption { - default = false; - example = true; - description = "Whether to enable redis database"; - type = lib.types.bool; - }; - socketsDir = lib.mkOption { - type = lib.types.path; - default = "/run/redis"; - description = '' - The directory where Redis puts sockets. - ''; - }; - # Output variables - sockets = lib.mkOption { - type = lib.types.attrsOf lib.types.path; - default = { - redis = "${cfg.socketsDir}/redis.sock"; - }; - readOnly = true; - description = '' - Redis sockets - ''; - }; - }; - - config = lib.mkIf cfg.enable { - users.users.redis.uid = config.ids.uids.redis; - users.groups.redis.gid = config.ids.gids.redis; - services.redis = rec { - enable = true; - bind = "127.0.0.1"; - unixSocket = cfg.sockets.redis; - extraConfig = '' - unixsocketperm 777 - maxclients 1024 - ''; - }; - systemd.services.redis.serviceConfig.Slice = "redis.slice"; - - services.spiped = { - enable = true; - config.redis = { - decrypt = true; - source = "0.0.0.0:16379"; - target = "/run/redis/redis.sock"; - keyfile = config.secrets.fullPaths."redis/spiped_keyfile"; - }; - }; - systemd.services.spiped_redis = { - description = "Secure pipe 'redis'"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - - serviceConfig = { - Slice = "redis.slice"; - Restart = "always"; - User = "spiped"; - PermissionsStartOnly = true; - SupplementaryGroups = "keys"; - }; - - script = "exec ${pkgs.spiped}/bin/spiped -F `cat /etc/spiped/redis.spec`"; - }; - - services.filesWatcher.predixy = { - restart = true; - paths = [ config.secrets.fullPaths."redis/predixy.conf" ]; - }; - - networking.firewall.allowedTCPPorts = [ 7617 16379 ]; - secrets.keys = { - "redis/predixy.conf" = { - user = "redis"; - group = "redis"; - permissions = "0400"; - text = '' - Name Predixy - Bind 127.0.0.1:7617 - ClientTimeout 300 - WorkerThreads 1 - - Authority { - Auth "${config.myEnv.databases.redis.predixy.read}" { - Mode read - } - } - - StandaloneServerPool { - Databases 16 - RefreshMethod fixed - Group shard001 { - + ${config.myEnv.databases.redis.socket} - } - } - ''; - }; - "redis/spiped_keyfile" = { - user = "spiped"; - group = "spiped"; - permissions = "0400"; - text = config.myEnv.databases.redis.spiped_key; - }; - }; - - systemd.slices.redis = { - description = "Redis slice"; - }; - - systemd.services.predixy = { - description = "Redis proxy"; - wantedBy = [ "multi-user.target" ]; - after = [ "redis.service" ]; - - serviceConfig = { - Slice = "redis.slice"; - User = "redis"; - Group = "redis"; - SupplementaryGroups = "keys"; - Type = "simple"; - - ExecStart = "${pkgs.predixy}/bin/predixy ${config.secrets.fullPaths."redis/predixy.conf"}"; - }; - - }; - }; -} - diff --git a/modules/private/databases/redis_replication.nix b/modules/private/databases/redis_replication.nix deleted file mode 100644 index 9e48939..0000000 --- a/modules/private/databases/redis_replication.nix +++ /dev/null @@ -1,171 +0,0 @@ -{ pkgs, config, lib, ... }: -let - cfg = config.myServices.databasesReplication.redis; -in -{ - options.myServices.databasesReplication.redis = { - enable = lib.mkEnableOption "Enable redis replication"; - base = lib.mkOption { - type = lib.types.path; - description = '' - Base path to put the replications - ''; - }; - hosts = lib.mkOption { - default = {}; - description = '' - Hosts to backup - ''; - type = lib.types.attrsOf (lib.types.submodule { - options = { - package = lib.mkOption { - type = lib.types.package; - default = pkgs.redis; - description = '' - Redis package for this host - ''; - }; - host = lib.mkOption { - type = lib.types.str; - description = '' - Host to connect to - ''; - }; - port = lib.mkOption { - type = lib.types.str; - description = '' - Port to connect to - ''; - }; - password = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = '' - Password to use - ''; - }; - }; - }); - }; - }; - - config = lib.mkIf cfg.enable { - users.users.redis = { - description = "Redis database user"; - group = "redis"; - uid = config.ids.uids.redis; - extraGroups = [ "keys" ]; - }; - users.groups.redis.gid = config.ids.gids.redis; - - services.spiped = { # sync from eldiron - enable = true; - config.redis = { - encrypt = true; - source = "127.0.0.1:16379"; - target = "${config.myEnv.servers.eldiron.ips.main.ip4}:16379"; - keyfile = config.secrets.fullPaths."redis/spiped_eldiron_keyfile"; - }; - }; - - secrets.keys = lib.mapAttrs' (name: hcfg: - lib.nameValuePair "redis_replication/${name}/config" { - user = "redis"; - group = "redis"; - permissions = "0400"; - text = '' - pidfile ${cfg.base}/${name}/redis/redis.pid - port 0 - unixsocket /run/redis_${name}/redis.sock - loglevel notice - logfile /dev/null - syslog-enabled yes - databases 16 - save 900 1 - save 300 10 - save 60 10000 - dbfilename dump.rdb - dir ${cfg.base}/${name}/redis/ - slaveof ${hcfg.host} ${hcfg.port} - ${if hcfg.password != null then "masterauth ${hcfg.password}" else ""} - appendOnly no - appendfsync everysec - slowlog-log-slower-than 10000 - slowlog-max-len 128 - unixsocketperm 777 - maxclients 1024 - ''; - } - ) cfg.hosts // { - "redis/spiped_eldiron_keyfile" = { # For eldiron only - user = "spiped"; - group = "spiped"; - permissions = "0400"; - text = config.myEnv.databases.redis.spiped_key; - }; - }; - - services.cron = { - enable = true; - systemCronJobs = lib.flatten (lib.mapAttrsToList (name: hcfg: - let - dataDir = "${cfg.base}/${name}/redis"; - backupDir = "${cfg.base}/${name}/redis_backup"; - backup_script = pkgs.writeScript "backup_redis_${name}" '' - #!${pkgs.stdenv.shell} - - ${pkgs.coreutils}/bin/cp ${cfg.base}/${name}/redis/dump.rdb \ - ${backupDir}/$(${pkgs.coreutils}/bin/date -Iminutes).rdb - ''; - u = pkgs.callPackage ./utils.nix {}; - cleanup_script = pkgs.writeScript "cleanup_redis_${name}" (u.exponentialDumps "rdb" backupDir); - in [ - "0 22,4,10,16 * * * root ${backup_script}" - "0 3 * * * root ${cleanup_script}" - ]) cfg.hosts); - }; - - system.activationScripts = lib.attrsets.mapAttrs' (name: hcfg: - lib.attrsets.nameValuePair "redis_replication_${name}" { - deps = [ "users" "groups" ]; - text = '' - install -m 0700 -o redis -g redis -d ${cfg.base}/${name}/redis - install -m 0700 -o redis -g redis -d ${cfg.base}/${name}/redis_backup - ''; - }) cfg.hosts; - - systemd.services = { - spiped_redis = { # For eldiron - description = "Secure pipe 'redis'"; - after = [ "network.target" ]; - wantedBy = [ "multi-user.target" ]; - - serviceConfig = { - Restart = "always"; - User = "spiped"; - PermissionsStartOnly = true; - SupplementaryGroups = "keys"; - }; - - script = "exec ${pkgs.spiped}/bin/spiped -F `cat /etc/spiped/redis.spec`"; - }; - } // lib.attrsets.mapAttrs' (name: hcfg: - let - dataDir = "${cfg.base}/${name}/redis"; - in - lib.attrsets.nameValuePair "redis_backup_${name}" { - description = "Redis replication for ${name}"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - unitConfig.RequiresMountsFor = dataDir; - - serviceConfig = { - ExecStart = "${hcfg.package}/bin/redis-server ${config.secrets.fullPaths."redis_replication/${name}/config"}"; - User = "redis"; - RuntimeDirectory = "redis_${name}"; - }; - }) cfg.hosts; - }; -} - - diff --git a/modules/private/databases/utils.nix b/modules/private/databases/utils.nix deleted file mode 100644 index 47988fc..0000000 --- a/modules/private/databases/utils.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ pkgs }: -{ - keepLastNDumps = ext: backupDir: n: '' - #!${pkgs.stdenv.shell} - - cd ${backupDir} - ${pkgs.coreutils}/bin/rm -f \ - $(${pkgs.coreutils}/bin/ls -1 *.${ext} \ - | ${pkgs.coreutils}/bin/sort -r \ - | ${pkgs.gnused}/bin/sed -e '1,${builtins.toString n}d') - ''; - exponentialDumps = ext: backupDir: let - log2rotateSrc = builtins.fetchGit { - url = "https://github.com/avian2/pylog2rotate"; - ref = "master"; - rev = "061f0564757289d3bea553b16f8fd5c4a0319c5e"; - }; - log2rotate = pkgs.writeScript "log2rotate" '' - #!${pkgs.python3}/bin/python - - ${builtins.readFile "${log2rotateSrc}/log2rotate.py"} - ''; - in '' - #!${pkgs.stdenv.shell} - - cd ${backupDir} - ${pkgs.coreutils}/bin/rm -f $(ls -1 *.${ext} | grep -v 'T22:' | sort -r | sed -e '1,12d') - ${pkgs.coreutils}/bin/rm -f $(ls -1 *T22*.${ext} | ${log2rotate} --skip 7 --fuzz 7 --delete --format='%Y-%m-%dT%H:%M+00:00.${ext}') - ''; -} -- cgit v1.2.3