Add monitoring for dilion and quatresaisons

[perso/Immae/Config/Nix.git] / modules / private / monitoring / default.nix

{ config, pkgs, lib, name, nodes, ... }:
let
  cfg = config.myServices.monitoring;
  send_mails = pkgs.runCommand "send_mails" {
    buildInputs = [ pkgs.makeWrapper ];
  } ''
    mkdir -p $out/bin
    cp ${./send_mails} $out/bin/send_mails
    patchShebangs $out
    wrapProgram $out/bin/send_mails --prefix PATH : ${lib.makeBinPath [
      pkgs.mailutils
    ]}
    '';
  postgresqlBinary = if config.myServices.databasesReplication.postgresql.enable
    then config.myServices.databasesReplication.postgresql.mainPackage
    else if config.myServices.databases.enable
    then config.myServices.databases.postgresql.package
    else pkgs.postgresql;
  zfsPlugin = pkgs.fetchurl {
    url = "https://www.claudiokuenzler.com/monitoring-plugins/check_zpools.sh";
    sha256 = "0p9ms9340in80jkds4kfspw62xnzsv5s7ni9m28kxyd0bnzkbzhf";
  };
  myplugins = pkgs.runCommand "buildplugins" {
    buildInputs = [ pkgs.makeWrapper pkgs.perl ];
  } ''
    mkdir $out
    cp ${zfsPlugin} $out/check_zpool.sh && chmod +x $out/check_zpool.sh
    cp ${./plugins}/* $out/
    patchShebangs $out
    wrapProgram $out/check_command --prefix PATH : ${config.security.wrapperDir}
    wrapProgram $out/check_zpool.sh --prefix PATH : ${lib.makeBinPath [
      pkgs.which pkgs.zfs pkgs.gawk
    ]}
    wrapProgram $out/send_nrdp.sh --prefix PATH : ${lib.makeBinPath [
      pkgs.curl pkgs.jq
    ]}
    wrapProgram $out/check_mem.sh --prefix PATH : ${lib.makeBinPath [
      pkgs.gnugrep pkgs.gawk pkgs.procps-ng
    ]}
    wrapProgram $out/check_postgres_replication --prefix PATH : ${lib.makeBinPath [
      postgresqlBinary
    ]}
    wrapProgram $out/check_redis_replication --prefix PATH : ${lib.makeBinPath [
      pkgs.gnugrep pkgs.coreutils pkgs.redis
    ]}
    wrapProgram $out/check_mysql_replication --prefix PATH : ${lib.makeBinPath [
      pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.mariadb
    ]}
    wrapProgram $out/check_openldap_replication --prefix PATH : ${lib.makeBinPath [
      pkgs.gnugrep pkgs.gnused pkgs.coreutils pkgs.openldap
    ]}
    wrapProgram $out/check_emails --prefix PATH : ${lib.makeBinPath [
      pkgs.openssh send_mails
    ]} --prefix PERL5LIB : ${pkgs.perlPackages.makePerlPath [
      pkgs.perlPackages.TimeDate
    ]}
    wrapProgram $out/check_ftp_database --prefix PATH : ${lib.makeBinPath [
      pkgs.lftp
    ]}
    wrapProgram $out/check_git --prefix PATH : ${lib.makeBinPath [
      pkgs.git pkgs.openssh
    ]}
    wrapProgram $out/check_imap_connection --prefix PATH : ${lib.makeBinPath [
      pkgs.openssl
    ]}
    wrapProgram $out/check_eriomem --prefix PATH : ${lib.makeBinPath [
      pkgs.s3cmd pkgs.python3
    ]}
    makeWrapper $out/check_backup_age $out/check_backup_eriomem_age --prefix PATH : ${lib.makeBinPath [
      pkgs.duplicity
    ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."eriomem_access_key"}
    makeWrapper $out/check_backup_age $out/check_backup_ovh_age --prefix PATH : ${lib.makeBinPath [
      pkgs.duplicity
    ]} --set SECRETS_PATH ${lib.optionalString cfg.master config.secrets.fullPaths."ovh_access_key"}
    wrapProgram $out/notify_by_email --prefix PATH : ${lib.makeBinPath [
      pkgs.mailutils
    ]}
    wrapProgram $out/notify_by_slack --prefix PATH : ${lib.makeBinPath [
      pkgs.curl pkgs.jq
    ]}
    wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
      (pkgs.python3.withPackages (ps: [ps.ovh]))
    ]}
    wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
      pkgs.iproute pkgs.bc
    ]}
    '';
  toObjects = pkgs.callPackage ./to_objects.nix {};
  commonConfig = {
    dilion = {
      processWarn = "250"; processAlert = "400";
      loadWarn = "8.0"; loadAlert = "10.0";
      interface = "eth0";
    };
    eldiron = {
      processWarn = "250"; processAlert = "400";
      loadWarn = "8.0"; loadAlert = "10.0";
      interface = "eth0";
    };
    backup-2 = {
      processWarn = "60"; processAlert = "70";
      loadWarn = "1.0"; loadAlert = "2.0";
      interface = "ens3";
    };
    monitoring-1 = {
      processWarn = "50"; processAlert = "60";
      loadWarn = "4.0"; loadAlert = "6.0";
      load15Warn = "1.0"; load15Alert = "2.0";
      interface = "ens3";
    };
    quatresaisons = {
      processWarn = "250"; processAlert = "400";
      loadWarn = "8.0"; loadAlert = "10.0";
      interface = "eth0";
    };
  };
  externalObjects = lib.genAttrs [ "tiboqorl-fr" ]
    (n: pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; });
  masterPassiveObjects = let
    passiveNodes = lib.attrsets.filterAttrs (n: _: builtins.elem n ["backup-2" "eldiron" "quatresaisons" "dilion"]) nodes;
    toPassiveServices = map (s: s.passiveInfo.filter s // s.passiveInfo);
    passiveServices = lib.flatten (lib.attrsets.mapAttrsToList
      (_: n: toPassiveServices n.config.myServices.monitoring.services)
      passiveNodes
      ) ++ lib.flatten (lib.attrsets.mapAttrsToList
      (_: n: toPassiveServices n.service)
      externalObjects);
  in {
    service = passiveServices;
    host = lib.lists.foldr
      (a: b: a//b)
      {}
      (lib.attrsets.mapAttrsToList (_: h: h.config.myServices.monitoring.hosts) passiveNodes
      ++ lib.attrsets.mapAttrsToList (_: n: n.host) externalObjects);
  };
  emailCheck = host: hostFQDN: let
    allCfg = config.myEnv.monitoring.email_check;
    cfg = allCfg."${host}";
    reverseTargets = builtins.attrNames (lib.attrsets.filterAttrs (k: v: builtins.elem host v.targets) allCfg);
    to_email = cfg': host':
      let sep = if lib.strings.hasInfix "+" cfg'.mail_address then "_" else "+";
      in "${cfg'.mail_address}${sep}${host'}@${cfg'.mail_domain}";
    mails_to_send = builtins.concatStringsSep "," (map (n: to_email allCfg."${n}" host) cfg.targets);
    mails_to_receive = builtins.concatStringsSep "," (map (n: "${to_email cfg n}:${n}") reverseTargets);
    command = if cfg.local
    then
      [ "check_emails_local" "/var/lib/naemon/checks/email" mails_to_send mails_to_receive ]
    else
      [ "check_emails" cfg.login cfg.port mails_to_send mails_to_receive ];
  in
    {
      service_description = "${hostFQDN} email service is active";
      use = "mail-service";
      host_name = hostFQDN;
      servicegroups = "webstatus-email";
      check_command = command;
    };
  otherObjects = map
    (n: (pkgs.callPackage (./. + "/objects_" + n + ".nix") { inherit emailCheck; }))
    [ "ulminfo-fr" "phare" ];
  masterObjects = pkgs.callPackage ./objects_master.nix { inherit config; };
  commonObjects = pkgs.callPackage ./objects_common.nix ({
    master = cfg.master;
    hostFQDN = config.hostEnv.fqdn;
    hostName = name;
    sudo = "/run/wrappers/bin/sudo";
  } // builtins.getAttr name commonConfig);
  hostObjects =
    let
      specific_file = ./. + "/objects_" + name + ".nix";
    in
      lib.attrsets.optionalAttrs
        (builtins.pathExists specific_file)
        (pkgs.callPackage specific_file {
          inherit config nodes emailCheck;
          hostFQDN = config.hostEnv.fqdn;
          hostName = name;
        });
  objectsFiles = lib.mapAttrs' (name: value: lib.nameValuePair
    "=/${name}/objects.conf" { alias = pkgs.writeText "objects.conf" (toObjects value); }
  ) externalObjects;
in
{
  options = {
    myServices.monitoring = {
      enable = lib.mkOption {
        type = lib.types.bool;
        default = false;
        description = ''
          Whether to enable monitoring.
        '';
      };
      master = lib.mkOption {
        type = lib.types.bool;
        default = false;
        description = ''
          This instance is the master instance
        '';
      };
      hosts = lib.mkOption {
        readOnly = true;
        description = "Hosts list for this host";
        default = (commonObjects.host or {}) // (hostObjects.host or {});
      };
      services = lib.mkOption {
        readOnly = true;
        description = "Services list for this host";
        default = commonObjects.service ++ hostObjects.service;
      };
    };
  };

  config = lib.mkIf cfg.enable {
    services.nginx = lib.mkIf config.myServices.status.enable {
      virtualHosts."status.immae.eu".locations = objectsFiles;
    };

    services.duplyBackup.profiles.monitoring = {
      rootDir = config.services.naemon.varDir;
    };
    security.sudo.extraRules = [
      {
        commands = [
          { command = "${pkgs.mdadm}/bin/mdadm --monitor --scan -1"; options = [ "NOPASSWD" ]; }
          { command = "${pkgs.postfix}/bin/mailq"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "root";
      }
      {
        commands = [
          { command = "${myplugins}/check_last_file_date /backup2/*"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "ALL";
      }
      {
        commands = [
          { command = "${myplugins}/check_postgres_replication *"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "postgres";
      }
      {
        commands = [
          { command = "${myplugins}/check_mysql_replication *"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "mysql";
      }
      {
        commands = [
          { command = "${myplugins}/check_openldap_replication *"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "openldap";
      }
      {
        commands = [
          { command = "${myplugins}/check_redis_replication *"; options = [ "NOPASSWD" ]; }
        ];
        users = [ "naemon" ];
        runAs = "redis";
      }
    ];
    environment.etc."mdadm.conf" = {
      enable = true;
      mode = "0644";
      user = "root";
      text = "MAILADDR ${config.myEnv.monitoring.email}";
    };

    secrets.keys = [
      {
        dest = "naemon/id_rsa";
        user = "naemon";
        group = "naemon";
        permissions = "0400";
        text = config.myEnv.monitoring.ssh_secret_key;
      }
    ] ++ lib.optionals cfg.master (
      lib.mapAttrsToList (k: v:
      {
        dest = "${k}_access_key";
        user = "naemon";
        group = "naemon";
        permissions = "0400";
        text = ''
          export AWS_ACCESS_KEY_ID="${v.accessKeyId}"
          export AWS_SECRET_ACCESS_KEY="${v.secretAccessKey}"
          export BASE_URL="${v.remote "immae-eldiron"}"
        '';
      }) config.myEnv.backup.remotes
    );
    # needed since extraResource is not in the closure
    systemd.services.naemon.path = [ myplugins ];
    services.naemon = {
      enable = true;
      extraConfig = ''
        use_syslog=1
        log_initial_states=1
        date_format=iso8601
        admin_email=${config.myEnv.monitoring.email}
      '' + lib.optionalString (!cfg.master) ''
        obsess_over_services=1
        ocsp_command=notify-master
      '' + lib.optionalString (cfg.master) ''
        broker_module=${pkgs.naemon-livestatus}/lib/naemon-livestatus/livestatus.so ${config.services.naemon.runDir}/live
        broker_module=${pkgs.status_engine.module}/lib/status-engine/naemon/statusengine-${pkgs.naemon.status_engine_version}.o use_service_perfdata=1 use_process_data=0 use_system_command_data=0 use_external_command_data=0 use_flapping_data=0 use_program_status_data=0 use_notification_data=0 use_contact_status_data=0 use_contact_notification_data=0 use_event_handler_data=0 use_object_data=0
      '';
      extraResource = ''
        $USER2$=${myplugins}
        $USER200$=${config.myEnv.monitoring.status_url}
        $USER201$=${config.myEnv.monitoring.status_token}
        $USER202$=${config.myEnv.monitoring.http_user_password}
        $USER203$=${config.secrets.fullPaths."naemon/id_rsa"}
        $USER204$=${config.myEnv.monitoring.imap_login}
        $USER205$=${config.myEnv.monitoring.imap_password}
        $USER206$=${config.myEnv.monitoring.slack_channel}
        $USER207$=${config.myEnv.monitoring.slack_url}
        $USER208$=${builtins.concatStringsSep "," (map (builtins.concatStringsSep ":") config.myEnv.monitoring.eriomem_keys)}
        $USER209$=${builtins.concatStringsSep "," [
          config.myEnv.monitoring.ovh_sms.endpoint
          config.myEnv.monitoring.ovh_sms.application_key
          config.myEnv.monitoring.ovh_sms.application_secret
          config.myEnv.monitoring.ovh_sms.consumer_key
          config.myEnv.monitoring.ovh_sms.account
        ]}
      '';
      objectDefs = toObjects commonObjects
        + toObjects hostObjects
        + lib.optionalString cfg.master (toObjects masterObjects)
        + lib.optionalString cfg.master (toObjects masterPassiveObjects)
        + lib.optionalString cfg.master (builtins.concatStringsSep "\n" (map toObjects otherObjects));
    };
  };
}