Merge branch 'monitoring' into dev

15 files changed, 411 insertions, 1 deletions

diff --git a/modules/profile/manifests/fstab.pp b/modules/profile/manifests/fstab.pp
index 8ecfe72..3af316e 100644
--- a/modules/profile/manifests/fstab.pp
+++ b/modules/profile/manifests/fstab.pp
@@ -15,6 +15,12 @@ class profile::fstab (
         device => "UUID=${infos[1]}",
         fstype => $infos[2]
       }
+
+      @profile::monitoring::local_service { "Size on ${infos[0]} partition":
+        local => {
+          check_command => "check_local_disk!10%!5%!${infos[0]}",
+        };
+      }
     }
   }
 }
diff --git a/modules/profile/manifests/monitoring.pp b/modules/profile/manifests/monitoring.pp
new file mode 100644
index 0000000..8633626
--- /dev/null
+++ b/modules/profile/manifests/monitoring.pp
@@ -0,0 +1,58 @@
+class profile::monitoring (
+  Optional[String] $naemon_url   = undef,
+  Optional[String] $naemon_token = undef,
+) inherits profile::monitoring::params {
+  ensure_packages(["naemon", "cnagios"])
+
+  file { "/etc/naemon":
+    ensure  => "directory",
+    recurse => true,
+    purge   => true,
+    force   => true,
+    require => Package["naemon"],
+  }
+  ->
+  file { "/etc/naemon/resource.cfg":
+    ensure  => "file",
+    owner   => "naemon",
+    group   => "naemon",
+    mode    => "0600",
+    content => template("profile/monitoring/resource.cfg.erb"),
+  }
+  ->
+  file { "/etc/naemon/naemon.cfg":
+    ensure  => "file",
+    owner   => "naemon",
+    group   => "naemon",
+    mode    => "0644",
+    content => template("profile/monitoring/naemon.cfg.erb"),
+  }
+  ->
+  file { $objects:
+    ensure => "file",
+    owner  => "naemon",
+    group  => "naemon",
+    mode   => "0600"
+  }
+  ->
+  service { "naemon":
+    ensure => "running",
+    enable => true,
+  }
+
+  unless ($naemon_url == undef or empty($naemon_url)) {
+    file { "/etc/naemon/send_nrdp.sh":
+      ensure  => "file",
+      owner   => "naemon",
+      group   => "naemon",
+      mode    => "0700",
+      content => template("profile/monitoring/send_nrdp.sh.erb"),
+    }
+  }
+
+  include "profile::monitoring::hosts"
+  include "profile::monitoring::services"
+  include "profile::monitoring::commands"
+  include "profile::monitoring::times"
+  include "profile::monitoring::contacts"
+}
diff --git a/modules/profile/manifests/monitoring/commands.pp b/modules/profile/manifests/monitoring/commands.pp
new file mode 100644
index 0000000..1c8d0b4
--- /dev/null
+++ b/modules/profile/manifests/monitoring/commands.pp
@@ -0,0 +1,70 @@
+class profile::monitoring::commands inherits profile::monitoring {
+  ensure_packages(["monitoring-plugins"])
+
+  file { $plugins:
+    ensure => "directory",
+    owner  => "root",
+    group  => "naemon",
+    mode   => "0755",
+  }
+
+  [
+    "check_command",
+    "check_md_raid",
+    "check_postgres_replication",
+    "check_last_file_date",
+  ].each |$file| {
+    file { "$plugins/$file":
+      ensure => "present",
+      owner  => "root",
+      group  => "naemon",
+      mode   => "0755",
+      source => "puppet:///modules/profile/monitoring/$file",
+    }
+  }
+
+  Nagios_command {
+    ensure => "present",
+    owner  => "naemon",
+    group  => "naemon",
+    target => $objects,
+    notify => Service["naemon"],
+    before => Service["naemon"],
+    require => File["/etc/naemon"],
+  }
+
+  nagios_command {
+    "check-host-alive":
+      command_line => '$USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 5';
+    "check_local_disk":
+      command_line => '$USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$';
+    "check_local_procs":
+      command_line => '$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$';
+    "check_local_load":
+      command_line => '$USER1$/check_load -w $ARG1$ -c $ARG2$';
+    "check_local_swap":
+      command_line => '$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$';
+    "check_ntp":
+      command_line => '$USER1$/check_ntp_time -H 0.arch.pool.ntp.org';
+    "check_md_raid":
+      command_line => '$USER2$/check_md_raid',
+      require      => File["$plugins/check_md_raid"];
+    "check_command_output":
+      command_line => '$USER2$/check_command -c "$ARG1$" -o "$ARG2$" $ARG3$',
+      require      => File["$plugins/check_command"];
+    "check_postgresql_replication":
+      command_line => '/usr/bin/sudo -u postgres $USER2$/check_postgres_replication "$ARG1$" "$ARG2$" "$ARG3$"',
+      require      => File["$plugins/check_postgres_replication"];
+    "check_last_file_date":
+      command_line => '$USER2$/check_last_file_date "$ARG1$" "$ARG2$" "$ARG3$"',
+      require      => File["$plugins/check_last_file_date"],
+  }
+
+  unless empty($naemon_url) {
+    nagios_command {
+      "notify-master":
+        command_line => '/etc/naemon/send_nrdp.sh -H "$HOSTADDRESS$" -s "$SERVICEDESC$" -S "$SERVICESTATEID$" -o "$SERVICEOUTPUT$"',
+        require      => File["/etc/naemon/send_nrdp.sh"];
+    }
+  }
+}
diff --git a/modules/profile/manifests/monitoring/contacts.pp b/modules/profile/manifests/monitoring/contacts.pp
new file mode 100644
index 0000000..a751153
--- /dev/null
+++ b/modules/profile/manifests/monitoring/contacts.pp
@@ -0,0 +1,16 @@
+class profile::monitoring::contacts inherits profile::monitoring::params {
+  Nagios_contactgroup {
+    ensure  => "present",
+    owner   => "naemon",
+    group   => "naemon",
+    target  => $objects,
+    notify  => Service["naemon"],
+    before  => Service["naemon"],
+    require => File["/etc/naemon"],
+  }
+
+  nagios_contactgroup { "admins":
+    alias => "System administrators",    
+  }
+
+}
diff --git a/modules/profile/manifests/monitoring/external_service.pp b/modules/profile/manifests/monitoring/external_service.pp
new file mode 100644
index 0000000..027dad8
--- /dev/null
+++ b/modules/profile/manifests/monitoring/external_service.pp
@@ -0,0 +1,16 @@
+define profile::monitoring::external_service (
+  Optional[String] $type   = undef,
+  Optional[Hash]   $master = {},
+) {
+  include profile::monitoring::params
+  $service_description = $title
+
+  nagios_service { $service_description:
+    service_description => $service_description,
+    host_name           => $::profile::monitoring::params::service_local["host_name"],
+    use                 => $::profile::monitoring::params::service_types[$type],
+    target              => $::profile::monitoring::params::services_for_master,
+    *                   => $master,
+  }
+
+}
diff --git a/modules/profile/manifests/monitoring/hosts.pp b/modules/profile/manifests/monitoring/hosts.pp
new file mode 100644
index 0000000..f7802be
--- /dev/null
+++ b/modules/profile/manifests/monitoring/hosts.pp
@@ -0,0 +1,45 @@
+class profile::monitoring::hosts inherits profile::monitoring::params {
+  $real_hostname = lookup("base_installation::real_hostname")
+
+  Nagios_hostgroup {
+    ensure  => "present",
+    owner   => "naemon",
+    group   => "naemon",
+    target  => $objects,
+    notify  => Service["naemon"],
+    before  => Service["naemon"],
+    require => File["/etc/naemon"],
+  }
+
+  Nagios_host {
+    ensure  => "present",
+    owner   => "naemon",
+    group   => "naemon",
+    target  => $objects,
+    notify  => Service["naemon"],
+    before  => Service["naemon"],
+    require => File["/etc/naemon"],
+  }
+
+  nagios_hostgroup { "linux-servers":
+    alias   => "Linux Servers",
+    members => [$real_hostname],
+  }
+
+  $host_linux_server = {
+    check_command         => "check-host-alive",
+    check_interval        => 5,
+    check_period          => "24x7",
+    contact_groups        => "admins",
+    max_check_attempts    => "10",
+    notification_interval => "120",
+    notification_options  => "d,u,r",
+    retry_interval        => "1",
+  }
+
+  nagios_host { $real_hostname:
+    address => $real_hostname;
+  default: * => $host_linux_server,
+  }
+
+}
diff --git a/modules/profile/manifests/monitoring/local_service.pp b/modules/profile/manifests/monitoring/local_service.pp
new file mode 100644
index 0000000..1f975f0
--- /dev/null
+++ b/modules/profile/manifests/monitoring/local_service.pp
@@ -0,0 +1,56 @@
+define profile::monitoring::local_service (
+  Optional[Hash] $sudos  = {},
+  Optional[Hash] $common = {},
+  Optional[Hash] $master = {},
+  Optional[Hash] $local  = {},
+) {
+  include profile::monitoring::params
+
+  $service_description = $title
+
+  Nagios_service {
+    ensure  => "present",
+    owner   => "naemon",
+    group   => "naemon",
+    notify  => Service["naemon"],
+    before  => Service["naemon"],
+    require => File["/etc/naemon"],
+  }
+
+  $sudos.each |$sudo_name, $content| {
+    ensure_resource("sudo::conf", $sudo_name, {
+      content => $content,
+      before  => Nagios_service[$service_description],
+    })
+  }
+
+  [true, false].each |$services_for_master| {
+    if $services_for_master {
+      $default_local = {
+        mode           => "0644",
+        target         => $::profile::monitoring::params::services_for_master,
+        check_interval => $::profile::monitoring::params::service_local["check_interval"],
+        retry_interval => $::profile::monitoring::params::service_local["retry_interval"],
+        host_name      => $::profile::monitoring::params::service_local["host_name"],
+        use            => $::profile::monitoring::params::service_types["passive"],
+        notify         => [],
+      }
+      $t = "master - "
+      $services_key = $master
+    } else {
+      $default_local = merge(
+        $::profile::monitoring::params::service_local,
+        { target => $::profile::monitoring::params::objects }
+      )
+      $t = ""
+      $services_key = $local
+    }
+
+    $hash = merge($default_local, $common, $services_key)
+
+    nagios_service { "$t$service_description":
+      service_description => $service_description,
+      *                   => $hash
+    }
+  }
+}
diff --git a/modules/profile/manifests/monitoring/params.pp b/modules/profile/manifests/monitoring/params.pp
new file mode 100644
index 0000000..27f895e
--- /dev/null
+++ b/modules/profile/manifests/monitoring/params.pp
@@ -0,0 +1,42 @@
+class profile::monitoring::params {
+  $real_hostname = lookup("base_installation::real_hostname")
+
+  $services_for_master = "/etc/naemon/services_for_master.cfg"
+  $objects             = "/etc/naemon/objects.cfg"
+  $plugins             = "/etc/naemon/monitoring-plugins"
+
+  $service_types = {
+    "passive" => "external-passive-service",
+    "web"     => "external-web-service",
+  }
+
+  $service_generic = {
+    active_checks_enabled        => "1",
+    check_freshness              => "0",
+    check_interval               => "10",
+    check_period                 => "24x7",
+    contact_groups               => "admins",
+    event_handler_enabled        => "1",
+    flap_detection_enabled       => "1",
+    is_volatile                  => "0",
+    max_check_attempts           => "3",
+    notification_interval        => "60",
+    notification_options         => "w,u,c,r",
+    notification_period          => "24x7",
+    notifications_enabled        => "0",
+    obsess_over_service          => "1",
+    passive_checks_enabled       => "1",
+    process_perf_data            => "1",
+    retain_nonstatus_information => "1",
+    retain_status_information    => "1",
+    retry_interval               => "2",
+  }
+
+  $service_local = merge($service_generic, {
+    host_name          => $real_hostname,
+    check_interval     => "5",
+    max_check_attempts => "4",
+    retry_interval     => "1",
+    })
+
+}
diff --git a/modules/profile/manifests/monitoring/services.pp b/modules/profile/manifests/monitoring/services.pp
new file mode 100644
index 0000000..95c6efb
--- /dev/null
+++ b/modules/profile/manifests/monitoring/services.pp
@@ -0,0 +1,42 @@
+class profile::monitoring::services {
+
+  profile::monitoring::local_service {
+    "Size on root partition":
+      local => {
+        check_command => "check_local_disk!20%!10%!/",
+      };
+    "Total number of process":
+      local => {
+        check_command => "check_local_procs!50!100!RSZDT",
+      };
+    "Average load":
+      local => {
+        check_command => "check_local_load!8.0,8.0,8.0!10.0,10.0,10.0",
+      };
+    "Swap usage":
+      local => {
+        check_command => "check_local_swap!20!10",
+      };
+    "fail2ban is active":
+      sudos   => {
+        "naemon-fail2ban" => "naemon  ALL=(root) NOPASSWD: /usr/bin/fail2ban-client ping",
+      },
+      local   => {
+        check_command => "check_command_output!fail2ban-client ping!pong!-r root",
+      };
+    "NTP is activated and working":
+      local => {
+        check_command => "check_ntp",
+      };
+    "No mdadm array is degraded":
+      common => {
+        ensure => (find_file("/proc/mdstat") == undef) ? { true => "absent", default =>"present" },
+      },
+      local => {
+        check_command => "check_md_raid",
+      };
+  }
+
+  Profile::Monitoring::Local_service <| |>
+  Profile::Monitoring::External_service <| |>
+}
diff --git a/modules/profile/manifests/monitoring/times.pp b/modules/profile/manifests/monitoring/times.pp
new file mode 100644
index 0000000..42f5d9c
--- /dev/null
+++ b/modules/profile/manifests/monitoring/times.pp
@@ -0,0 +1,23 @@
+class profile::monitoring::times inherits profile::monitoring::params {
+  Nagios_timeperiod {
+    ensure  => "present",
+    owner   => "naemon",
+    group   => "naemon",
+    target  => $objects,
+    notify  => Service["naemon"],
+    before  => Service["naemon"],
+    require => File["/etc/naemon"],
+  }
+
+  nagios_timeperiod { "24x7":
+    alias     => "24 Hours A Day, 7 Days A Week",
+    monday    => "00:00-24:00",
+    tuesday   => "00:00-24:00",
+    wednesday => "00:00-24:00",
+    thursday  => "00:00-24:00",
+    friday    => "00:00-24:00",
+    saturday  => "00:00-24:00",
+    sunday    => "00:00-24:00",
+  }
+
+}
diff --git a/modules/profile/manifests/postgresql.pp b/modules/profile/manifests/postgresql.pp
index 97ce572..fedbcb1 100644
--- a/modules/profile/manifests/postgresql.pp
+++ b/modules/profile/manifests/postgresql.pp
@@ -28,5 +28,13 @@ class profile::postgresql (
 
   profile::postgresql::base_pg_hba_rules { "default": }
 
+  @profile::monitoring::local_service { "Databases are present in postgresql":
+    sudos => {
+      "naemon-postgresql-database-public" => "naemon ALL=(postgres) NOPASSWD: /usr/bin/psql -c select\\ nspname\\ from\\ pg_catalog.pg_namespace"
+    },
+    local => {
+      check_command => "check_command_output!psql -c 'select nspname from pg_catalog.pg_namespace'!public!-r postgres",
+    }
+  }
 }
 
diff --git a/modules/profile/manifests/postgresql/backup_dump.pp b/modules/profile/manifests/postgresql/backup_dump.pp
index 53fb20e..e247cf0 100644
--- a/modules/profile/manifests/postgresql/backup_dump.pp
+++ b/modules/profile/manifests/postgresql/backup_dump.pp
@@ -57,4 +57,13 @@ define profile::postgresql::backup_dump (
       },
     ]
   }
+
+  @profile::monitoring::local_service { "Last postgresql dump in $pg_backup_path is not too old":
+    sudos => {
+      "naemon-postgresql-dumps-$pg_host" => "naemon  ALL=($pg_user) NOPASSWD: /usr/bin/find $pg_backup_path -mindepth 1 -maxdepth 1 -printf %T@?n",
+    },
+    local => {
+      check_command => "check_last_file_date!$pg_backup_path!7!$pg_user",
+    }
+  }
 }
diff --git a/modules/profile/manifests/postgresql/backup_pgbouncer.pp b/modules/profile/manifests/postgresql/backup_pgbouncer.pp
index 45b8ed5..5fd7861 100644
--- a/modules/profile/manifests/postgresql/backup_pgbouncer.pp
+++ b/modules/profile/manifests/postgresql/backup_pgbouncer.pp
@@ -48,6 +48,16 @@ define profile::postgresql::backup_pgbouncer (
     content => "${pg_infos[pgbouncer_dbname]} = host=$pg_path$pg_port user=${pg_infos[dbuser]} dbname=${pg_infos[dbname]}",
   }
 
+  # Current pam configuration requires password for postgres
+  # @profile::monitoring::local_service { "Database ${pg_infos[pgbouncer_dbname]} is available in pgbouncer":
+  #   sudos => {
+  #     "naemon-postgresql-database-public" => "naemon ALL=(postgres) NOPASSWD: /usr/bin/psql -c select\ nspname\ from\ pg_catalog.pg_namespace ${pg_infos[pgbouncer_dbname]}"
+  #   },
+  #   local => {
+  #     check_command => "check_command_output!psql -c 'select nspname from pg_catalog.pg_namespace' ${pg_infos[pgbouncer_dbname]}!public!-r postgres",
+  #   }
+  # }
+
   # pg_hba for accessed cluster
   postgresql::server::pg_hba_rule { "$pg_backup_host - local access as ${pg_infos[dbuser]} user":
     description => "Allow local access to ${pg_infos[dbuser]} user",
diff --git a/modules/profile/manifests/postgresql/master.pp b/modules/profile/manifests/postgresql/master.pp
index 02315a6..99ac4c4 100644
--- a/modules/profile/manifests/postgresql/master.pp
+++ b/modules/profile/manifests/postgresql/master.pp
@@ -59,5 +59,15 @@ define profile::postgresql::master (
       handle_slot   => true,
       add_self_role => true,
     }
+
+    @profile::monitoring::local_service { "Postgresql replication for $backup_host is up to date":
+      sudos => {
+        "naemon-postgresql-replication-$backup_host" => "naemon ALL=(postgres) NOPASSWD: /etc/naemon/monitoring-plugins/check_postgres_replication $backup_host /run/postgresql 5432"
+
+      },
+      local => {
+        check_command => "check_postgresql_replication!$backup_host!/run/postgresql!5432",
+      }
+    }
   }
 }
diff --git a/modules/profile/manifests/postgresql/ssl.pp b/modules/profile/manifests/postgresql/ssl.pp
index dc56c0b..b809a9d 100644
--- a/modules/profile/manifests/postgresql/ssl.pp
+++ b/modules/profile/manifests/postgresql/ssl.pp
@@ -78,5 +78,4 @@ define profile::postgresql::ssl (
       content => "ssl = on\nssl_key_file = '$ssl_key'\nssl_cert_file = '$ssl_cert'\n"
     }
   }
-
 }