Adjust monitoring resources and add bandwidth checks

4 files changed, 148 insertions, 3 deletions

diff --git a/modules/private/monitoring/default.nix b/modules/private/monitoring/default.nix
index 316c2dd..8f8b6c0 100644
--- a/modules/private/monitoring/default.nix
+++ b/modules/private/monitoring/default.nix
@@ -70,20 +70,26 @@ let
     wrapProgram $out/check_ovh_sms --prefix PATH : ${lib.makeBinPath [
       (pkgs.python3.withPackages (ps: [ps.ovh]))
     ]}
+    wrapProgram $out/check_bandwidth --prefix PATH : ${lib.makeBinPath [
+      pkgs.iproute pkgs.bc
+    ]}
     '';
   toObjects = pkgs.callPackage ./to_objects.nix {};
   commonConfig = {
     eldiron = {
       processWarn = "250"; processAlert = "400";
       loadWarn = "8.0"; loadAlert = "10.0";
+      interface = "eth0";
     };
     backup-2 = {
       processWarn = "60"; processAlert = "70";
       loadWarn = "1.0"; loadAlert = "2.0";
+      interface = "ens3";
     };
     monitoring-1 = {
       processWarn = "50"; processAlert = "60";
       loadWarn = "1.0"; loadAlert = "2.0";
+      interface = "ens3";
     };
   };
   masterPassiveObjects = let
diff --git a/modules/private/monitoring/objects_common.nix b/modules/private/monitoring/objects_common.nix
index c0a17e6..d3a46ce 100644
--- a/modules/private/monitoring/objects_common.nix
+++ b/modules/private/monitoring/objects_common.nix
@@ -1,5 +1,6 @@
 { hostFQDN
 , hostName
+, interface ? "eth0"
 , processWarn ? "250"
 , processAlert ? "400"
 , loadWarn ? "8.0"
@@ -51,6 +52,20 @@ in
     }
     {
       passiveInfo = defaultPassiveInfo;
+      service_description = "Network bandwidth";
+      use = "local-service";
+      check_interval = "2";
+      max_check_attempts = "20";
+      retry_interval = "2";
+      check_command = [
+        "check_local_bandwidth"
+        interface
+        "20480" # kb/s
+        "51200" # kb/s
+      ];
+    }
+    {
+      passiveInfo = defaultPassiveInfo;
       service_description = "Average load";
       use = "local-service";
       check_command = [
@@ -108,6 +123,7 @@ in
     check_local_procs = "$USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$";
     check_local_load = "$USER1$/check_load -w $ARG1$ -c $ARG2$";
     check_local_swap = "$USER1$/check_swap -n ok -w $ARG1$ -c $ARG2$";
+    check_local_bandwidth = "$USER2$/check_bandwidth -i=$ARG1$ -w $ARG2$ -c $ARG3$";
     check_memory = "$USER2$/check_mem.sh -w $ARG1$ -c $ARG2$";
     check_command_match = "$USER2$/check_command -c \"$ARG1$\" -C \"$ARG2$\" $ARG3$";
     check_command_output = "$USER2$/check_command -c \"$ARG1$\" -s 0 -o \"$ARG2$\" $ARG3$";
diff --git a/modules/private/monitoring/objects_monitoring-1.nix b/modules/private/monitoring/objects_monitoring-1.nix
index 02870ed..320f6e3 100644
--- a/modules/private/monitoring/objects_monitoring-1.nix
+++ b/modules/private/monitoring/objects_monitoring-1.nix
@@ -15,7 +15,7 @@ let
     check_command = ["check_eriomem_age" name];
 
     check_interval = "120";
-    notification_interval = "120";
+    notification_interval = "1440";
   };
 in
 {
@@ -106,7 +106,7 @@ in
       check_command = "check_ovh_sms";
 
       check_interval = "120";
-      notification_interval = "120";
+      notification_interval = "1440";
     }
 
     # Backup services
@@ -117,7 +117,7 @@ in
       check_command = "check_eriomem";
 
       check_interval = "120";
-      notification_interval = "120";
+      notification_interval = "1440";
 
       servicegroups = "webstatus-backup";
     }
diff --git a/modules/private/monitoring/plugins/check_bandwidth b/modules/private/monitoring/plugins/check_bandwidth
new file mode 100755
index 0000000..53c5d85
--- /dev/null
+++ b/modules/private/monitoring/plugins/check_bandwidth
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# ============================== SUMMARY =====================================
+#Author : Ken Roulamellah
+#Date : 19/07/2018
+#Version : 1.0
+# Licence : GPL
+# ===================== INFORMATION ABOUT THIS PLUGIN ========================
+#
+# This plugin checks the average RX and TX bandwidth utilisation. It use
+# kbytes as measure unite.
+#
+# ========================== START OF PROGRAM CODE ===========================
+
+STATE_OK=0
+STATE_WARNING=1
+STATE_CRITICAL=2
+STATE_UNKNOWN=3
+
+interface=$( ip route | grep default | awk '{print $5}' | head -n1)
+function print_usage()
+{
+  echo "Usage :"
+  echo "$0 [ -i=INTERFACE]  [ -ct=COUNT ] -w WARNING -c CRITICAL"
+  echo "This script calculate the average bandwith usage."
+  echo "Default values | interface: ${interface}, counter: 10"
+}
+
+counter=10
+warning=-1
+critical=-1
+
+sum_rx=0
+sum_tx=0
+avg_rx=
+avg_tx=
+i=
+
+
+if [[ $# -lt 4 ]];
+then
+        echo "Error: Arguments are missing"
+        print_usage
+        exit $STATE_UNKNOWN
+fi
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -i=*)
+            interface="$(cut -d'=' -f2 <<<"$1")"
+            shift
+        ;;
+        -ct=*)
+            counter="$(cut -d'=' -f2 <<<"$1")"
+            shift
+        ;;
+        -w)
+            warning=$2
+            shift 2
+        ;;
+        -c)
+            critical=$2
+            shift 2
+        ;;
+        *)
+            printf "\nError: Invalid option '$1'"
+            print_usage
+            exit $STATE_UNKNOWN
+        ;;
+    esac
+done
+
+if [ $warning -lt 0 ] || [ $critical -lt 0 ];
+then
+        echo "Error: You need to specify a warning and critical treshold"
+        print_usage
+    exit $STATE_UNKNOWN
+fi
+
+grep -q "up" /sys/class/net/$interface/operstate || exec echo "$interface: no such device or down"
+
+read rx <"/sys/class/net/$interface/statistics/rx_bytes"
+read tx <"/sys/class/net/$interface/statistics/tx_bytes"
+
+i=$counter
+while [ $i -gt 0 ]; do
+    sleep 1
+    read newrx <"/sys/class/net/$interface/statistics/rx_bytes"
+    read newtx <"/sys/class/net/$interface/statistics/tx_bytes"
+
+    #echo "old rx :$rx"
+    #echo "new rx :$newrx"
+    rx_cal=$(bc <<< "scale=2; ($newrx-$rx) / 1000")
+    tx_cal=$(bc <<< "scale=2; ($newtx-$tx) / 1000")
+
+    sum_rx=$(bc <<< "scale=2;$sum_rx+$rx_cal")
+    sum_tx=$(bc <<< "scale=2;$sum_tx+$tx_cal")
+
+    #echo  "$interface {rx: $rx_cal ko/s, tx: $tx_cal ko/s}"
+    rx=$newrx
+    tx=$newtx
+    ((i --))
+done
+
+avg_rx=$(bc <<< "scale=2;$sum_rx/$counter")
+avg_tx=$(bc <<< "scale=2;$sum_tx/$counter")
+
+#echo "$avg_rx"
+#echo "$avg_tx"
+
+
+if [ $(bc <<< "$avg_rx > $critical || $avg_tx > $critical") -eq 1 ]; then
+        echo "$interface CRITICAL - AVG_RX: $avg_rx kb/s,  AVG_TX:
+        $avg_tx kb/s | RX="$avg_rx"kbps;0;0;0; TX="$avg_tx"kbps;0;0;0;"
+        exit $STATE_CRITICAL
+elif [ $(bc <<< "$avg_rx > $warning || $avg_tx > $warning") -eq 1 ]; then
+        echo "$interface WARNING - AVG_RX: $avg_rx kb/s,  AVG_TX: $avg_tx kb/s | RX="$avg_rx"kbps;0;0;0; TX="$avg_tx"kbps;0;0;0;"
+        exit $STATE_WARNING
+else
+        echo "$interface - OK AVG_RX: $avg_rx kb/s,  AVG_TX: $avg_tx kb/s | RX="$avg_rx"kbps;0;0;0; TX="$avg_tx"kbps;0;0;0;"
+        exit $STATE_OK
+fi
+exit 3