From 9eae2b47b7b315b05a0e010f3003bd875685e260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Bouya?= Date: Sat, 11 May 2019 10:23:33 +0200 Subject: Move webstats outside of nixops --- modules/default.nix | 1 + modules/webapps/webstats/default.nix | 84 ++++++++++++++++++ modules/webapps/webstats/goaccess.conf | 99 ++++++++++++++++++++++ nixops/modules/websites/aten/default.nix | 9 +- nixops/modules/websites/chloe/default.nix | 9 +- nixops/modules/websites/commons/goaccess.conf | 99 ---------------------- nixops/modules/websites/commons/stats.nix | 69 --------------- nixops/modules/websites/connexionswing/default.nix | 9 +- nixops/modules/websites/default.nix | 6 +- nixops/modules/websites/ftp/denisejerome.nix | 9 +- nixops/modules/websites/ftp/immae.nix | 9 +- nixops/modules/websites/ftp/jerome.nix | 9 +- nixops/modules/websites/ftp/leila.nix | 7 +- nixops/modules/websites/ftp/nassime.nix | 9 +- nixops/modules/websites/ftp/release.nix | 9 +- nixops/modules/websites/ludivine/default.nix | 5 +- nixops/modules/websites/piedsjaloux/default.nix | 9 +- 17 files changed, 198 insertions(+), 253 deletions(-) create mode 100644 modules/webapps/webstats/default.nix create mode 100644 modules/webapps/webstats/goaccess.conf delete mode 100644 nixops/modules/websites/commons/goaccess.conf delete mode 100644 nixops/modules/websites/commons/stats.nix diff --git a/modules/default.nix b/modules/default.nix index c920a51..2c993c5 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -2,6 +2,7 @@ myids = ./myids.nix; secrets = ./secrets.nix; + webstats = ./webapps/webstats; diaspora = ./webapps/diaspora.nix; etherpad-lite = ./webapps/etherpad-lite.nix; mastodon = ./webapps/mastodon.nix; diff --git a/modules/webapps/webstats/default.nix b/modules/webapps/webstats/default.nix new file mode 100644 index 0000000..f4916bd --- /dev/null +++ b/modules/webapps/webstats/default.nix @@ -0,0 +1,84 @@ +{ lib, pkgs, config, mylibs, ... }: +let + name = "goaccess"; + cfg = config.services.webstats; +in { + options.services.webstats = { + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/${name}"; + description = '' + The directory where Goaccess stores its data. + ''; + }; + sites = lib.mkOption { + type = lib.types.listOf (lib.types.submodule { + options = { + conf = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + use custom goaccess configuration file instead of the + default one. + ''; + }; + name = lib.mkOption { + type = lib.types.string; + description = '' + Domain name. Corresponds to the Apache file name and the + folder name in which the state will be saved. + ''; + }; + }; + }); + default = []; + description = "Sites to generate stats"; + }; + }; + + config = lib.mkIf (builtins.length cfg.sites > 0) { + users.users.root.packages = [ + pkgs.goaccess + ]; + + services.cron = { + enable = true; + systemCronJobs = let + stats = domain: conf: let + config = if builtins.isNull conf + then pkgs.runCommand "goaccess.conf" { + dbPath = "${cfg.dataDir}/${domain}"; + } "substituteAll ${./goaccess.conf} $out" + else conf; + d = pkgs.writeScriptBin "stats-${domain}" '' + #!${pkgs.stdenv.shell} + set -e + shopt -s nullglob + date_regex=$(LC_ALL=C date -d yesterday +'%d\/%b\/%Y') + TMPFILE=$(mktemp) + trap "rm -f $TMPFILE" EXIT + + cat /var/log/httpd/access_log-${domain} | sed -n "/\\[$date_regex/ p" > $TMPFILE + for i in /var/log/httpd/access_log-${domain}*.gz; do + zcat "$i" | sed -n "/\\[$date_regex/ p" >> $TMPFILE + done + ${pkgs.goaccess}/bin/goaccess $TMPFILE --no-progress -o ${cfg.dataDir}/${domain}/index.html -p ${config} + ''; + in "${d}/bin/stats-${domain}"; + allStats = sites: pkgs.writeScript "stats" '' + #!${pkgs.stdenv.shell} + + ${builtins.concatStringsSep "\n" (map (v: stats v.name v.conf) sites)} + ''; + in + [ + "5 0 * * * root ${allStats cfg.sites}" + ]; + }; + + system.activationScripts.goaccess = '' + mkdir -p /var/lib/goaccess + '' + + builtins.concatStringsSep "\n" (map (v: "mkdir -p ${cfg.dataDir}/${v.name}") cfg.sites); + }; +} diff --git a/modules/webapps/webstats/goaccess.conf b/modules/webapps/webstats/goaccess.conf new file mode 100644 index 0000000..4918988 --- /dev/null +++ b/modules/webapps/webstats/goaccess.conf @@ -0,0 +1,99 @@ +time-format %H:%M:%S +date-format %d/%b/%Y + +#sur immae.eu +#log-format %v %h %^[%d:%t %^] "%r" %s %b "%R" "%u" $^ + +log-format VCOMBINED +#= %v:%^ %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +html-prefs {"theme":"bright","layout":"vertical"} + +exclude-ip 188.165.209.148 +exclude-ip 178.33.252.96 +exclude-ip 2001:41d0:2:9c94::1 +exclude-ip 2001:41d0:2:9c94:: +exclude-ip 176.9.151.89 +exclude-ip 2a01:4f8:160:3445:: +exclude-ip 82.255.56.72 + +no-query-string true + +keep-db-files true +load-from-disk true +db-path @dbPath@ + +ignore-panel REFERRERS +ignore-panel KEYPHRASES + +static-file .css +static-file .js +static-file .jpg +static-file .png +static-file .gif +static-file .ico +static-file .jpeg +static-file .pdf +static-file .csv +static-file .mpeg +static-file .mpg +static-file .swf +static-file .woff +static-file .woff2 +static-file .xls +static-file .xlsx +static-file .doc +static-file .docx +static-file .ppt +static-file .pptx +static-file .txt +static-file .zip +static-file .ogg +static-file .mp3 +static-file .mp4 +static-file .exe +static-file .iso +static-file .gz +static-file .rar +static-file .svg +static-file .bmp +static-file .tar +static-file .tgz +static-file .tiff +static-file .tif +static-file .ttf +static-file .flv +#static-file .less +#static-file .ac3 +#static-file .avi +#static-file .bz2 +#static-file .class +#static-file .cue +#static-file .dae +#static-file .dat +#static-file .dts +#static-file .ejs +#static-file .eot +#static-file .eps +#static-file .img +#static-file .jar +#static-file .map +#static-file .mid +#static-file .midi +#static-file .ogv +#static-file .webm +#static-file .mkv +#static-file .odp +#static-file .ods +#static-file .odt +#static-file .otf +#static-file .pict +#static-file .pls +#static-file .ps +#static-file .qt +#static-file .rm +#static-file .svgz +#static-file .wav +#static-file .webp + + diff --git a/nixops/modules/websites/aten/default.nix b/nixops/modules/websites/aten/default.nix index 5dff008..fd3f7cc 100644 --- a/nixops/modules/websites/aten/default.nix +++ b/nixops/modules/websites/aten/default.nix @@ -19,17 +19,10 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkMerge [ (lib.mkIf cfg.production.enable { secrets.keys = aten_prod.keys; - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "aten.pro"; } - ]; + services.webstats.sites = [ { name = "aten.pro"; } ]; security.acme.certs."aten" = config.services.myCertificates.certConfig // { domain = "aten.pro"; diff --git a/nixops/modules/websites/chloe/default.nix b/nixops/modules/websites/chloe/default.nix index 8980077..a542d70 100644 --- a/nixops/modules/websites/chloe/default.nix +++ b/nixops/modules/websites/chloe/default.nix @@ -19,17 +19,10 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkMerge [ (lib.mkIf cfg.production.enable { secrets.keys = chloe_prod.keys; - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "osteopathe-cc.fr"; } - ]; + services.webstats.sites = [ { name = "osteopathe-cc.fr"; } ]; security.acme.certs."chloe" = config.services.myCertificates.certConfig // { domain = "osteopathe-cc.fr"; diff --git a/nixops/modules/websites/commons/goaccess.conf b/nixops/modules/websites/commons/goaccess.conf deleted file mode 100644 index 4918988..0000000 --- a/nixops/modules/websites/commons/goaccess.conf +++ /dev/null @@ -1,99 +0,0 @@ -time-format %H:%M:%S -date-format %d/%b/%Y - -#sur immae.eu -#log-format %v %h %^[%d:%t %^] "%r" %s %b "%R" "%u" $^ - -log-format VCOMBINED -#= %v:%^ %h %^[%d:%t %^] "%r" %s %b "%R" "%u" - -html-prefs {"theme":"bright","layout":"vertical"} - -exclude-ip 188.165.209.148 -exclude-ip 178.33.252.96 -exclude-ip 2001:41d0:2:9c94::1 -exclude-ip 2001:41d0:2:9c94:: -exclude-ip 176.9.151.89 -exclude-ip 2a01:4f8:160:3445:: -exclude-ip 82.255.56.72 - -no-query-string true - -keep-db-files true -load-from-disk true -db-path @dbPath@ - -ignore-panel REFERRERS -ignore-panel KEYPHRASES - -static-file .css -static-file .js -static-file .jpg -static-file .png -static-file .gif -static-file .ico -static-file .jpeg -static-file .pdf -static-file .csv -static-file .mpeg -static-file .mpg -static-file .swf -static-file .woff -static-file .woff2 -static-file .xls -static-file .xlsx -static-file .doc -static-file .docx -static-file .ppt -static-file .pptx -static-file .txt -static-file .zip -static-file .ogg -static-file .mp3 -static-file .mp4 -static-file .exe -static-file .iso -static-file .gz -static-file .rar -static-file .svg -static-file .bmp -static-file .tar -static-file .tgz -static-file .tiff -static-file .tif -static-file .ttf -static-file .flv -#static-file .less -#static-file .ac3 -#static-file .avi -#static-file .bz2 -#static-file .class -#static-file .cue -#static-file .dae -#static-file .dat -#static-file .dts -#static-file .ejs -#static-file .eot -#static-file .eps -#static-file .img -#static-file .jar -#static-file .map -#static-file .mid -#static-file .midi -#static-file .ogv -#static-file .webm -#static-file .mkv -#static-file .odp -#static-file .ods -#static-file .odt -#static-file .otf -#static-file .pict -#static-file .pls -#static-file .ps -#static-file .qt -#static-file .rm -#static-file .svgz -#static-file .wav -#static-file .webp - - diff --git a/nixops/modules/websites/commons/stats.nix b/nixops/modules/websites/commons/stats.nix deleted file mode 100644 index 73595f1..0000000 --- a/nixops/modules/websites/commons/stats.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ lib, pkgs, config, mylibs, ... }: -let - cfg = config.services.myWebsites.commons.stats; -in { - options = { - services.myWebsites.commons.stats = { - enable = lib.mkEnableOption "enable statistics"; - sites = lib.mkOption { - type = lib.types.listOf (lib.types.submodule { - options = { - conf = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - }; - name = lib.mkOption { type = lib.types.string; }; - }; - }); - default = []; - description = "Sites to generate stats"; - }; - }; - }; - - config = lib.mkIf cfg.enable { - users.users.root.packages = [ - pkgs.goaccess - ]; - - services.cron = { - enable = true; - systemCronJobs = let - stats = domain: conf: let - config = if builtins.isNull conf - then pkgs.runCommand "goaccess.conf" { - dbPath = "/var/lib/goaccess/${domain}"; - } "substituteAll ${./goaccess.conf} $out" - else conf; - d = pkgs.writeScriptBin "stats-${domain}" '' - #!${pkgs.stdenv.shell} - set -e - shopt -s nullglob - date_regex=$(LC_ALL=C date -d yesterday +'%d\/%b\/%Y') - TMPFILE=$(mktemp) - trap "rm -f $TMPFILE" EXIT - - cat /var/log/httpd/access_log-${domain} | sed -n "/\\[$date_regex/ p" > $TMPFILE - for i in /var/log/httpd/access_log-${domain}*.gz; do - zcat "$i" | sed -n "/\\[$date_regex/ p" >> $TMPFILE - done - ${pkgs.goaccess}/bin/goaccess $TMPFILE --no-progress -o /var/lib/goaccess/${domain}/index.html -p ${config} - ''; - in "${d}/bin/stats-${domain}"; - allStats = sites: pkgs.writeScript "stats" '' - #!${pkgs.stdenv.shell} - - ${builtins.concatStringsSep "\n" (map (v: stats v.name v.conf) sites)} - ''; - in - [ - "5 0 * * * root ${allStats cfg.sites}" - ]; - }; - - system.activationScripts.goaccess = '' - mkdir -p /var/lib/goaccess - '' + - builtins.concatStringsSep "\n" (map (v: "mkdir -p /var/lib/goaccess/${v.name}") cfg.sites); - }; -} diff --git a/nixops/modules/websites/connexionswing/default.nix b/nixops/modules/websites/connexionswing/default.nix index 37426dd..773c8de 100644 --- a/nixops/modules/websites/connexionswing/default.nix +++ b/nixops/modules/websites/connexionswing/default.nix @@ -19,17 +19,10 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkMerge [ (lib.mkIf cfg.production.enable { secrets.keys = connexionswing_prod.keys; - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "connexionswing.com"; } - ]; + services.webstats.sites = [ { name = "connexionswing.com"; } ]; security.acme.certs."connexionswing" = config.services.myCertificates.certConfig // { domain = "connexionswing.com"; diff --git a/nixops/modules/websites/default.nix b/nixops/modules/websites/default.nix index ceef1e1..79cd1a1 100644 --- a/nixops/modules/websites/default.nix +++ b/nixops/modules/websites/default.nix @@ -260,13 +260,13 @@ in stats = { extraConfig = '' - Alias /awstats /var/lib/goaccess/%{domain} - + Alias /webstats ${config.services.webstats.dataDir}/%{domain} + DirectoryIndex index.html AllowOverride None Require all granted - + Use LDAPConnect Require ldap-group cn=%{domain},ou=stats,cn=httpd,ou=services,dc=immae,dc=eu diff --git a/nixops/modules/websites/ftp/denisejerome.nix b/nixops/modules/websites/ftp/denisejerome.nix index f39e5c2..4069b46 100644 --- a/nixops/modules/websites/ftp/denisejerome.nix +++ b/nixops/modules/websites/ftp/denisejerome.nix @@ -10,15 +10,8 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkIf cfg.production.enable { - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "denisejerome.piedsjaloux.fr"; } - ]; + services.webstats.sites = [ { name = "denisejerome.piedsjaloux.fr"; } ]; security.acme.certs."denisejerome" = config.services.myCertificates.certConfig // { domain = "denisejerome.piedsjaloux.fr"; diff --git a/nixops/modules/websites/ftp/immae.nix b/nixops/modules/websites/ftp/immae.nix index 2ecca1f..d4e6d39 100644 --- a/nixops/modules/websites/ftp/immae.nix +++ b/nixops/modules/websites/ftp/immae.nix @@ -10,15 +10,8 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkIf cfg.production.enable { - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "www.immae.eu"; } - ]; + services.webstats.sites = [ { name = "www.immae.eu"; } ]; security.acme.certs."eldiron".extraDomains."www.immae.eu" = null; diff --git a/nixops/modules/websites/ftp/jerome.nix b/nixops/modules/websites/ftp/jerome.nix index e04d640..60e4dfb 100644 --- a/nixops/modules/websites/ftp/jerome.nix +++ b/nixops/modules/websites/ftp/jerome.nix @@ -11,15 +11,8 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkIf cfg.production.enable { - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "naturaloutil.immae.eu"; } - ]; + services.webstats.sites = [ { name = "naturaloutil.immae.eu"; } ]; security.acme.certs."ftp".extraDomains."naturaloutil.immae.eu" = null; security.acme.certs."naturaloutil" = config.services.myCertificates.certConfig // { diff --git a/nixops/modules/websites/ftp/leila.nix b/nixops/modules/websites/ftp/leila.nix index d5c27fa..c5c27b1 100644 --- a/nixops/modules/websites/ftp/leila.nix +++ b/nixops/modules/websites/ftp/leila.nix @@ -9,10 +9,6 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = (lib.mkIf cfg.production.enable { security.acme.certs."leila" = config.services.myCertificates.certConfig // { domain = "leila.bouya.org"; @@ -37,8 +33,7 @@ in { php_admin_value[open_basedir] = "${varDir}:/tmp" ''; - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ + services.webstats.sites = [ { name = "leila.bouya.org"; } { name = "chorale.leila.bouya.org"; } ]; diff --git a/nixops/modules/websites/ftp/nassime.nix b/nixops/modules/websites/ftp/nassime.nix index d7ced7e..f295045 100644 --- a/nixops/modules/websites/ftp/nassime.nix +++ b/nixops/modules/websites/ftp/nassime.nix @@ -10,15 +10,8 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkIf cfg.production.enable { - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "nassime.bouya.org"; } - ]; + services.webstats.sites = [ { name = "nassime.bouya.org"; } ]; security.acme.certs."ftp".extraDomains."nassime.bouya.org" = null; security.acme.certs."nassime" = config.services.myCertificates.certConfig // { diff --git a/nixops/modules/websites/ftp/release.nix b/nixops/modules/websites/ftp/release.nix index 2050a6c..896691b 100644 --- a/nixops/modules/websites/ftp/release.nix +++ b/nixops/modules/websites/ftp/release.nix @@ -10,15 +10,8 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkIf cfg.production.enable { - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "release.immae.eu"; } - ]; + services.webstats.sites = [ { name = "release.immae.eu"; } ]; security.acme.certs."eldiron".extraDomains."release.immae.eu" = null; diff --git a/nixops/modules/websites/ludivine/default.nix b/nixops/modules/websites/ludivine/default.nix index af18d71..2fcd94a 100644 --- a/nixops/modules/websites/ludivine/default.nix +++ b/nixops/modules/websites/ludivine/default.nix @@ -22,10 +22,7 @@ in { config = lib.mkMerge [ (lib.mkIf cfg.production.enable { secrets.keys = ludivinecassal_prod.keys; - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "ludivinecassal.com"; } - ]; + services.webstats.sites = [ { name = "ludivinecassal.com"; } ]; security.acme.certs."ludivinecassal" = config.services.myCertificates.certConfig // { domain = "ludivinecassal.com"; diff --git a/nixops/modules/websites/piedsjaloux/default.nix b/nixops/modules/websites/piedsjaloux/default.nix index 1657914..9f9697d 100644 --- a/nixops/modules/websites/piedsjaloux/default.nix +++ b/nixops/modules/websites/piedsjaloux/default.nix @@ -19,17 +19,10 @@ in { }; }; - imports = [ - ../commons/stats.nix - ]; - config = lib.mkMerge [ (lib.mkIf cfg.production.enable { secrets.keys = piedsjaloux_prod.keys; - services.myWebsites.commons.stats.enable = true; - services.myWebsites.commons.stats.sites = [ - { name = "piedsjaloux.fr"; } - ]; + services.webstats.sites = [ { name = "piedsjaloux.fr"; } ]; security.acme.certs."piedsjaloux" = config.services.myCertificates.certConfig // { domain = "piedsjaloux.fr"; -- cgit v1.2.3