From: Ismaël Bouya Date: Tue, 14 Sep 2021 00:28:09 +0000 (+0200) Subject: Add flask app paste X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=a9f52ec521e45204ad9363dd143b32ac9910b6b3;p=perso%2FImmae%2FConfig%2FNix.git Add flask app paste --- diff --git a/flakes/paste/flake.lock b/flakes/paste/flake.lock new file mode 100644 index 0000000..559e64b --- /dev/null +++ b/flakes/paste/flake.lock @@ -0,0 +1,42 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1631561581, + "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1631570365, + "narHash": "sha256-vc6bfo0hijpicdUDiui2DvZXmpIP2iqOFZRcpMOuYPo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "df7113c0727881519248d4c7d080324e0ee3327b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/paste/flake.nix b/flakes/paste/flake.nix new file mode 100644 index 0000000..08d0681 --- /dev/null +++ b/flakes/paste/flake.nix @@ -0,0 +1,138 @@ +{ + inputs.flake-utils.url = "github:numtide/flake-utils"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + + description = "Pastebin-like service"; + + outputs = { self, flake-utils, nixpkgs }: flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; overlays = []; }; + in rec { + hydraJobs = checks; + checks = pkgs.lib.optionalAttrs (builtins.elem system pkgs.lib.systems.doubles.linux) { + test = + let testing = import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }; + in testing.makeTest { + nodes = { + server = { pkgs, ... }: { + imports = [ self.nixosModule ]; + config = { + environment.systemPackages = [ pkgs.curl ]; + services.httpd = { + enable = true; + adminAddr = "foo@example.org"; + extraConfig = '' + ProxyPass / unix:///run/paste/gunicorn.sock|http://localhost/ + ProxyPassReverse / unix:///run/paste/gunicorn.sock|http://localhost/ + ''; + }; + services.paste.enable = true; + }; + }; + }; + testScript = '' + start_all() + server.wait_for_unit("httpd.service") + server.wait_for_unit("paste.service") + server.wait_until_succeeds("[ -S /run/paste/gunicorn.sock ]", 10) + server.succeed("curl -f http://localhost/") + server.succeed("curl -f http://localhost/ | grep -q 'Get the source'") + ''; + }; + }; + }) // rec { + nixosModule = { config, lib, pkgs, ... }: + let + cfg = config.services.paste; + in { + options = { + services.paste = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Whether to enable the pastebin service"; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/paste"; + description = '' + The directory where Paste stores its data. + ''; + }; + + socketsDir = lib.mkOption { + type = lib.types.str; + default = "/run/paste"; + description = "Socket which is used for communication with Paste."; + }; + + webDirectory = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Subdirectory url to which the app will be served"; + }; + + # Output variables + systemdStateDirectory = lib.mkOption { + type = lib.types.str; + # Use ReadWritePaths= instead if varDir is outside of /var/lib + default = assert lib.strings.hasPrefix "/var/lib/" cfg.dataDir; + lib.strings.removePrefix "/var/lib/" cfg.dataDir; + description = '' + Adjusted paste data directory for systemd + ''; + readOnly = true; + }; + systemdRuntimeDirectory = lib.mkOption { + type = lib.types.str; + # Use ReadWritePaths= instead if socketsDir is outside of /run + default = assert lib.strings.hasPrefix "/run/" cfg.socketsDir; + lib.strings.removePrefix "/run/" cfg.socketsDir; + description = '' + Adjusted paste sockets directory for systemd + ''; + readOnly = true; + }; + sockets = lib.mkOption { + type = lib.types.attrsOf lib.types.path; + default = { + pid = "${cfg.socketsDir}/gunicorn.pid"; + gunicorn = "${cfg.socketsDir}/gunicorn.sock"; + }; + readOnly = true; + description = '' + Paste sockets + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.paste = { + description = "Pastebin like service"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Environment = pkgs.lib.optionals (cfg.webDirectory != null) [ "SCRIPT_NAME=${cfg.webDirectory}" ]; + Type = "simple"; + User = config.services.httpd.user; + ExecStart = let + python = pkgs.python3.withPackages (p: [p.gunicorn p.flask p.pygments p.python_magic ]); + in + "${python}/bin/gunicorn -w4 -p ${cfg.sockets.pid} -e PASTE_DIRECTORY=${cfg.dataDir} --bind unix:${cfg.sockets.gunicorn} --chdir ${./paste} paste:app"; + Restart = "always"; + RestartSec = "5s"; + PIDFile = cfg.sockets.pid; + RuntimeDirectory = cfg.systemdRuntimeDirectory; + StateDirectory = cfg.systemdStateDirectory; + StandardOutput = "journal"; + StandardError = "inherit"; + }; + + }; + }; + }; + }; +} diff --git a/flakes/paste/paste/paste.py b/flakes/paste/paste/paste.py new file mode 100644 index 0000000..86666b8 --- /dev/null +++ b/flakes/paste/paste/paste.py @@ -0,0 +1,124 @@ +import os +import secrets +from flask import Flask, abort, request, Response, url_for +from pygments.formatters.html import HtmlFormatter +from pygments import highlight +import pygments.lexers as lexers +import base64 +import magic +import mimetypes + +magic = magic.Magic(mime=True) + +config = { + "directory": os.environ["PASTE_DIRECTORY"], + "self_paste_id": "abcd123", + "max_content_length": 16 * 1000 * 1000 + } + +app = Flask(__name__) +app.config['MAX_CONTENT_LENGTH'] = config["max_content_length"] + +def file_location(paste_id): + if paste_id == config["self_paste_id"]: + return os.path.realpath(__file__) + else: + return os.path.join(config['directory'], paste_id + ".dat") + +def read_paste(paste_id): + file = file_location(paste_id) + if os.path.isfile(file): + content = open(file, "rb").read() + mime = magic.from_buffer(content) + if mime.startswith("text/x-script."): + mime="text/plain" + return (content, mime) + else: + abort(404) + +def generate_paste_id(n=3, attempts=0): + path = secrets.token_hex(n)[:n] + file = file_location(path) + if os.path.isfile(file): + attempts = attempts + 1 + if attempts > 5: + n = n + 1 + return generate_paste_id(n, attempts) + return path + +@app.route('/', methods=["GET"]) +def index(): + return Response('''
+$ curl -X POST --data-binary @{self} {host}
+{paste}
+
+-> GET {paste}
+   guesses mimetype
+-> GET {paste}/raw
+   text/plain
+-> GET {paste}/bin
+   application/octet-stream
+-> GET {paste}/b64
+   base64 encoded
+-> GET {paste}/ub64
+   tries to decode base64
+-> GET {paste}/python
+   pretty-print python (replace with anything known by pygments)
+-> GET {paste}/guess
+   pretty-print (language guessed by pygments)
+-> GET {paste}/download
+   force download of file
+
+Get the source +'''.format(host=url_for('post_paste', _external=True, _scheme="https"), + paste=url_for('get_paste', _external=True, _scheme="https", paste_id=config["self_paste_id"]), + self=os.path.basename(__file__) + ), mimetype="text/html") + +@app.route('/', methods=["POST"]) +def post_paste(): + content = request.get_data() + paste_id = generate_paste_id() + with open(file_location(paste_id), "wb") as f: + f.write(content) + return url_for('get_paste', _external=True, _scheme="https", paste_id=paste_id) + "\n" + + +@app.route('/', methods=["GET"]) +def get_paste(paste_id): + content, mime = read_paste(paste_id) + return Response(content, mimetype=mime) + +@app.route('//', methods=["GET"]) +def get_paste_with(paste_id, lang): + content, mime = read_paste(paste_id) + if lang == "raw": + return Response(content, mimetype="text/plain") + elif lang == "bin": + return Response(content, mimetype="application/octet-stream") + elif lang == "b64": + return Response(base64.encodebytes(content), mimetype="text/plain") + elif lang == "download": + extension = mimetypes.guess_extension(mime, strict=False) + if extension is None: + cd = "attachment" + else: + cd = 'attachment; filename="{}{}"'.format(paste_id, extension) + return Response(content, mimetype=mime, + headers={"Content-Disposition": cd}) + elif lang == "ub64": + try: + return base64.b64decode(content) + except: + abort(400) + try: + if lang == "guess": + lexer = lexers.guess_lexer(content) + else: + lexer = lexers.find_lexer_class_by_name(lang)() + except: + abort(400) + fmter = HtmlFormatter( + noclasses=True, full=True, style="colorful", linenos="table") + + return Response(highlight(content, lexer, fmter), mimetype="text/html") diff --git a/flakes/private/paste/flake.lock b/flakes/private/paste/flake.lock new file mode 100644 index 0000000..939d589 --- /dev/null +++ b/flakes/private/paste/flake.lock @@ -0,0 +1,72 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1631561581, + "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-lib": { + "locked": { + "lastModified": 1631655525, + "narHash": "sha256-8U7zAdbjNItXo6eqI/rhtOa3LUPGD6yE9PTZQkrSGHo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "cf0caf529c33c140863ebfa43691f7b69fe2233c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1631570365, + "narHash": "sha256-vc6bfo0hijpicdUDiui2DvZXmpIP2iqOFZRcpMOuYPo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "df7113c0727881519248d4c7d080324e0ee3327b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "paste": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "narHash": "sha256-oSabBrUGIkY8lKktXlIM4uYSVYI54wKnIjjVZwMOd70=", + "path": "../../paste", + "type": "path" + }, + "original": { + "path": "../../paste", + "type": "path" + } + }, + "root": { + "inputs": { + "nix-lib": "nix-lib", + "paste": "paste" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flakes/private/paste/flake.nix b/flakes/private/paste/flake.nix new file mode 100644 index 0000000..71314e8 --- /dev/null +++ b/flakes/private/paste/flake.nix @@ -0,0 +1,23 @@ +{ + inputs.paste = { + path = "../../paste"; + type = "path"; + }; + inputs.nix-lib.url = "github:NixOS/nixpkgs"; + + description = "Private configuration for paste"; + outputs = { self, nix-lib, paste }: + let + cfg = name': { config, lib, pkgs, name, ... }: { + config = lib.mkIf (name == name') { + services.paste = { + enable = true; + webDirectory = "/paste"; + }; + }; + }; + in + paste.outputs // + { nixosModules = paste.nixosModules or {} // nix-lib.lib.genAttrs ["eldiron"] cfg; }; +} + diff --git a/modules/default.nix b/modules/default.nix index 1b09c94..7ce1cc2 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -13,6 +13,7 @@ in mediagoblin = ./webapps/mediagoblin.nix; peertube = (flakeCompat ../flakes/peertube).nixosModule; fiche = ./webapps/fiche.nix; + paste = (flakeCompat ../flakes/paste).nixosModule; opendmarc = (flakeCompat ../flakes/opendmarc).nixosModule; openarc = (flakeCompat ../flakes/openarc).nixosModule; diff --git a/modules/private/websites/tools/tools/default.nix b/modules/private/websites/tools/tools/default.nix index 6464206..ac92ef4 100644 --- a/modules/private/websites/tools/tools/default.nix +++ b/modules/private/websites/tools/tools/default.nix @@ -1,5 +1,7 @@ { lib, pkgs, config, ... }: let + flakeCompat = import ../../../../../lib/flake-compat.nix; + adminer = pkgs.callPackage ./adminer.nix { inherit (pkgs.webapps) adminer; }; @@ -64,6 +66,9 @@ let cfg = config.myServices.websites.tools.tools; pcfg = config.services.phpfpm.pools; in { + imports = + builtins.attrValues (flakeCompat ../../../../../flakes/private/paste).nixosModules; + options.myServices.websites.tools.tools = { enable = lib.mkEnableOption "enable tools website"; }; @@ -165,13 +170,16 @@ in { (phpbb.apache.vhostConf pcfg.phpbb.socket) (dmarc-reports.apache.vhostConf pcfg.dmarc-reports.socket) '' - Alias /paste /var/lib/fiche - - DirectoryIndex index.txt index.html - AllowOverride None - Require all granted - Options -Indexes - + + ProxyPass unix://${config.services.paste.sockets.gunicorn}|http://tools.immae.eu/paste/ + ProxyPassReverse unix://${config.services.paste.sockets.gunicorn}|http://tools.immae.eu/paste/ + ProxyPreserveHost on + + + ProxyPass unix://${config.services.paste.sockets.gunicorn}|http://tools.immae.eu/paste/ + ProxyPassReverse unix://${config.services.paste.sockets.gunicorn}|http://tools.immae.eu/paste/ + ProxyPreserveHost on + Alias /BIP39 /var/lib/buildbot/outputs/immae/bip39 @@ -437,12 +445,6 @@ in { paths = [ "/var/secrets/webapps/tools-wallabag" ]; }; - services.fiche = { - enable = true; - port = config.myEnv.ports.fiche; - domain = "tools.immae.eu/paste"; - https = true; - }; }; } diff --git a/modules/private/websites/tools/tools/ympd.nix b/modules/private/websites/tools/tools/ympd.nix index 72d45d4..531b1a9 100644 --- a/modules/private/websites/tools/tools/ympd.nix +++ b/modules/private/websites/tools/tools/ympd.nix @@ -26,12 +26,12 @@ let ProxyPass ws://${config.webPort}/ws - ProxyPass unix:///run/mpd/mp3.sock|http://tools.immae.eu/ - ProxyPassReverse unix:///run/mpd/mp3.sock|http://tools.immae.eu/ + ProxyPass unix:///run/mpd/mp3.sock|http://tools.immae.eu/mpd/mp3 + ProxyPassReverse unix:///run/mpd/mp3.sock|http://tools.immae.eu/mpd/mp3 - ProxyPass unix:///run/mpd/ogg.sock|http://tools.immae.eu/ - ProxyPassReverse unix:///run/mpd/ogg.sock|http://tools.immae.eu/ + ProxyPass unix:///run/mpd/ogg.sock|http://tools.immae.eu/mpd/ogg + ProxyPassReverse unix:///run/mpd/ogg.sock|http://tools.immae.eu/mpd/ogg ''; };