Migrate to morph as a replacement to nixops

The deployment tasks are now independent of any state

20 files changed, 195 insertions, 211 deletions

diff --git a/.envrc b/.envrc
index 6eeaba9..8aa086a 100644
--- a/.envrc
+++ b/.envrc
@@ -1,7 +1,5 @@
 # vim: filetype=bash
 export PASSWORD_STORE_DIR=$(expand_path nixops/secrets)
-export NIXOPS_STATE=$(expand_path nixops/state/immaeEu.nixops)
-export NIXOPS_DEPLOYMENT=cef694f3-081d-11e9-b31f-0242ec186adf
 export NIX_PATH=nixpkgs=$(cat $(expand_path nix/sources.json) | jq -r '."nixpkgs-nixops".url')
 
 export NIXOPS_ENV_LOADED=1
diff --git a/.gitignore b/.gitignore
index 6786d42..ff9c9cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /result*
 /versions_log
 .direnv/
+/nixops/.gcroots
diff --git a/Makefile b/Makefile
index 6ba059c..9c12ff4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,7 @@
 subrecipes = setup nix-info edit_env
-subrecipes += nixops ssh-eldiron ssh-backup-2 ssh-monitoring-1
-subrecipes += info debug dry-run build upload deploy deploy-reboot reboot
+subrecipes += ssh-eldiron ssh-backup-2 ssh-monitoring-1
+subrecipes += debug build upload deploy deploy-reboot
 subrecipes += list-generations delete-generations cleanup
-subrecipes += pull_deployment deployment_is_set push_deployment
 ${subrecipes}:
         @$(MAKE) --no-print-directory -C nixops/ $@
 .PHONY: ${subrecipes}
diff --git a/docs/migrate_hetzner.md b/docs/migrate_hetzner.md
deleted file mode 100644
index c7fbe20..0000000
--- a/docs/migrate_hetzner.md
+++ /dev/null
@@ -1,20 +0,0 @@
-nixops show a deprecation message at each deployment because hetzner
-info is outdated. To fix it:
-
-    cp -a ~/.nixops ~/.nixops.bak
-
-    nixops export --all > all.json
-
-    network=$(cat all.json| jq -r '."cef694f3-081d-11e9-b31f-0242ec186adf".resources.eldiron."hetzner.networkInfo"' | jq -r -c '.networking.interfaces.eth0 = { "ipv4": { "addresses": [ { "address": .networking.interfaces.eth0.ipAddress, "prefixLength": .networking.interfaces.eth0.prefixLength } ] } }')
-
-    cat all.json | jq --arg network "$network" '."cef694f3-081d-11e9-b31f-0242ec186adf".resources.eldiron."hetzner.networkInfo" = $network' > all_new.json
-
-    nixops delete --force -d eldiron
-
-    nixops import < all_new.json
-
-    rm all.json all_new.json
-
-*check that everything works*, then:
-
-    rm -rf ~/.nixops.bak
diff --git a/modules/private/system.nix b/modules/private/system.nix
index b667ee8..6b4ef6e 100644
--- a/modules/private/system.nix
+++ b/modules/private/system.nix
@@ -1,6 +1,12 @@
-{ pkgs, lib, config, name, ... }:
+{ pkgs, lib, config, name, nodes, ... }:
 {
   config = {
+    networking.extraHosts = builtins.concatStringsSep "\n"
+      (lib.mapAttrsToList (n: v: "${v.config.hostEnv.ips.main.ip4} ${n}") nodes);
+
+    users.extraUsers.root.openssh.authorizedKeys.keyFiles = [ "${config.myEnv.privateFiles}/id_ed25519.pub" ];
+    services.openssh.enable = true;
+
     services.duplyBackup.profiles.system = {
       rootDir = "/var/lib";
       excludeFile = lib.mkAfter ''
@@ -73,6 +79,8 @@
     environment.systemPackages = [
       pkgs.git
       pkgs.vim
+      pkgs.rsync
+      pkgs.strace
     ] ++
     (lib.optional (builtins.length (config.hostEnv.users pkgs) > 0) pkgs.home-manager);
 
diff --git a/modules/private/system/backup-2.nix b/modules/private/system/backup-2.nix
index a49c374..d1064c7 100644
--- a/modules/private/system/backup-2.nix
+++ b/modules/private/system/backup-2.nix
@@ -1,26 +1,22 @@
 { privateFiles }:
 { config, pkgs, resources, name, ... }:
 {
+  deployment = {
+    targetUser = "root";
+    targetHost = config.hostEnv.ips.main.ip4;
+    substituteOnDestination = true;
+  };
   boot.kernelPackages = pkgs.linuxPackages_latest;
   myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };
 
-  imports = builtins.attrValues (import ../..);
-
-  deployment = {
-    targetEnv = "hetznerCloud";
-    hetznerCloud = {
-      authToken = config.myEnv.hetznerCloud.authToken;
-      datacenter = "hel1-dc2";
-      location  ="hel1";
-      serverType = "cx11";
-    };
-  };
+  imports = [ <nixpkgs/nixos/modules/profiles/qemu-guest.nix> ] ++ builtins.attrValues (import ../..);
 
   fileSystems = {
     "/backup2" = {
       fsType = "ext4";
       device = "UUID=b9425333-f567-435d-94d8-b26c22d93426";
     };
+    "/" = { device = "/dev/sda1"; fsType = "ext4"; };
   };
 
   networking = {
@@ -34,6 +30,8 @@
     defaultGateway6 = { address = "fe80::1"; interface = "ens3"; };
   };
 
+  boot.loader.grub.device = "nodev";
+
   myServices.certificates.enable = true;
   security.acme.certs."${name}" = {
     user = config.services.nginx.user;
diff --git a/modules/private/system/dilion.nix b/modules/private/system/dilion.nix
index 5c24656..5abaac0 100644
--- a/modules/private/system/dilion.nix
+++ b/modules/private/system/dilion.nix
@@ -1,59 +1,81 @@
 { privateFiles }:
 { config, pkgs, name, lib, ... }:
 {
-  boot.supportedFilesystems = [ "zfs" ];
-  boot.kernelPackages = pkgs.linuxPackages_latest;
+  deployment = {
+    targetUser = "root";
+    targetHost = config.hostEnv.ips.main.ip4;
+    substituteOnDestination = true;
+  };
+  nixpkgs.system = lib.mkOverride 900 "x86_64-linux";
+  boot = {
+    loader = {
+      grub = {
+        version = 2;
+        devices = [ "/dev/sda" "/dev/sdb" "/dev/sdc" "/dev/sdd" ];
+      };
+      timeout = 1;
+    };
+    blacklistedKernelModules = [ "nvidiafb" ];
+    supportedFilesystems = [ "zfs" ];
+    kernelPackages = pkgs.linuxPackages_latest;
+    kernelModules = [ "kvm-intel" ];
+    initrd.availableKernelModules = [ "ahci" "sd_mod" ];
+    initrd.secrets = {
+      "/boot/pass.key" = "/boot/pass.key";
+    };
+    kernel.sysctl."vm.nr_hugepages" = 256; # for xmr-stak
+  };
+  nix.maxJobs = 8;
+  powerManagement.cpuFreqGovernor = "powersave";
+  hardware.enableRedistributableFirmware = true;
+
   myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };
 
+  swapDevices = [ { label = "swap"; } ];
   fileSystems = {
-    "/"           = lib.mkForce { fsType = "zfs"; device = "zpool/root"; };
+    "/"           = { fsType = "zfs"; device = "zpool/root"; };
     "/boot"       = { fsType = "ext4"; device = "/dev/disk/by-uuid/fd1c511e-2bc0-49d5-b8bb-95e7e8c8c816"; };
     "/etc"        = { fsType = "zfs"; device = "zpool/root/etc"; };
     "/home"       = { fsType = "zfs"; device = "zpool/root/home"; };
     "/home/immae" = { fsType = "zfs"; device = "zpool/root/home/immae"; };
     "/tmp"        = { fsType = "zfs"; device = "zpool/root/tmp"; };
     "/var"        = { fsType = "zfs"; device = "zpool/root/var"; };
+    "/data"       = { fsType = "ext4"; label = "data"; };
+    "/nix"        = { fsType = "ext4"; label = "nix"; };
   };
-  boot.initrd.secrets = {
-    "/boot/pass.key" = "/boot/pass.key";
-  };
+
+  services.udev.extraRules = ''
+    ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="10:bf:48:7f:e6:3b", NAME="eth0"
+  '';
 
   networking = {
     hostId = "27c3048d"; # generated with head -c4 /dev/urandom | od -A none -t x4
     firewall.enable = false;
-    interfaces."eth0".ipv4.addresses = pkgs.lib.attrsets.mapAttrsToList
-      (n: ips: { address = ips.ip4; prefixLength = 32; })
-      (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips);
-    interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
-      (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
-      config.hostEnv.ips);
+    interfaces."eth0".ipv4.addresses =
+      [ { address = config.hostEnv.ips.main.ip4; prefixLength = 27; } ]
+      ++ pkgs.lib.attrsets.mapAttrsToList
+        (n: ips: { address = ips.ip4; prefixLength = 32; })
+        (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips);
+    interfaces."eth0".ipv6.addresses =
+      [ { address = "2a01:4f8:141:53e7::"; prefixLength = 64; } ]
+      ++ pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
+        (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
+        config.hostEnv.ips);
+    defaultGateway = { address = "176.9.10.225"; interface = "eth0"; };
+    defaultGateway6 = { address = "fe80::1"; interface = "eth0"; };
+    nameservers = [
+      "213.133.98.98"
+      "213.133.99.99"
+      "213.133.100.100"
+      "2a01:4f8:0:a0a1::add:1010"
+      "2a01:4f8:0:a102::add:9999"
+      "2a01:4f8:0:a111::add:9898"
+    ];
   };
 
   myServices.ssh.modules = [ config.myServices.ssh.predefinedModules.regular ];
   imports = builtins.attrValues (import ../..);
 
-  deployment = {
-    targetEnv = "hetzner";
-    hetzner = {
-      robotUser = config.myEnv.hetzner.user;
-      robotPass = config.myEnv.hetzner.pass;
-      mainIPv4 = config.hostEnv.ips.main.ip4;
-      partitions = ''
-        clearpart --all --initlabel --drives=sda,sdb,sdc,sdd
-
-        part swap --recommended --label=swap --fstype=swap --ondisk=sda
-
-        part raid.1 --grow --ondisk=sdc
-        part raid.2 --grow --ondisk=sdd
-
-        raid / --level=1 --device=md0 --fstype=ext4 --label=root raid.1 raid.2
-
-        part /nix --grow --label=nix --ondisk=sda
-        part /data --grow --label=data --ondisk=sdb
-      '';
-    };
-  };
-
   system.nssModules = [ pkgs.libvirt ];
   system.nssDatabases.hosts = lib.mkForce [ "files" "libvirt_guest" "mymachines" "dns" "myhostname" ];
   programs.zsh.enable = true;
@@ -78,7 +100,6 @@
     }
   ];
 
-  boot.kernel.sysctl."vm.nr_hugepages" = 256; # for xmr-stak
   system.activationScripts.libvirtd_exports = ''
     install -m 0755 -o root -g root -d /var/lib/caldance
   '';
diff --git a/modules/private/system/eldiron.nix b/modules/private/system/eldiron.nix
index 83e52b8..bb8bbfc 100644
--- a/modules/private/system/eldiron.nix
+++ b/modules/private/system/eldiron.nix
@@ -1,22 +1,45 @@
 { privateFiles }:
 { config, pkgs, lib, ... }:
 {
-  boot.supportedFilesystems = [ "zfs" ];
-  boot.kernelParams = ["zfs.zfs_arc_max=6442450944"];
-  boot.kernelPackages = pkgs.linuxPackages_latest;
+  deployment = {
+    targetUser = "root";
+    targetHost = config.hostEnv.ips.main.ip4;
+    substituteOnDestination = true;
+  };
+  boot = {
+    kernelModules = [ "kvm-intel" ];
+    blacklistedKernelModules = [ "nvidiafb" ];
+    loader.timeout = 1;
+    loader.grub.devices = [ "/dev/sda" "/dev/sdb" ];
+    kernel.sysctl = {
+      # https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
+      "net.ipv4.tcp_sack" = 0;
+    };
+    supportedFilesystems = [ "zfs" ];
+    kernelParams = ["zfs.zfs_arc_max=6442450944"];
+    kernelPackages = pkgs.linuxPackages_latest;
+    initrd.availableKernelModules = [ "ahci" "sd_mod" ];
+    initrd.secrets = {
+      "/boot/pass.key" = "/boot/pass.key";
+    };
+  };
+  services.udev.extraRules = ''
+    ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="c8:60:00:56:a0:88", NAME="eth0"
+  '';
+  nix.maxJobs = 8;
+  powerManagement.cpuFreqGovernor = "powersave";
   myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };
 
   fileSystems = {
-    "/"     = lib.mkForce { fsType = "zfs"; device = "zpool/root"; };
+    "/"     = { fsType = "zfs"; device = "zpool/root"; };
     "/boot" = { fsType = "ext4"; device = "/dev/disk/by-uuid/e6bb18fb-ff56-4b5f-ae9f-e60d40dc0622"; };
     "/etc"  = { fsType = "zfs"; device = "zpool/root/etc"; };
     "/nix"  = { fsType = "zfs"; device = "zpool/root/nix"; };
     "/tmp"  = { fsType = "zfs"; device = "zpool/root/tmp"; };
     "/var"  = { fsType = "zfs"; device = "zpool/root/var"; };
   };
-  boot.initrd.secrets = {
-    "/boot/pass.key" = "/boot/pass.key";
-  };
+  swapDevices = [ { label = "swap1"; } { label = "swap2"; } ];
+  hardware.enableRedistributableFirmware = true;
 
   services.zfs = {
     autoScrub = {
@@ -27,20 +50,34 @@
     hostId = "8262ca33"; # generated with head -c4 /dev/urandom | od -A none -t x4
     firewall.enable = true;
     # 176.9.151.89 declared in nixops -> infra / tools
-    interfaces."eth0".ipv4.addresses = pkgs.lib.attrsets.mapAttrsToList
-      (n: ips: { address = ips.ip4; prefixLength = 32; })
-      (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips);
+    interfaces."eth0".ipv4.addresses =
+      pkgs.lib.attrsets.mapAttrsToList
+        (n: ips: { address = ips.ip4; prefixLength = 32; })
+        (pkgs.lib.attrsets.filterAttrs (n: v: n != "main") config.hostEnv.ips)
+      ++ [ { address = config.hostEnv.ips.main.ip4; prefixLength = 27; } ];
     interfaces."eth0".ipv6.addresses = pkgs.lib.flatten (pkgs.lib.attrsets.mapAttrsToList
       (n: ips: map (ip: { address = ip; prefixLength = (if n == "main" && ip == pkgs.lib.head ips.ip6 then 64 else 128); }) (ips.ip6 or []))
       config.hostEnv.ips);
+    defaultGateway = "176.9.151.65";
+    localCommands = ''
+      # FIXME: Those commands were added by nixops and may not be
+      # actually needed
+      ip -6 addr add '2a01:4f8:160:3445::/64' dev 'eth0' || true
+      ip -4 route change '176.9.151.64/27' via '176.9.151.65' dev 'eth0' || true
+      ip -6 route add default via 'fe80::1' dev eth0 || true
+    '';
+    nameservers = [
+      "213.133.98.98"
+      "213.133.99.99"
+      "213.133.100.100"
+      "2a01:4f8:0:a0a1::add:1010"
+      "2a01:4f8:0:a102::add:9999"
+      "2a01:4f8:0:a111::add:9898"
+    ];
   };
 
   imports = builtins.attrValues (import ../..);
 
-  boot.kernel.sysctl = {
-    # https://github.com/Netflix/security-bulletins/blob/master/advisories/third-party/2019-001.md
-    "net.ipv4.tcp_sack" = 0;
-  };
   myServices.buildbot.enable = true;
   myServices.databases.enable = true;
   myServices.gitolite.enable = true;
@@ -76,26 +113,6 @@
       "${profile.host_key_type} ${profile.host_key}";
   };
 
-  deployment = {
-    targetEnv = "hetzner";
-    hetzner = {
-      robotUser = config.myEnv.hetzner.user;
-      robotPass = config.myEnv.hetzner.pass;
-      mainIPv4 = config.hostEnv.ips.main.ip4;
-      partitions = ''
-        clearpart --all --initlabel --drives=sda,sdb
-
-        part swap1 --recommended --label=swap1 --fstype=swap --ondisk=sda
-        part swap2 --recommended --label=swap2 --fstype=swap --ondisk=sdb
-
-        part raid.1 --grow --ondisk=sda
-        part raid.2 --grow --ondisk=sdb
-
-        raid / --level=1 --device=md0 --fstype=ext4 --label=root raid.1 raid.2
-      '';
-    };
-  };
-
   services.cron = {
     enable = true;
     mailto = "cron@immae.eu";
diff --git a/modules/private/system/monitoring-1.nix b/modules/private/system/monitoring-1.nix
index c87c784..2241b2b 100644
--- a/modules/private/system/monitoring-1.nix
+++ b/modules/private/system/monitoring-1.nix
@@ -1,20 +1,15 @@
 { privateFiles }:
 { config, pkgs, resources, ... }:
 {
+  deployment = {
+    targetUser = "root";
+    targetHost = config.hostEnv.ips.main.ip4;
+    substituteOnDestination = true;
+  };
   boot.kernelPackages = pkgs.linuxPackages_latest;
   myEnv = import "${privateFiles}/environment.nix" // { inherit privateFiles; };
 
-  imports = builtins.attrValues (import ../..);
-
-  deployment = {
-    targetEnv = "hetznerCloud";
-    hetznerCloud = {
-      authToken = config.myEnv.hetznerCloud.authToken;
-      datacenter = "hel1-dc2";
-      location  ="hel1";
-      serverType = "cx11";
-    };
-  };
+  imports = [ <nixpkgs/nixos/modules/profiles/qemu-guest.nix> ] ++ builtins.attrValues (import ../..);
 
   myServices.monitoring.enable = true;
   myServices.monitoring.master = true;
@@ -29,6 +24,8 @@
       config.hostEnv.ips);
     defaultGateway6 = { address = "fe80::1"; interface = "ens3"; };
   };
+  boot.loader.grub.device = "nodev";
+  fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
   myServices.mailRelay.enable = true;
 
   security.pki.certificateFiles = [
diff --git a/modules/secrets.nix b/modules/secrets.nix
index a2424e9..a149f02 100644
--- a/modules/secrets.nix
+++ b/modules/secrets.nix
@@ -61,14 +61,13 @@
         fi
         '';
     };
-    deployment.keys."secrets.tar" = {
+    system.extraDependencies = [ secrets ];
+    deployment.secrets."secrets.tar" = {
+      source = "${secrets}";
+      destination = "/run/keys/secrets.tar";
+      owner.user = "root";
+      owner.group = "root";
       permissions = "0400";
-      # keyFile below is not evaluated at build time by nixops, so the
-      # `secrets` path doesn’t necessarily exist when uploading the
-      # keys, and nixops is unhappy.
-      user = "root${builtins.substring 10000 1 secrets}";
-      group = "root";
-      keyFile = "${secrets}";
     };
   };
 }
diff --git a/nixops/Makefile b/nixops/Makefile
index 18d48eb..0bebaf8 100644
--- a/nixops/Makefile
+++ b/nixops/Makefile
@@ -2,8 +2,6 @@ ifndef NIXOPS_ENV_LOADED
   $(error "Please load environment with direnv")
 endif
 
-NIXOPS_PRIV = ./scripts/with_env nixops
-
 ###### Current channel information
 nix-info:
         @version=$$(nix eval --raw nixpkgs.lib.version) && \
@@ -17,100 +15,69 @@ setup:
         ./scripts/setup
 .PHONY: setup
 
-###### Nixops regular tasks
+###### Morph regular tasks
+PROFILE=/nix/var/nix/profiles/per-user/immae/morph/immaeEu
 TARGET ?=
-NIXOPS_ARGS ?=
+MORPH_ARGS ?=
 ifdef TARGET
-  override NIXOPS_ARGS +=--include=$(TARGET)
+  # multiple targets: --on="{machine1,machine2}" (works with * glob too)
+  override MORPH_ARGS +=--on=$(TARGET)
 endif
 SSH_ARGS ?=
 
 edit_env:
         pass edit Nixops/files/environment.nix || true
 
-nixops:
-        $(NIXOPS_PRIV) $(NIXOPS_ARGS)
-
 ssh-eldiron:
-        $(NIXOPS_PRIV) ssh eldiron -- $(SSH_ARGS)
+        ./scripts/with_env bash -c 'ssh -i $$SSH_IDENTITY_FILE root@eldiron $(SSH_ARGS)'
 
 ssh-dilion:
-        $(NIXOPS_PRIV) ssh dilion -- $(SSH_ARGS)
+        ./scripts/with_env bash -c 'ssh -i $$SSH_IDENTITY_FILE root@dilion $(SSH_ARGS)'
 
 ssh-backup-2:
-        $(NIXOPS_PRIV) ssh backup-2 -- $(SSH_ARGS)
+        ./scripts/with_env bash -c 'ssh -i $$SSH_IDENTITY_FILE root@backup-2 $(SSH_ARGS)'
 
 ssh-monitoring-1:
-        $(NIXOPS_PRIV) ssh monitoring-1 -- $(SSH_ARGS)
-
-info:
-        $(NIXOPS_PRIV) list
-        $(NIXOPS_PRIV) info
+        ./scripts/with_env bash -c 'ssh -i $$SSH_IDENTITY_FILE root@monitoring-1 $(SSH_ARGS)'
 
 debug:
-        $(NIXOPS_PRIV) deploy --build-only --show-trace $(NIXOPS_ARGS)
-
-dry-run:
-        $(NIXOPS_PRIV) deploy --dry-run $(NIXOPS_ARGS)
+        ./scripts/with_env morph build --show-trace default.nix $(MORPH_ARGS)
 
 build:
-        $(NIXOPS_PRIV) deploy --build-only $(NIXOPS_ARGS)
+        ./scripts/with_env morph build default.nix $(MORPH_ARGS)
 
 upload:
-        $(NIXOPS_PRIV) deploy --copy-only $(NIXOPS_ARGS)
+        ./scripts/with_env morph push default.nix $(MORPH_ARGS)
 
 deploy:
-        $(NIXOPS_PRIV) deploy $(NIXOPS_ARGS)
+        ./scripts/with_env morph deploy default.nix switch --keep-result --upload-secrets $(MORPH_ARGS)
+        nix-env -p $(PROFILE) --set .gcroots/default.nix
 
 deploy-reboot:
-        $(NIXOPS_PRIV) deploy --force-reboot $(NIXOPS_ARGS)
+        ./scripts/with_env morph deploy default.nix boot --reboot --upload-secrets $(MORPH_ARGS)
 
-reboot:
-        $(NIXOPS_PRIV) reboot --include=$(TARGET)
-.PHONY: nixops ssh-eldiron info debug dry-run build upload deploy deploy-reboot reboot
+.PHONY: ssh-eldiron ssh-dilion ssh-monitoring-1 ssh-backup-2 debug build upload deploy deploy-reboot
 
 ###### Cleanup generations and garbage collection
-profile := $$($(NIXOPS_PRIV) info | grep "^Nix profile: " | sed -e "s/^Nix profile: //")
 GEN ?= "+3"
 
 list-generations:
-        nix-env -p $(profile) --list-generations
-        $(NIXOPS_PRIV) ssh eldiron -- nix-env -p /nix/var/nix/profiles/system --list-generations
+        nix-env -p $(PROFILE) --list-generations
+        $(MAKE) ssh-eldiron SSH_ARGS="nix-env -p /nix/var/nix/profiles/system --list-generations"
 .PHONY: list-generations
 
 delete-generations:
-        nix-env -p $(profile) --delete-generations $(GEN)
-        $(NIXOPS_PRIV) ssh eldiron -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
-        $(NIXOPS_PRIV) ssh dilion -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
-        $(NIXOPS_PRIV) ssh backup-2 -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
-        $(NIXOPS_PRIV) ssh monitoring-1 -- nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)
+        nix-env -p $(PROFILE) --delete-generations $(GEN)
+        $(MAKE) ssh-eldiron SSH_ARGS="nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)"
+        $(MAKE) ssh-dilion SSH_ARGS="nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)"
+        $(MAKE) ssh-backup-2 SSH_ARGS="nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)"
+        $(MAKE) ssh-monitoring-1 SSH_ARGS="nix-env -p /nix/var/nix/profiles/system --delete-generations $(GEN)"
 .PHONY: delete-generations
 
 cleanup: delete-generations
         nix-store --gc
-        $(NIXOPS_PRIV) ssh eldiron -- nix-store --gc
-        $(NIXOPS_PRIV) ssh dilion -- nix-store --gc
-        $(NIXOPS_PRIV) ssh backup-2 -- nix-store --gc
-        $(NIXOPS_PRIV) ssh monitoring-1 -- nix-store --gc
+        $(MAKE) ssh-eldiron SSH_ARGS="nix-store --gc"
+        $(MAKE) ssh-dilion SSH_ARGS="nix-store --gc"
+        $(MAKE) ssh-backup-2 SSH_ARGS="nix-store --gc"
+        $(MAKE) ssh-monitoring-1 SSH_ARGS="nix-store --gc"
 .PHONY: cleanup
-
-###### Pull environment and deployment from remote
-pull_deployment:
-        @if nixops info -d $(NIXOPS_DEPLOYMENT) 2>/dev/null >/dev/null ; then \
-          echo "This will remove your current deployment file and recreate it!. Continue? [y/N]" && \
-          read y && \
-          [ "$$y" = "y" -o "$$y" = "Y" ] && \
-          nixops delete --force -d $(NIXOPS_DEPLOYMENT); \
-        fi
-        pass show Nixops/Deployment | nixops import
-        nixops modify -d $(NIXOPS_DEPLOYMENT) "$$(pwd)/default.nix"
-.PHONY: pull_deployment
-
-deployment_is_set:
-        nixops info -d $(NIXOPS_DEPLOYMENT) 2>/dev/null >/dev/null
-.PHONY: deployment_is_set
-
-###### Push deployment information to password store
-push_deployment:
-        nixops export | pass insert -m Nixops/Deployment
-.PHONY: push
diff --git a/nixops/default.nix b/nixops/default.nix
index 56b86e8..7c6dd38 100644
--- a/nixops/default.nix
+++ b/nixops/default.nix
@@ -1,12 +1,7 @@
-{ privateFiles ? ./. }:
+let
+  privateFiles = <privateFiles>;
+in
 {
-  network = {
-    description = "Immae's network";
-    enableRollback = true;
-  };
-
-  # Used by hetzner cloud to provision machines
-  resources.sshKeyPairs.ssh-key = {};
   dilion = import ../modules/private/system/dilion.nix { inherit privateFiles; };
   eldiron = import ../modules/private/system/eldiron.nix { inherit privateFiles; };
   backup-2 = import ../modules/private/system/backup-2.nix { inherit privateFiles; };
diff --git a/nixops/scripts/setup b/nixops/scripts/setup
index 22f43ce..e161e43 100755
--- a/nixops/scripts/setup
+++ b/nixops/scripts/setup
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-MAKEFILE_DIR="$( cd "$( dirname $( dirname "${BASH_SOURCE[0]}" ))" >/dev/null 2>&1 && pwd )"
-
 if ! which nix 2>/dev/null >/dev/null; then
   cat <<-EOF
         nix is needed, please install it:
@@ -62,20 +60,6 @@ if nix show-config --json | jq -e '.sandbox.value == "true"' >/dev/null; then
   read y
 fi
 
-if ! make -C $MAKEFILE_DIR deployment_is_set 2>/dev/null >/dev/null; then
-  cat <<-EOF
-        Importing deployment file into nixops:
-        Continue? [y/N]
-        EOF
-  read y
-  if [ "$y" = "y" -o "$y" = "Y" ]; then
-    make -C $MAKEFILE_DIR pull_deployment
-  else
-    echo "Aborting"
-    exit 1
-  fi
-fi
-
 cat <<-EOF
         All set up.
         Please make sure you’re using make commands when deploying
diff --git a/nixops/scripts/with_env b/nixops/scripts/with_env
index 9882f78..f8e5537 100755
--- a/nixops/scripts/with_env
+++ b/nixops/scripts/with_env
@@ -5,12 +5,12 @@ if [ -z "$NIXOPS_ENV_LOADED" ]; then
   exit 1;
 fi
 
+umask 0077
 TEMP=$(mktemp -d /tmp/XXXXXX-nixops-files)
 chmod go-rwx $TEMP
 
 finish() {
   rm -rf "$TEMP"
-  nixops set-args --unset privateFiles
 }
 
 trap finish EXIT
@@ -21,6 +21,8 @@ files=$(pass ls Nixops/files | sed -e '1d' -e 's/^.* //')
 for file in $files; do
   pass show "Nixops/files/$file" > $TEMP/$file
 done
-nixops set-args --argstr privateFiles "$TEMP"
+
+export NIX_PATH="privateFiles=$TEMP:$NIX_PATH"
+export SSH_IDENTITY_FILE="$TEMP/id_ed25519"
 
 "$@"
diff --git a/nixops/secrets b/nixops/secrets
-Subproject 79b991028b09aa59f719059de8dc1fba7d6b04f
+Subproject def4146f0e703ca9a32735fbbc3820d6f9bb22e
diff --git a/nixops/state/.gitkeep b/nixops/state/.gitkeep
deleted file mode 100644
index e69de29..0000000
--- a/nixops/state/.gitkeep
+++ /dev/null
diff --git a/overlays/default.nix b/overlays/default.nix
index 0c7c738..7444e15 100644
--- a/overlays/default.nix
+++ b/overlays/default.nix
@@ -32,5 +32,6 @@
   doing = import ./doing;
   khal = import ./khal;
   nix-direnv = import ./nix-direnv;
+  morph = import ./morph;
 }
 // import ./python-packages
diff --git a/overlays/morph/default.nix b/overlays/morph/default.nix
new file mode 100644
index 0000000..ee59e25
--- /dev/null
+++ b/overlays/morph/default.nix
@@ -0,0 +1,5 @@
+self: super: {
+  morph = super.morph.overrideAttrs(old: {
+    patches = (old.patches or []) ++ [ ./verbose_nix.patch ];
+  });
+}
diff --git a/overlays/morph/verbose_nix.patch b/overlays/morph/verbose_nix.patch
new file mode 100644
index 0000000..389a79c
--- /dev/null
+++ b/overlays/morph/verbose_nix.patch
@@ -0,0 +1,12 @@
+diff --git a/nix/nix.go b/nix/nix.go
+index bb63870..7fe04aa 100644
+--- a/nix/nix.go
++++ b/nix/nix.go
+@@ -347,6 +347,7 @@ func Push(ctx *ssh.SSHContext, host Host, paths ...string) (err error) {
+        options := mkOptions(host)
+        for _, path := range paths {
+                args := []string{
++                       "-v",
+                        "copy",
+                        path,
+                        "--to", "ssh://" + userArg + host.TargetHost + keyArg,
diff --git a/shell.nix b/shell.nix
index 70c7604..2a95658 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,4 +1,4 @@
 { pkgs ? import <nixpkgs> { overlays = builtins.attrValues (import ./overlays); } }:
 pkgs.mkShell {
-  buildInputs = [ pkgs.nixops pkgs.niv pkgs.pass pkgs.curl pkgs.shellcheck pkgs.jq pkgs.gnumake ];
+  buildInputs = [ pkgs.morph pkgs.niv pkgs.pass pkgs.curl pkgs.shellcheck pkgs.jq pkgs.gnumake ];
 }