From 272e50d0b0262e49cdcaad42cdab57aad183d1c2 Mon Sep 17 00:00:00 2001 From: goodraven Date: Thu, 3 May 2018 22:24:58 -0700 Subject: [PATCH] Initial commit adding support for hetzner cloud This is based on the digital ocean backend. It also uses nixos-infect. I extended nixos-infect to be generic for both backends. Fixes #855 --- examples/trivial-hetzner-cloud.nix | 12 ++ nix/eval-machine-info.nix | 1 + nix/hetzner-cloud.nix | 56 +++++++ nix/options.nix | 1 + nixops/backends/hetzner_cloud.py | 230 +++++++++++++++++++++++++++++ nixops/data/nixos-infect | 77 +++++++--- 6 files changed, 354 insertions(+), 23 deletions(-) create mode 100644 examples/trivial-hetzner-cloud.nix create mode 100644 nix/hetzner-cloud.nix create mode 100644 nixops/backends/hetzner_cloud.py diff --git a/examples/trivial-hetzner-cloud.nix b/examples/trivial-hetzner-cloud.nix new file mode 100644 index 000000000..c61add6bb --- /dev/null +++ b/examples/trivial-hetzner-cloud.nix @@ -0,0 +1,12 @@ +{ + resources.sshKeyPairs.ssh-key = {}; + + machine = { config, pkgs, ... }: { + services.openssh.enable = true; + + deployment.targetEnv = "hetznerCloud"; + deployment.hetznerCloud.serverType = "cx11"; + + networking.firewall.allowedTCPPorts = [ 22 ]; + }; +} diff --git a/nix/eval-machine-info.nix b/nix/eval-machine-info.nix index 2884b4b47..6a7205786 100644 --- a/nix/eval-machine-info.nix +++ b/nix/eval-machine-info.nix @@ -309,6 +309,7 @@ rec { digitalOcean = optionalAttrs (v.config.deployment.targetEnv == "digitalOcean") v.config.deployment.digitalOcean; gce = optionalAttrs (v.config.deployment.targetEnv == "gce") v.config.deployment.gce; hetzner = optionalAttrs (v.config.deployment.targetEnv == "hetzner") v.config.deployment.hetzner; + hetznerCloud = optionalAttrs (v.config.deployment.targetEnv == "hetznerCloud") v.config.deployment.hetznerCloud; container = optionalAttrs (v.config.deployment.targetEnv == "container") v.config.deployment.container; route53 = v.config.deployment.route53; virtualbox = diff --git a/nix/hetzner-cloud.nix b/nix/hetzner-cloud.nix new file mode 100644 index 000000000..21d148c1a --- /dev/null +++ b/nix/hetzner-cloud.nix @@ -0,0 +1,56 @@ +{ config, pkgs, lib, utils, ... }: + +with utils; +with lib; +with import ./lib.nix lib; + +let + cfg = config.deployment.hetznerCloud; +in +{ + ###### interface + options = { + + deployment.hetznerCloud.authToken = mkOption { + default = ""; + example = "8b2f4e96af3997853bfd4cd8998958eab871d9614e35d63fab45a5ddf981c4da"; + type = types.str; + description = '' + The API auth token. We're checking the environment for + HETZNER_CLOUD_AUTH_TOKEN first and if that is + not set we try this auth token. + ''; + }; + + deployment.hetznerCloud.datacenter = mkOption { + example = "fsn1-dc8"; + default = null; + type = types.nullOr types.str; + description = '' + The datacenter. + ''; + }; + + deployment.hetznerCloud.location = mkOption { + example = "fsn1"; + default = null; + type = types.nullOr types.str; + description = '' + The location. + ''; + }; + + deployment.hetznerCloud.serverType = mkOption { + example = "cx11"; + type = types.str; + description = '' + Name or id of server types. + ''; + }; + }; + + config = mkIf (config.deployment.targetEnv == "hetznerCloud") { + nixpkgs.system = mkOverride 900 "x86_64-linux"; + services.openssh.enable = true; + }; +} diff --git a/nix/options.nix b/nix/options.nix index 0866c3ab8..db021f74d 100644 --- a/nix/options.nix +++ b/nix/options.nix @@ -22,6 +22,7 @@ in ./keys.nix ./gce.nix ./hetzner.nix + ./hetzner-cloud.nix ./container.nix ./libvirtd.nix ]; diff --git a/nixops/backends/hetzner_cloud.py b/nixops/backends/hetzner_cloud.py new file mode 100644 index 000000000..a2cb176b9 --- /dev/null +++ b/nixops/backends/hetzner_cloud.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +""" +A backend for hetzner cloud. + +This backend uses nixos-infect (which uses nixos LUSTRATE) to infect a +hetzner cloud instance. The setup requires two reboots, one for +the infect itself, another after we pushed the nixos image. +""" +import os +import os.path +import time +import socket + +import requests + +import nixops.resources +from nixops.backends import MachineDefinition, MachineState +from nixops.nix_expr import Function, RawValue +import nixops.util +import nixops.known_hosts + +infect_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data', 'nixos-infect')) + +API_HOST = 'api.hetzner.cloud' + +class ApiError(Exception): + pass + +class ApiNotFoundError(ApiError): + pass + +class HetznerCloudDefinition(MachineDefinition): + @classmethod + def get_type(cls): + return "hetznerCloud" + + def __init__(self, xml, config): + MachineDefinition.__init__(self, xml, config) + self.auth_token = config["hetznerCloud"]["authToken"] + self.location = config["hetznerCloud"]["location"] + self.datacenter = config["hetznerCloud"]["datacenter"] + self.server_type = config["hetznerCloud"]["serverType"] + + def show_type(self): + return "{0} [{1}]".format(self.get_type(), self.location or self.datacenter or 'any location') + + +class HetznerCloudState(MachineState): + @classmethod + def get_type(cls): + return "hetznerCloud" + + state = nixops.util.attr_property("state", MachineState.MISSING, int) # override + public_ipv4 = nixops.util.attr_property("publicIpv4", None) + public_ipv6 = nixops.util.attr_property("publicIpv6", None) + location = nixops.util.attr_property("hetznerCloud.location", None) + datacenter = nixops.util.attr_property("hetznerCloud.datacenter", None) + server_type = nixops.util.attr_property("hetznerCloud.serverType", None) + auth_token = nixops.util.attr_property("hetznerCloud.authToken", None) + server_id = nixops.util.attr_property("hetznerCloud.serverId", None, int) + + def __init__(self, depl, name, id): + MachineState.__init__(self, depl, name, id) + self.name = name + + def get_ssh_name(self): + return self.public_ipv4 + + def get_ssh_flags(self, *args, **kwargs): + super_flags = super(HetznerCloudState, self).get_ssh_flags(*args, **kwargs) + return super_flags + [ + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + '-i', self.get_ssh_private_key_file(), + ] + + def get_physical_spec(self): + return Function("{ ... }", { + 'imports': [ RawValue('') ], + ('boot', 'loader', 'grub', 'device'): 'nodev', + ('fileSystems', '/'): { 'device': '/dev/sda1', 'fsType': 'ext4'}, + ('users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'): [self.depl.active_resources.get('ssh-key').public_key], + }) + + def get_ssh_private_key_file(self): + return self.write_ssh_private_key(self.depl.active_resources.get('ssh-key').private_key) + + def create_after(self, resources, defn): + # make sure the ssh key exists before we do anything else + return { + r for r in resources if + isinstance(r, nixops.resources.ssh_keypair.SSHKeyPairState) + } + + def get_auth_token(self): + return os.environ.get('HETZNER_CLOUD_AUTH_TOKEN', self.auth_token) + + def _api(self, path, method=None, data=None, json=True): + """Basic wrapper around requests that handles auth and serialization.""" + assert path[0] == '/' + url = 'https://%s%s' % (API_HOST, path) + token = self.get_auth_token() + if not token: + raise Exception('No hetzner cloud auth token set') + headers = { + 'Authorization': 'Bearer '+self.get_auth_token(), + } + res = requests.request( + method=method, + url=url, + json=data, + headers=headers) + + if res.status_code == 404: + raise ApiNotFoundError('Not Found: %r' % path) + elif not res.ok: + raise ApiError('Response for %s %s has status code %d: %s' % (method, path, res.status_code, res.content)) + if not json: + return + try: + res_data = res.json() + except ValueError as e: + raise ApiError('Response for %s %s has invalid JSON (%s): %r' % (method, path, e, res.content)) + return res_data + + + def destroy(self, wipe=False): + if not self.server_id: + self.log('server {} was never made'.format(self.name)) + return + self.log('destroying server {} with id {}'.format(self.name, self.server_id)) + try: + res = self._api('/v1/servers/%s' % (self.server_id), method='DELETE') + except ApiNotFoundError: + self.log("server not found - assuming it's been destroyed already") + + self.public_ipv4 = None + self.server_id = None + + return True + + def _create_ssh_key(self, public_key): + """Create or get an ssh key and return an id.""" + public_key = public_key.strip() + res = self._api('/v1/ssh_keys', method='GET') + name = 'nixops-%s-%s' % (self.depl.uuid, self.name) + deletes = [] + for key in res['ssh_keys']: + if key['public_key'].strip() == public_key: + return key['id'] + if key['name'] == name: + deletes.append(key['id']) + for d in deletes: + # This reply is empty, so don't decode json. + self._api('/v1/ssh_keys/%d' % d, method='DELETE', json=False) + res = self._api('/v1/ssh_keys', method='POST', data={ + 'name': name, + 'public_key': public_key, + }) + return res['ssh_key']['id'] + + def create(self, defn, check, allow_reboot, allow_recreate): + ssh_key = self.depl.active_resources.get('ssh-key') + if ssh_key is None: + raise Exception('Please specify a ssh-key resource (resources.sshKeyPairs.ssh-key = {}).') + + self.set_common_state(defn) + + if self.server_id is not None: + return + + ssh_key_id = self._create_ssh_key(ssh_key.public_key) + + req = { + 'name': self.name, + 'server_type': defn.server_type, + 'start_after_create': True, + 'image': 'debian-9', + 'ssh_keys': [ + ssh_key_id, + ], + } + + if defn.datacenter: + req['datacenter'] = defn.datacenter + elif defn.location: + req['location'] = defn.location + + self.log_start("creating server ...") + create_res = self._api('/v1/servers', method='POST', data=req) + self.server_id = create_res['server']['id'] + self.public_ipv4 = create_res['server']['public_net']['ipv4']['ip'] + self.public_ipv6 = create_res['server']['public_net']['ipv6']['ip'] + self.datacenter = create_res['server']['datacenter']['name'] + self.location = create_res['server']['datacenter']['location']['name'] + + action = create_res['action'] + action_path = '/v1/servers/%d/actions/%d' % (self.server_id, action['id']) + + while action['status'] == 'running': + time.sleep(1) + res = self._api(action_path, method='GET') + action = res['action'] + + if action['status'] != 'success': + raise Exception('unexpected status: %s' % action['status']) + + self.log_end("{}".format(self.public_ipv4)) + + self.wait_for_ssh() + self.log_start("running nixos-infect") + self.run_command('bash &1', stdin=open(infect_path)) + self.reboot_sync() + + def reboot(self, hard=False): + if hard: + self.log("sending hard reset to server...") + res = self._api('/v1/servers/%d/actions/reset' % self.server_id, method='POST') + action = res['action'] + action_path = '/v1/servers/%d/actions/%d' % (self.server_id, action['id']) + while action['status'] == 'running': + time.sleep(1) + res = self._api(action_path, method='GET') + action = res['action'] + if action['status'] != 'success': + raise Exception('unexpected status: %s' % action['status']) + self.wait_for_ssh() + self.state = self.STARTING + else: + MachineState.reboot(self, hard=hard) diff --git a/nixops/data/nixos-infect b/nixops/data/nixos-infect index 66634357b..437a2ec61 100644 --- a/nixops/data/nixos-infect +++ b/nixops/data/nixos-infect @@ -68,26 +68,49 @@ makeConf() { } EOF # (nixos-generate-config will add qemu-user and bind-mounts, so avoid) + local disk + if [ -e /dev/sda ]; then + disk=/dev/sda + else + disk=/dev/vda + fi cat > /etc/nixos/hardware-configuration.nix << EOF { ... }: { imports = [ ]; - boot.loader.grub.device = "/dev/vda"; - fileSystems."/" = { device = "/dev/vda1"; fsType = "ext4"; }; + boot.loader.grub.device = "${disk}"; + fileSystems."/" = { device = "${disk}1"; fsType = "ext4"; }; } EOF local IFS=$'\n' - ens3_ip4s=($(ip address show dev eth0 | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) - ens3_ip6s=($(ip address show dev eth0 | grep 'inet6 .*global' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) - ens4_ip4s=($(ip address show dev eth1 | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) - ens4_ip6s=($(ip address show dev eth1 | grep 'inet6 .*global' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) - gateway=($(ip route show dev eth0 | grep default | sed -r 's|default via ([0-9.]+).*|\1|')) - gateway6=($(ip -6 route show dev eth0 | grep default | sed -r 's|default via ([0-9a-f:]+).*|\1|')) - ether0=($(ip address show dev eth0 | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')) - ether1=($(ip address show dev eth1 | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')) + gateway=($(ip route show | grep default | sed -r 's|default via ([0-9.]+).*|\1|')) + gateway6=($(ip -6 route show | grep default | sed -r 's|default via ([0-9a-f:]+).*|\1|')) + interfaces=($(ip link | awk -F ': ' '/^[0-9]*: / {if ($2 != "lo") {print $2}}')) nameservers=($(grep ^nameserver /etc/resolv.conf | cut -f2 -d' ')) + # Predict the predictable name for each interface since that is enabled in + # the nixos system. + declare -A predictable_names + for interface in ${interfaces[@]}; do + # udevadm prints out the candidate names which will be selected if + # available in this order. + local name=$(udevadm info /sys/class/net/$interface | awk -F = ' + /^E: ID_NET_NAME_FROM_DATABASE=/ {arr[1]=$2} + /^E: ID_NET_NAME_ONBOARD=/ {arr[2]=$2} + /^E: ID_NET_NAME_SLOT=/ {arr[3]=$2} + /^E: ID_NET_NAME_PATH=/ {arr[4]=$2} + /^E: ID_NET_NAME_MAC=/ {arr[5]=$2} + END {for (i=1;i<6;i++) {if (length(arr[i]) > 0) { print arr[i]; break}}}') + if [ -z "$name" ]; then + echo Could not determine predictable name for interface $interface + fi + predictable_names[$interface]=$name + done + + # Take a gamble on the first interface being able to reach the gateway. + local default_interface=${predictable_names[${interfaces[0]}]} + cat > /etc/nixos/networking.nix << EOF { ... }: { # This file was populated at runtime with the networking @@ -96,25 +119,27 @@ EOF nameservers = [$(for a in ${nameservers[@]}; do echo -n " \"$a\""; done) ]; - defaultGateway = "${gateway}"; - defaultGateway6 = "${gateway6}"; + defaultGateway = {address = "${gateway}"; interface = "${default_interface}";}; + defaultGateway6 = {address = "${gateway6}"; interface = "${default_interface}";}; interfaces = { - ens3 = { - ip4 = [$(for a in ${ens3_ip4s[@]}; do echo -n " - $a"; done) - ]; - ip6 = [$(for a in ${ens3_ip6s[@]}; do echo -n " - $a"; done) - ]; - }; - ens4 = { - ip4 = [$(for a in ${ens4_ip4s[@]}; do echo -n " +EOF + + for interface in ${interfaces[@]}; do + ip4s=($(ip address show dev $interface | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) + ip6s=($(ip address show dev $interface | grep 'inet6 .*global' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) + cat >> /etc/nixos/networking.nix << EOF + ${predictable_names[$interface]} = { + ip4 = [$(for a in ${ip4s[@]}; do echo -n " $a"; done) ]; - ip6 = [$(for a in ${ens4_ip6s[@]}; do echo -n " + ip6 = [$(for a in ${ip6s[@]}; do echo -n " $a"; done) ]; }; +EOF + done + + cat >> /etc/nixos/networking.nix << EOF }; }; } @@ -154,6 +179,12 @@ export HOME="/root" groupadd -r nixbld -g 30000 seq 1 10 | xargs -I{} useradd -c "Nix build user {}" -d /var/empty -g nixbld -G nixbld -M -N -r -s `which nologin` nixbld{} +if ! which curl >/dev/null 2>/dev/null; then + if which apt-get >/dev/null 2>/dev/null; then + apt-get update && apt-get install -y curl + fi +fi + curl https://nixos.org/nix/install | sh source ~/.nix-profile/etc/profile.d/nix.sh