diff --git a/common/cascade-networking.nix b/common/cascade-networking.nix index 25e88ca..883e84c 100644 --- a/common/cascade-networking.nix +++ b/common/cascade-networking.nix @@ -1,21 +1,43 @@ +with builtins; +with import (toString ../functions); + {config, lib, ...}: { options = with lib; with types; { cascade.bridge-interface = mkOption { - type = str; + type = nullOr str; + default = null; description = "interface on which to create primary bridge (br0)"; }; -}; + cascade.bridge-mac-prefix = mkOption { + type = str; + description = "prefix of mac addresses generated for primary bridge (br0)"; + }; +}; config = with lib; { - networking = { + cascade.bridge-mac-prefix = mkDefault "02:${name-to-mac 2 config.networking.domain}"; + + networking = mkIf (config.cascade.bridge-interface != null) { useNetworkd = mkForce true; bridges.br0.interfaces = [config.cascade.bridge-interface]; interfaces.br0.useDHCP = mkImageMediaOverride true; + interfaces.br0.macAddress = mkImageMediaOverride "${config.cascade.bridge-mac-prefix}:${name-to-mac 3 config.networking.hostName}"; + + # also work around an issue with ipv6 and reverse lookups + hosts = lib.mkForce { + "127.0.1.1" = ["${config.networking.hostName}.${config.networking.domain}" config.networking.hostName]; + }; }; - #systemd.network.links."05-br0".matchConfig.Name = "br0"; - #systemd.network.links."05-br0".linkConfig.MACAddressPolicy = "none"; + + #services.resolved.enable = mkForce false; + + # let's just do this here to be sure... + system.activationScripts.cascade-networking-domainname = '' + domainname "${config.networking.domain}" + hostname "${config.networking.hostName}" + ''; }; } diff --git a/common/cascade-source.nix b/common/cascade-source.nix new file mode 100644 index 0000000..9e65d9c --- /dev/null +++ b/common/cascade-source.nix @@ -0,0 +1,36 @@ +with builtins; +with import (toString ../functions); + +{config, lib, ...}: { + +options = with lib; with types; { + environment.cascade-source.enable = mkOption { + default = true; + type = bool; + description = '' + Include cascade source code configured for nixos-rebuild. + ''; + }; +}; + +config = with lib; { + system.activationScripts.cascade-source = mkMerge [ + (mkIf config.environment.cascade-source.enable '' + mkdir -m 0755 -p /usr/src /etc/nixos + [ -h /usr/src/cascade ] && rm /usr/src/cascade # do this first so the dir test that comes next won't read the symlink as a dir + [ -d /usr/src/cascade ] && mv /usr/src/cascade /usr/src/cascade.before-nixos + ln -sfn ${./..} /usr/src/cascade # but why isn't this atomic? + ln -sf /usr/src/cascade/hosts/${config.networking.hostName}.nix /etc/nixos/configuration.nix + '') + (mkIf (!config.environment.cascade-source.enable) '' + # if we delete the symlink version of cascade, we delete a symlink in /etc/nixos as well + [ -h /usr/src/cascade ] && rm /usr/src/cascade && \ + [ -h /etc/nixos/configuration.nix ] && rm /etc/nixos/configuration.nix + '') + ]; + nix.nixPath = [ + "nixos-config=/etc/nixos/configuration.nix" + ]; +}; + +} diff --git a/hosts/_basic.nix b/hosts/_basic.nix index 6381be9..7aca0c2 100644 --- a/hosts/_basic.nix +++ b/hosts/_basic.nix @@ -1,10 +1,14 @@ { imports = [ - (toString ../profiles/qemu-vm) + (toString ../profiles/qemu-vm-install-media) ]; config = { #deployment.targetHost = ""; #deployment.targetUser = ""; #deployment.targetPort = ""; + networking.hostName = "nixos"; + #networking.interfaces.br0.ipv4.addresses = [ ]; + #networking.interfaces.br0.ipv4.routes = [ {address = "0.0.0.0"; prefixLength = 0; via = "172.16.1.1"; } ]; + #networking.interfaces.br0.useDHCP = false; }; } diff --git a/network.nix b/network.nix index b1e108e..a02d3d0 100644 --- a/network.nix +++ b/network.nix @@ -1,7 +1,7 @@ with builtins; with import ./functions; -let pkgs = import {}; +let pkgs = (import "${import ./nixpkgs-path.nix}" {}); network = { inherit pkgs; description = "cascade"; diff --git a/nixpkgs-path.nix b/nixpkgs-path.nix new file mode 100644 index 0000000..ac4d545 --- /dev/null +++ b/nixpkgs-path.nix @@ -0,0 +1,15 @@ +let pkgs = import {}; +in + +if pkgs == null then +builtins.fetchGit { + url = "https://github.com/nixos/nixpkgs"; + ref = "master"; + rev = "da4c6be0187a694bdeb3efc28b29ee0e4c30702f"; + shallow = true; +} +else pkgs.fetchgit { + url = "https://github.com/nixos/nixpkgs"; + rev = "da4c6be0187a694bdeb3efc28b29ee0e4c30702f"; + sha256 = "sha256-1rcG6x0vKnnzGhABPg/QvL75DzhJxM810wZKAukoF1M="; +} diff --git a/profiles/api/default.nix b/profiles/api/default.nix index be4de35..dceb8e6 100644 --- a/profiles/api/default.nix +++ b/profiles/api/default.nix @@ -41,8 +41,7 @@ boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - #networking.meth.meth0.replaceInterface = "enp1s0"; - # cascade.bridge-interface = "enp1s0"; + systemd.network.wait-online.ignoredInterfaces = lib.mkDefault [ "wlp0s20u3" ]; }; } diff --git a/profiles/base/default.nix b/profiles/base/default.nix index eb4612e..48fad46 100644 --- a/profiles/base/default.nix +++ b/profiles/base/default.nix @@ -1,4 +1,4 @@ -{pkgs,...}: +{pkgs ? (import "${import ../../nixpkgs-path.nix}" {}), ...}: { imports = [ (toString ../../common) @@ -7,23 +7,29 @@ # used for deployment. This is done automatically with shell.nix. ]; config = { - environment.systemPackages = with pkgs; [ bridge-utils ]; + environment.systemPackages = with pkgs; [ bridge-utils git ]; programs.neovim.enable = true; programs.neovim.vimAlias = true; programs.neovim.viAlias = true; + networking.domain = "cascade.strudelline.net"; + networking.search = [ "cascade.strudelline.net" "strudelline.net" ]; + networking.nameservers = [ "172.16.44.1" "172.16.1.1" ]; + cascade.bridge-mac-prefix = "00:80:10"; + environment.binbash.enable = true; - services.getty.autologinUser = "root"; + services.getty.autologinUser = pkgs.lib.mkForce "root"; services.sshd.enable = true; networking.firewall.allowedTCPPorts = [ 22 ]; system.stateVersion = "22.11"; + nix.nixPath = with pkgs; [ - "nixpkgs=/usr/src/nixpkgs" - "home-manager=/usr/src/nixpkgs" + "nixpkgs=${ import ../../nixpkgs-path.nix }" + "home-manager=/usr/src/home-manager" "morph-options=${morph.lib}/options.nix" ]; }; diff --git a/profiles/qemu-vm-install-media/default.nix b/profiles/qemu-vm-install-media/default.nix new file mode 100644 index 0000000..ea4e294 --- /dev/null +++ b/profiles/qemu-vm-install-media/default.nix @@ -0,0 +1,10 @@ +{lib, config, ...}: + +with lib; +{ + imports = [ (toString ../qemu-vm) ]; + + config = { + cascade.bridge-interface = mkForce null; # let it come up with its default interface with dhcp first + }; +} diff --git a/profiles/qemu-vm/default.nix b/profiles/qemu-vm/default.nix index 1169f84..2842a63 100644 --- a/profiles/qemu-vm/default.nix +++ b/profiles/qemu-vm/default.nix @@ -6,7 +6,7 @@ with lib; config = { deployment.targetUser = lib.mkDefault "root"; - cascade.bridge-interface = "ens18"; + cascade.bridge-interface = mkImageMediaOverride "ens18"; fileSystems."/" = { device = "/dev/disk/by-label/nixos"; fsType = "ext4"; diff --git a/shell.nix b/shell.nix index 838393e..4302a48 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ -{ pkgs ? import {}, ... }: +{ pkgs ? (import (import ./nixpkgs-path.nix) {}), ... }: let vault_addr = "http://vault:8200"; @@ -10,16 +10,17 @@ with lib; stdenv.mkDerivation { name = "commands-nix"; - buildInputs = [ terraform vault coreutils-full dig bash samba4Full morph ]; + buildInputs = [ terraform vault gnused coreutils-full dig bash samba4Full morph xxd ]; shellHook = '' - export NIX_PATH="nixpkgs=${toString }:morph-options=${morph.lib}/options.nix" + export NIX_PATH="nixpkgs=${import ./nixpkgs-path.nix}:morph-options=${morph.lib}/options.nix" export LD_LIBRARY_PATH="${libvirt}/lib:$LD_LIBRARY_PATH" export VAULT_ADDR=${escapeShellArg vault_addr} export DOMAIN="$(hostname -d)" PROXMOX_STORAGE=hdd-fs MEMORY=2048 +NETWORK_PREFIX_LENGTH=12 rebuild-nixos-image() { echo "$(nix-build custom-image.nix)/nixos".img @@ -33,8 +34,11 @@ mkvirt() { 1>&2 echo "there is already a file at ./hosts/$name"".nix. move it or remove it." return 1 fi - cp "./hosts/_basic.nix" "./hosts/""$name"".nix" - morph build network.nix --on="$name" & + + # free-ips.txt contains fre IPs, one per line + # this is currently not being used in favor of DHCP towards the end + #IPALLOC="$(head -1 free-ips.txt)" + #sed -i -e '1d' free-ips.txt [ -f "result/nixos.img" ] || ( 1>&2 echo "you do not seem to have a result/nixos.img file. building one now." @@ -61,33 +65,49 @@ mkvirt() { --start 1 [ $? -eq 0 ] || return 7 - IP="$(wait-for-vm-ipv4 $VMID)" - [ $? -eq 0 ] || return 8 - samba-create-ipv4-records "$IP" "$name" "$DOMAIN" - [ $? -eq 0 ] || return 9 - 1>&2 echo "removing any stale ssh keys" - ssh-keygen -f "$HOME/.ssh/known_hosts" -R "$name" - ssh-keygen -f "$HOME/.ssh/known_hosts" -R "$name.$DOMAIN" - ssh-keygen -f "$HOME/.ssh/known_hosts" -R "$IP" - 1>&2 echo "scanning for new ssh keys" - # head -1 allows us to grab the ed25519 key before the rsa key. - # at worst, it will yield a single key of some sort which is also fine. - ( - echo - ssh-keyscan "$IP" | sort | head -1 - echo - ssh-keyscan "$name" | sort | head -1 - echo - ssh-keyscan "$name.$DOMAIN" | sort | head -1 - echo - ) 2> /dev/null >> ~/.ssh/known_hosts + IP="$(wait-for-vm-ipv4 $VMID)" ; [ $? -eq 0 ] || return 8 + + ssh-rescan "$IP" + + sed -e ' + s#profiles/qemu-vm-install-media#profiles/qemu-vm#g + s#networking[.]hostName = "nixos";#networking.hostName = "'"$name"'";#g + ' "./hosts/_basic.nix" > "./hosts/""$name"".nix" + + 1>&2 echo "performing initial deployment to temporary IP: $IP" + rsync -a ./ "$IP":/tmp/cascade/ + guestbash $VMID 'nixos-rebuild -I nixos-config=/tmp/cascade/hosts/'"$name"'.nix switch && systemctl restart systemd-networkd' + + 1>&2 echo "retrieving permanent IP" + IP="$(wait-for-vm-ipv4 $VMID)" ; [ $? -eq 0 ] || return 8 + samba-create-ipv4-records "$IP" "$name" "$DOMAIN" ; [ $? -eq 0 ] || return 9 + + ssh-rescan "$name" "$name.$DOMAIN" "$IP" - 1>&2 echo "waiting for morph build to finish" - wait 1>&2 echo "morphing host" morph deploy network.nix switch --on="$name" } +file-ends-with-newline() { + [ x"$(xxd -ps "$1" | sed -E -e '$! d' -e 's/.*(..)$/\1/')" = x"0a" ] +} + +ensure-file-ends-with-newline() { + file-ends-with-newline "$1" || echo >> "$1" +} + +ssh-rescan() { + 1>&2 echo "ensuring known_hosts ends with a newline" + ensure-file-ends-with-newline "$HOME/.ssh/known_hosts" + + while [ $# -gt 0 ];do + 1>&2 echo "replacing any stale ssh keys that can be found" + ssh-keygen -f "$HOME/.ssh/known_hosts" -R "$1" 2> /dev/null || true + ssh-keyscan "$1" 2> /dev/null | sort | head -1 >> "$HOME/.ssh/known_hosts" + shift + done +} + samba-create-ipv4-records() { eval "$(env_cascade)" IP="$1" @@ -108,7 +128,6 @@ samba-create-ipv4-records() { } wait-for-vm-ipv4() { -set -x VMID="$1" while true;do IP="$( sudo qm guest cmd $VMID network-get-interfaces | \ @@ -126,7 +145,6 @@ set -x fi sleep 1 done - set +x } env_cascade() { @@ -187,7 +205,7 @@ destroy-host() { while [ $# -gt 0 ];do eval "$(env_name "$1")" - VMID="$(sudo qm list | awk -v N="$1" '$2 == N {print $name}')" + VMID="$(sudo qm list | awk -v N="$1" '$2 == N {print $1}')" echo "destroying hosts/$1.nix and VM #$VMID" 1>&2 sleep 1 @@ -215,7 +233,7 @@ You may also register an existing host in DNS using: Then morph may be used to deploy: - # Create a hosts/hostname-here.nix file based on hosts/basic.nix.sample + # Create a hosts/hostname-here.nix file based on hosts/_basic.nix deploy test # this is an alias which runs morph on network.nix. deploy switch # the argument is the same as to morph deploy and nixos-rebuild