add support for config.boot.initrd.systemd.enable

.vscode/settings.json

@@ -1,4 +1,9 @@
 {
+	"[nix]": {
+		"editor.detectIndentation": true,
+		"editor.insertSpaces": true,
+		"editor.tabSize": 4,
+	},
 	"markdown.validate.ignoredLinks": [
 		"./modules/",
 		"./patches/",
@@ -11,6 +16,7 @@
 		"acltype", // zfs
 		"acpi", // abbr
 		"ahci", // abbr
+		"asDropin", // systemd
 		"ashift", // zfs
 		"askpass", // program
 		"attrset", "attrsets", // nix/abbr (attribute set)
@@ -40,6 +46,7 @@
 		"createrawvmdk", // virtual box
 		"createvm", // virtual box
 		"cryptsetup", // program
+		"crypttab", // linux
 		"dedup", // zfs
 		"devs", // abbr (devices)
 		"diffutils", // package
@@ -61,12 +68,14 @@
 		"fetchurl", // nix function
 		"fido2", // protocol
 		"filesystems", // plural
+		"findutils", // package
 		"fmask", // mount
 		"foldl", // nix (fold left)
 		"foldr", // nix (fold right)
 		"gcroots", // Nix
 		"gdisk", // program
 		"getsize64", // cli arg
+		"getty", // program
 		"gnugrep", // package
 		"gnused", // package
 		"gollenstede", // name
@@ -169,10 +178,14 @@
 		"stdenv", // nix
 		"storageattach", // virtual box
 		"swsuspend", // parameter
+		"sysinit", // systemd
 		"syslinux", // package
+		"sysroot", // systemd
 		"sysrq", // linux
+		"sysusr", // systemd
 		"temproot", // abbr (temporary root (FS))
 		"timesync", // systemd
+		"tmpdir", // abbr
 		"TMPDIR", // env var
 		"tmpfiles", // nixos option
 		"tmpfs", // linux

hosts/example.nix.md

@@ -26,7 +26,7 @@ dirname: inputs: { config, pkgs, lib, name, ... }: let lib = inputs.self.lib.__i
     #suffix = builtins.head (builtins.match ''example-(.*)'' name); # make differences in config based on this when using »preface.instances«
     hash = builtins.substring 0 8 (builtins.hashString "sha256" name);
 in { preface = { # (any »preface« options have to be defined here)
-    instances = [ "example-explicit" "example" "example-minimal" "example-raidz" ]; # Generate multiple variants of this host, with these »name«s.
+    instances = [ "example-explicit" "example" "example-minimal" "example-crypt" "example-raidz" ]; # Generate multiple variants of this host, with these »name«s.
 }; imports = [ ({ ## Hardware
 
     nixpkgs.hostPlatform = "x86_64-linux"; system.stateVersion = "22.05";
@@ -38,6 +38,8 @@ in { preface = { # (any »preface« options have to be defined here)
     # Example of adding and/or overwriting setup/maintenance functions:
     #installer.scripts.install-overwrite = { path = ../example/install.sh.md; order = 1500; };
 
+    boot.initrd.systemd.enable = true;
+
 
 }) (lib.mkIf (name == "example-explicit") { ## Minimal explicit FS setup
 
@@ -96,6 +98,25 @@ in { preface = { # (any »preface« options have to be defined here)
     setup.temproot.remote.type = "none";
 
 
+}) (lib.mkIf (name == "example-crypt") { ## Minimal automatic FS setup
+
+    setup.keystore.enable = true;
+    setup.keystore.keys."luks/keystore-${hash}/0" = "random"; # (this makes little practical sense)
+    setup.keystore.keys."luks/keystore-${hash}/1" = "constant=insecure"; # static password: "insecure"
+    #setup.keystore.keys."luks/keystore-${hash}/2" = "password"; # password prompted at installation
+    #setup.keystore.keys."luks/rpool-${hash}/0" = "random";
+    setup.temproot = {
+        enable = true;
+        temp.type = "zfs"; local.type = "zfs"; remote.type = "zfs";
+        #temp.type = "zfs"; local.type = "zfs"; remote.type = "none";
+        #local.bind.base = "f2fs"; remote.type = "none";
+        swap = { size = "2G"; asPartition = true; encrypted = true; };
+    };
+    setup.keystore.unlockMethods.pinThroughYubikey = true;
+    #setup.keystore.keys."zfs/rpool-${hash}/remote" = null;
+    #setup.keystore.keys."luks/rpool-${hash}/0" = "random";
+
+
 }) (lib.mkIf (name == "example-raidz") { ## Multi-disk ZFS setup
 
     boot.loader.extlinux.enable = lib.mkForce false; # use UEFI boot this time
@@ -133,6 +154,13 @@ in { preface = { # (any »preface« options have to be defined here)
 
     services.getty.autologinUser = "root"; users.users.root.password = "root";
 
-    boot.kernelParams = [ /* "console=tty1" */ "console=ttyS0" "boot.shell_on_fail" ];
+    boot.kernelParams = [ /* "console=tty1" */ "console=ttyS0" "boot.shell_on_fail" ]; # [ "rd.systemd.unit=emergency.target" ]; # "rd.systemd.debug_shell" "rd.systemd.debug-shell=1"
+    boot.initrd.systemd.emergencyAccess = true;
+
+    systemd.extraConfig = "StatusUnitFormat=name"; # Show unit names instead of descriptions during boot.
+    boot.initrd.systemd.extraConfig = "StatusUnitFormat=name";
+
+    boot.loader.timeout = lib.mkDefault 1; # save 4 seconds on startup
+
 
 }) ]; }

lib/setup-scripts/maintenance.sh

@@ -183,15 +183,14 @@ function run-qemu {
 }
 
 declare-command add-bootkey-to-keydev blockDev << 'EOD'
-Creates a random static key on a new key partition on the GPT partitioned »$blockDev«. The drive can then be used as headless but removable disk unlock method.
+Creates a random static key on a new key partition on the GPT partitioned »$blockDev«. The drive can then be used as headless but removable disk unlock method (»usbPartition«/»usb-part«).
 To create/clear the GPT beforehand, run: $ sgdisk --zap-all "$blockDev"
 EOD
 function add-bootkey-to-keydev {
-    local blockDev=$1 ; local hostHash=${2:-@{config.networking.hostName!hashString.sha256}}
+    local blockDev=$1 ; local bootkeyPartlabel=bootkey-@{config.networking.hostName!hashString.sha256:0:8}
-    local bootkeyPartlabel=bootkey-${hostHash:0:8}
+    @{native.gptfdisk}/bin/sgdisk --new=0:0:+1 --change-name=0:"$bootkeyPartlabel" --typecode=0:8301 "$blockDev" || return # create new 1 sector (512b) partition
-    @{native.gptfdisk}/bin/sgdisk --new=0:0:+1 --change-name=0:"$bootkeyPartlabel" --typecode=0:0000 "$blockDev" || exit # create new 1 sector (512b) partition
+    @{native.parted}/bin/partprobe "$blockDev" && @{native.systemd}/bin/udevadm settle -t 15 || return # wait for partitions to update
-    @{native.parted}/bin/partprobe "$blockDev" && @{native.systemd}/bin/udevadm settle -t 15 || exit # wait for partitions to update
+    { </dev/urandom tr -dc 0-9a-f || true ; } | head -c 512 >/dev/disk/by-partlabel/"$bootkeyPartlabel" || return
-    </dev/urandom tr -dc 0-9a-f | head -c 512 >/dev/disk/by-partlabel/"$bootkeyPartlabel" || exit
 }
 
 declare-command mount-keystore-luks cryptsetupOptions... << 'EOD'

modules/filesystems/pre-mount-commands.nix.md

@@ -1,6 +1,6 @@
 /*
 
-# `fileSystems.*.preMountCommands`
+# `fileSystems.*.preMountCommands`/`.postUnmountCommands`
 
 ## Implementation
 
@@ -14,7 +14,6 @@ in {
             preMountCommands = lib.mkOption { description = ''
                 Commands to be run as root every time before mounting this filesystem **via systemd**, but after all its dependents were mounted.
                 This does not order itself before or after `systemd-fsck@''${utils.escapeSystemdPath device}.service`.
-                This is not implemented for mounts in the initrd (those that are `neededForBoot`) yet.
                 Note that if a symlink exists at a mount point when systemd's fstab-generator runs, it will read/resolve the symlink and use the link's target as the mount point, resulting in mismatching unit names for that mount, effectively disabling its `.preMountCommands`.
                 This does not (apparently and unfortunately) run when mounting via the `mount` command (and probably not with the `mount` system call either).
             ''; type = lib.types.lines; default = ""; };
@@ -26,29 +25,34 @@ in {
     }; };
 
     config = let
-    in ({
 
-        assertions = lib.mapAttrsToList (name: fs: {
+        assertions = lib.mkIf (!config.boot.initrd.systemd.enable) (lib.mapAttrsToList (name: fs: {
             assertion = (fs.preMountCommands == "") || (!utils.fsNeededForBoot fs);
-            message = ''The filesystem "${name}" has `.preMountCommands` but is also (possibly implicitly) `.neededForBoot`. This is not currently supported.'';
+            message = ''The filesystem "${name}" has `.preMountCommands` but is also (possibly implicitly) `.neededForBoot`. This is not supported without `boot.initrd.systemd.enable`.'';
-        }) config.fileSystems;
+        }) config.fileSystems);
 
         # The implementation is derived from the "mkfs-${device'}" service in nixpkgs.
-        systemd.services = lib.fun.mapMergeUnique (_: args@{ mountPoint, device, depends, ... }: if (args.preMountCommands != "") || (args.postUnmountCommands != "") then let
+        services = initrd: lib.fun.mapMergeUnique (_: fs@{ mountPoint, device, depends, ... }: if
+            (fs.preMountCommands != "" || fs.postUnmountCommands != "") && initrd == utils.fsNeededForBoot fs
+        then let
             isDevice = lib.fun.startsWith "/dev/" device;
             mountPoint' = utils.escapeSystemdPath mountPoint;
             device' = utils.escapeSystemdPath device;
-        in { "pre-mount-${mountPoint'}" = rec {
+        in { "pre-mount-${mountPoint'}" = rec { # TODO: in initrd (or during installation), how to deal with the fact that the system is not mounted at "/"?
             description = "Prepare mounting ${device} at ${mountPoint}";
             wantedBy = [ "${mountPoint'}.mount" ]; before = wantedBy; partOf = wantedBy;
             requires = lib.optional isDevice "${device'}.device"; after = lib.optional isDevice "${device'}.device";
             unitConfig.RequiresMountsFor = map utils.escapeSystemdExecArg (depends ++ (lib.optional (lib.hasPrefix "/" device) device) ++ [ (builtins.dirOf mountPoint) ]);
             unitConfig.DefaultDependencies = false; restartIfChanged = false;
             serviceConfig.Type = "oneshot"; serviceConfig.RemainAfterExit = true;
-            script = lib.mkIf (args.preMountCommands != "") args.preMountCommands;
+            script = lib.mkIf (fs.preMountCommands != "") fs.preMountCommands;
-            preStop = lib.mkIf (args.postUnmountCommands != "") args.postUnmountCommands; # ("preStop" still runs post unmount)
+            preStop = lib.mkIf (fs.postUnmountCommands != "") fs.postUnmountCommands; # ("preStop" still runs post unmount)
         }; } else { }) config.fileSystems;
 
-    });
+    in {
+        inherit assertions;
+        systemd.services = services false;
+        boot.initrd.systemd.services = lib.mkIf (config.boot.initrd.systemd.enable) (services true);
+    };
 
 }

modules/installer.nix.md

@@ -48,11 +48,11 @@ in {
         };
         build.scripts = lib.mkOption {
             type = lib.types.functionTo lib.types.str;  internal = true; readOnly = true;
-            default = context: lib.fun.substituteImplicit { # This replaces the `@{}` references in the scripts with normal bash variables that hold serializations of the Nix values they refer to.
+            default = context: "${lib.fun.substituteImplicit { # This replaces the `@{}` references in the scripts with normal bash variables that hold serializations of the Nix values they refer to.
                 inherit pkgs; scripts = lib.sort (a: b: a.order < b.order) (lib.attrValues cfg.scripts);
                 context = { inherit config options pkgs; inherit (moduleArgs) inputs; } // context;
                 # inherit (builtins) trace;
-            };
+            }}";
         };
     }; };
 

modules/setup/disks.nix.md

@@ -48,7 +48,7 @@ in {
                 position = lib.mkOption { description = "Position at which to create the partition. The default »+0« means the beginning of the largest free block."; type = lib.types.str; default = "+0"; };
                 alignment = lib.mkOption { description = "Adjusted alignment quantifier for this partition only."; type = lib.types.nullOr lib.types.int; default = null; example = 1; };
                 index = lib.mkOption { description = "Optionally explicit partition table index to place this partition in. Use ».order« to make sure that this index hasn't been used yet.."; type = lib.types.nullOr lib.types.int; default = null; };
-                order = lib.mkOption { description = "Creation order ranking of this partition. Higher orders will be created first, and will thus be placed earlier in the partition table (if ».index« isn't explicitly set) and also further to the front of the disk space."; type = lib.types.int; default = if config.size == null then 500 else (1000 - (if config.index == null then 0 else 256 - config.index)); };
+                order = lib.mkOption { description = "Creation order ranking of this partition. Higher orders will be created first, and will thus be placed earlier in the partition table (if ».index« isn't explicitly set) and also further to the front of the disk space. No partitions may follow one that has ».size == null«. The computed default puts the partition(s) with ».size == null« last (»500«), partitions with an ».index« before that (»744 + .index«, sorted), and all other partitions at the front (»1000«, unordered)."; type = lib.types.int; default = if config.size == null then 500 else (1000 - (if config.index == null then 0 else 256 - config.index)); };
             }; })));
             default = { };
             apply = lib.filterAttrs (k: v: v != null);

modules/setup/keystore.nix.md

@@ -27,7 +27,7 @@ Any number of other devices may thus specify paths in the keystore as keylocatio
 
 ```nix
 #*/# end of MarkDown, beginning of NixOS module:
-dirname: inputs: { config, pkgs, lib, ... }: let lib = inputs.self.lib.__internal__; in let
+dirname: inputs: { config, pkgs, lib, utils, ... }: let lib = inputs.self.lib.__internal__; in let
     inherit (inputs.config.rename) setup installer;
     cfg = config.${setup}.keystore;
     hash = builtins.substring 0 8 (builtins.hashString "sha256" config.networking.hostName);
@@ -50,14 +50,19 @@ in let module = {
 
 
     config = let
-    in lib.mkIf cfg.enable (lib.mkMerge [ ({
+    in lib.mkIf (cfg.enable) (lib.mkMerge [ ({
 
+        ${setup}.keystore.keys."luks/keystore-${hash}/0" = lib.mkOptionDefault "hostname"; # (This is the only key that the setup scripts unconditionally require to be declared.)
         assertions = [ {
             assertion = cfg.keys?"luks/keystore-${hash}/0";
             message = ''At least one key (»0«) for »luks/keystore-${hash}« must be specified!'';
         } ];
 
-        boot.initrd.luks.devices = lib.mkIf cfg.enableLuksGeneration (lib.fun.mapMerge (key: let
+    }) ({ ## Declare LUKS devices for all LUKS keys:
+        ${setup}.keystore.enableLuksGeneration = lib.mkIf (config.virtualisation.useDefaultFilesystems or false) (lib.mkVMOverride false);
+    }) (lib.mkIf (cfg.enableLuksGeneration) {
+
+        boot.initrd.luks.devices = (lib.fun.mapMerge (key: let
             label = builtins.substring 5 ((builtins.stringLength key) - 7) key;
         in { ${label} = {
             device = lib.mkDefault "/dev/disk/by-partlabel/${label}";
@@ -65,27 +70,16 @@ in let module = {
             allowDiscards = lib.mkDefault true; # If attackers can observe trimmed blocks, then they can probably do much worse ...
         }; }) (lib.fun.filterMatching ''^luks/.*/0$'' (lib.attrNames cfg.keys)));
 
-        ${setup}.keystore.keys."luks/keystore-${hash}/0" = lib.mkOptionDefault "hostname"; # (This is the only key that the setup scripts unconditionally require to be declared.)
+        boot.initrd.systemd.services = lib.mkIf (config.boot.initrd.systemd.enable) (lib.fun.mapMerge (key: let
+            label = builtins.substring 5 ((builtins.stringLength key) - 7) key;
+        in if label == "keystore-${hash}" || (!config.boot.initrd.luks.devices?${label}) then { } else { "systemd-cryptsetup@${utils.escapeSystemdPath label}" = rec {
+            overrideStrategy = "asDropin"; # (could this be set via a x-systemd.after= crypttab option?)
+            after = [ "systemd-cryptsetup@keystore\\x2d${hash}.service" ]; wants = after; # (this may be implicit if systemd knew about the /run/keystore-... mount point)
+        }; }) (lib.fun.filterMatching ''^luks/.*/0$'' (lib.attrNames cfg.keys)));
 
-    }) ({
 
-        boot.initrd.luks.devices."keystore-${hash}" = {
+    }) ({ ## Create and populate keystore during installation:
-            device = "/dev/disk/by-partlabel/keystore-${hash}";
-            postOpenCommands = ''
-                echo "Mounting ${keystore}"
-                mkdir -p ${keystore}
-                mount -o nodev,umask=0277,ro /dev/mapper/keystore-${hash} ${keystore}
-            '';
-            preLVM = true; # ensure the keystore is opened early (»preLVM« also seems to be pre zpool import, and it is the only option that affects the opening order)
-            keyFile = lib.mkMerge [
-                (lib.mkIf cfg.unlockMethods.trivialHostname "${pkgs.writeText "hostname" config.networking.hostName}")
-                (lib.mkIf cfg.unlockMethods.usbPartition "/dev/disk/by-partlabel/bootkey-${hash}")
-            ];
-            fallbackToPassword = true; # (might as well)
-            preOpenCommands = lib.mkIf cfg.unlockMethods.pinThroughYubikey verbose.doOpenWithYubikey;
-        };
 
-        # Create and populate keystore during installation:
         fileSystems.${keystore} = { fsType = "vfat"; device = "/dev/mapper/keystore-${hash}"; options = [ "ro" "nosuid" "nodev" "noexec" "noatime" "umask=0277" "noauto" ]; formatArgs = [ ]; };
 
         ${setup}.disks.partitions."keystore-${hash}" = { type = lib.mkDefault "8309"; order = lib.mkDefault 1375; disk = lib.mkDefault "primary"; size = lib.mkDefault "32M"; };
@@ -94,10 +88,32 @@ in let module = {
             ${pkgs.rsync}/bin/rsync -a ${keystore}/ $tmp/
         )'';
 
-    }) (lib.mkIf (config.boot.initrd.luks.devices?"keystore-${hash}") { # (this will be false when overwritten by »nixos/modules/virtualisation/qemu-vm.nix«)
+
+        ## Unlocking and closing during early boot
+    }) (lib.mkIf (!(config.virtualisation.useDefaultFilesystems or false)) (let # (don't bother with any of this if »boot.initrd.luks.devices« is forced to »{ }« in »nixos/modules/virtualisation/qemu-vm.nix«)
+    in lib.mkMerge [ ({
+
+        boot.initrd.luks.devices."keystore-${hash}".keyFile = lib.mkMerge [
+            (lib.mkIf (cfg.unlockMethods.trivialHostname) "${pkgs.writeText "hostname" config.networking.hostName}")
+            (lib.mkIf (cfg.unlockMethods.usbPartition) "/dev/disk/by-partlabel/bootkey-${hash}")
+        ];
+        boot.initrd.systemd.storePaths = lib.mkIf (cfg.unlockMethods.trivialHostname && config.boot.initrd.systemd.enable) [ "${pkgs.writeText "hostname" config.networking.hostName}" ];
 
         boot.initrd.supportedFilesystems = [ "vfat" ];
 
+    }) (lib.mkIf (!config.boot.initrd.systemd.enable) { # Legacy initrd
+
+        boot.initrd.luks.devices."keystore-${hash}" = {
+            preLVM = true; # ensure the keystore is opened early (»preLVM« also seems to be pre zpool import, and it is the only option that affects the opening order)
+            preOpenCommands = lib.mkIf (cfg.unlockMethods.pinThroughYubikey) verbose.doOpenWithYubikey; # TODO: required?
+            fallbackToPassword = true; # (might as well)
+            postOpenCommands = ''
+                echo "Mounting ${keystore}"
+                mkdir -p ${keystore}
+                mount -o nodev,umask=0277,ro /dev/mapper/keystore-${hash} ${keystore}
+            '';
+        };
+
         boot.initrd.postMountCommands = ''
             ${if (lib.any (lib.fun.matches "^home/.*$") (lib.attrNames cfg.keys)) then ''
                 echo "Transferring home key composites"
@@ -111,23 +127,68 @@ in let module = {
             cryptsetup close /dev/mapper/keystore-${hash}
         '';
 
-        boot.initrd.luks.yubikeySupport = lib.mkIf cfg.unlockMethods.pinThroughYubikey true;
+        boot.initrd.luks.yubikeySupport = lib.mkIf (cfg.unlockMethods.pinThroughYubikey) true;
-        boot.initrd.extraUtilsCommands = lib.mkIf cfg.unlockMethods.pinThroughYubikey (lib.mkAfter ''
+        boot.initrd.extraUtilsCommands = lib.mkIf (cfg.unlockMethods.pinThroughYubikey) (lib.mkAfter ''
             copy_bin_and_libs ${verbose.askPassWithYubikey}/bin/cryptsetup-askpass
             sed -i "s|/bin/sh|$out/bin/sh|" "$out/bin/cryptsetup-askpass"
         '');
 
-    }) ]);
+    }) (lib.mkIf (config.boot.initrd.systemd.enable) (let # Systemd initrd
+
+        unlockWithYubikey = pkgs.writeShellScript "unlock-keystore" (let
+            dev = config.boot.initrd.luks.devices."keystore-${hash}";
+        in ''
+            ${verbose.tryYubikey}
+            ${lib.optionalString (dev.keyFile != null) ''
+                if systemd-cryptsetup attach 'keystore-${hash}' '/dev/disk/by-partlabel/keystore-${hash}' ${lib.escapeShellArg dev.keyFile} '${lib.optionalString dev.allowDiscards "discard,"}headless' ; then exit ; fi
+                printf '%s\n\n' 'Unlocking keystore-${hash} with '${lib.escapeShellArg dev.keyFile}' failed.' >/dev/console
+            ''}
+            for attempt in "" 2 3 ; do (
+                passphrase=$( systemd-ask-password 'Please enter passphrase for disk keystore-${hash}'"''${attempt:+ (attempt $attempt/3)}" ) || exit
+                passphrase="$( tryYubikey "$passphrase" 2>/dev/console )" || exit
+                systemd-cryptsetup attach 'keystore-${hash}' '/dev/disk/by-partlabel/keystore-${hash}' <( printf %s "$passphrase" ) '${lib.optionalString dev.allowDiscards "discard,"}headless' || exit
+            ) && break ; done || exit
+        '');
+    in {
+        boot.initrd.systemd.services = {
+            "systemd-cryptsetup@keystore\\x2d${hash}" = {
+                overrideStrategy = "asDropin";
+                serviceConfig.ExecStart = lib.mkIf (cfg.unlockMethods.pinThroughYubikey) [ "" "${unlockWithYubikey}" ];
+                postStart = ''
+                    echo "Mounting ${keystore}"
+                    mkdir -p ${keystore}
+                    mount -o nodev,umask=0277,ro /dev/mapper/keystore-${hash} ${keystore}
+                '';
+            };
+            # lib.mkIf (lib.any (lib.fun.matches "^home/.*$") (lib.attrNames cfg.keys))
+            initrd-nixos-activation.postStart = ''
+                mkdir -pm 551 /sysroot/run/keys/home-composite/
+                if [[ -e ${keystore}/home/ ]] ; then
+                    cp -a ${keystore}/home/*.key /sysroot/run/keys/home-composite/
+                fi
+            '';
+            initrd-cleanup.preStart = ''
+                umount ${keystore} || true
+                rmdir ${keystore} || true
+                ${config.systemd.package}/lib/systemd/systemd-cryptsetup detach keystore-${hash}
+            '';
+        };
+        boot.initrd.luks.devices."keystore-${hash}".keyFileTimeout = 10;
+        boot.initrd.systemd.storePaths = lib.mkIf (cfg.unlockMethods.pinThroughYubikey) [ unlockWithYubikey ];
+        boot.initrd.systemd.initrdBin = lib.mkIf (cfg.unlockMethods.pinThroughYubikey) [ pkgs.yubikey-personalization ];
+
+    })) ])) ]);
+
 
 }; verbose = rec {
 
     tryYubikey = ''tryYubikey () { # 1: key
         local key="$1" ; local slot
-        if   [ "$(ykinfo -q -2 2>/dev/null)" = '1' ] ; then slot=2 ;
+        if   [ "$( ykinfo -q -2 2>/dev/null )" = '1' ] ; then slot=2 ;
-        elif [ "$(ykinfo -q -1 2>/dev/null)" = '1' ] ; then slot=1 ; fi
+        elif [ "$( ykinfo -q -1 2>/dev/null )" = '1' ] ; then slot=1 ; fi
         if [ "$slot" ] ; then
-            echo >&2 ; echo "Using slot $slot of detected Yubikey ..." >&2
+            echo "Using slot $slot of detected Yubikey ..." >&2
-            key="$(ykchalresp -$slot "$key" 2>/dev/null || true)"
+            key="$( ykchalresp -$slot "$key" 2>/dev/null )" || true
             if [ "$key" ] ; then echo "Got response from Yubikey" >&2 ; fi
         fi
         printf '%s' "$key"

modules/setup/temproot.nix.md

@@ -57,14 +57,16 @@ This completely configures the disks, partitions, pool, datasets, and mounts for
   setup.temproot.temp.type = "zfs";
   setup.temproot.local.type = "zfs";
   setup.temproot.remote.type = "zfs";
+  setup.temproot.swap = { size = "2G"; asPartition = true; encrypted = true; };
 
   # Change/set the pools storage layout (see above), then adjust the partitions disks/sizes. Declaring disks requires them to be passed to the system installer.
   setup.zfs.pools."rpool-${hash}".vdevArgs = [ "raidz1" "rpool-rz1-${hash}" "rpool-rz2-${hash}" "rpool-rz3-${hash}" "log" "rpool-zil-${hash}" "cache" "rpool-arc-${hash}" ];
   setup.disks.partitions."rpool-rz1-${hash}" = { disk = "raidz1"; };
   setup.disks.partitions."rpool-rz2-${hash}" = { disk = "raidz2"; };
   setup.disks.partitions."rpool-rz3-${hash}" = { disk = "raidz3"; };
+  setup.disks.partitions."swap-${hash}" = { };  # ... (created & configured implicitly)
   setup.disks.partitions."rpool-zil-${hash}" = { size = "2G"; };
-  setup.disks.partitions."rpool-arc-${hash}" = { }; } # (this is actually already implicitly declared)
+  setup.disks.partitions."rpool-arc-${hash}" = { }; } # (this is also already implicitly declared)
 ```
 
 On a less beefy system, but also with less data to manage, `tmpfs` works fine for `tmp`, and `f2fs` promises to get more performance out of the flash/ram/cpu:
@@ -265,15 +267,31 @@ in {
         }) ]))) ({ "/" = { options = { }; uid = 0; gid = 0; mode = "755"; }; } // cfg.${type}.mounts);
 
 
-    })) (lib.mkIf (cfg.temp.type == "zfs") {
+    })) (lib.mkIf (cfg.temp.type == "zfs") (let
+        description = "ZFS rollback to ${cfg.temp.zfs.dataset}/**@empty";
+        command = ''zfs list -H -o name -t snapshot -r ${lib.escapeShellArg cfg.temp.zfs.dataset} | grep '@empty$' | xargs -n1 -- zfs rollback -r'';
+        pool = builtins.head (builtins.split "/" cfg.temp.zfs.dataset);
+    in {
 
-        boot.initrd.postDeviceCommands = lib.mkAfter ''
+        boot.initrd.postDeviceCommands = lib.mkIf (!config.boot.initrd.systemd.enable) (lib.mkAfter ''
-            echo 'Clearing root ZFS'
+            echo '${description}'
-            ( zfs list -H -o name -t snapshot -r ${cfg.temp.zfs.dataset} | grep '@empty$' | xargs -n1 --no-run-if-empty zfs rollback -r )
+            ( ${command} )
-        '';
+        '');
+
+        boot.initrd.systemd.services.zfs-rollback-temp = lib.mkIf (config.boot.initrd.systemd.enable) (rec {
+            after = [ "zfs-import-${pool}.service" ]; before = [ "zfs-import.target" ]; wantedBy = before;
+            serviceConfig.Type = "oneshot";
+            #inherit description; script = "PATH=${lib.makeBinPath [ pkgs.findutils pkgs.gnugrep ]}:$PATH ; ${command}";
+            inherit description; script = command;
+        });
+        #boot.initrd.systemd.initrdBin = lib.mkIf (config.boot.initrd.systemd.enable) [ pkgs.findutils pkgs.gnugrep ];
+        boot.initrd.systemd.extraBin = lib.mkIf (config.boot.initrd.systemd.enable) {
+            grep = lib.getExe pkgs.gnugrep;
+            xargs = "${pkgs.findutils}/bin/xargs";
+        };
 
 
-    }) (lib.mkIf (cfg.temp.type == "bind") { # (TODO: this should completely clear or even recreate the »cfg.temp.bind.source«)
+    })) (lib.mkIf (cfg.temp.type == "bind") { # (TODO: this should completely clear or even recreate the »cfg.temp.bind.source«)
 
         boot.cleanTmpDir = true; # Clear »/tmp« on reboot.
 
@@ -357,7 +375,7 @@ in {
 
         ${setup} = {
             zfs.enable = true;
-            zfs.pools.${lib.head (lib.splitString "/" dataset)} = { }; # ensure the pool exists (all properties can be adjusted)
+            zfs.pools.${lib.head (builtins.split "/" dataset)} = { }; # ensure the pool exists (all properties can be adjusted)
             keystore.keys."zfs/${dataset}" = lib.mkIf (type == "remote" && config.${setup}.keystore.enable) (lib.mkOptionDefault "random"); # the entire point of ZFS remote are backups, and those should be encrypted
 
             zfs.datasets = {

modules/setup/zfs.nix.md

@@ -71,7 +71,7 @@ in let module = {
 
         ## Implement »cfg.datasets.*.mount«:
         fileSystems = lib.fun.mapMerge (path: { props, mount, ... }: if mount != false then {
-            "${props.mountpoint}" = { fsType = "zfs"; device = path; options = [ "zfsutil" ] ++ (lib.optionals (mount == "noauto") [ "noauto" ]); };
+            "${props.mountpoint}" = { fsType = "zfs"; device = path; options = [ "zfsutil" "x-systemd.after=zfs-import.target" ] ++ (lib.optionals (mount == "noauto") [ "noauto" ]); };
         } else { }) cfg.datasets;
 
         ## Load keys (only) for (all) datasets that are declared as encryption roots and aren't disabled:
@@ -86,7 +86,7 @@ in let module = {
                 compression = lib.mkOptionDefault "lz4"; # Seems to be the best compromise between compression and CPU load.
                 atime = lib.mkOptionDefault "off"; relatime = lib.mkOptionDefault "on"; # Very much don't need access times at all.
 
-                acltype = lib.mkOptionDefault "posix"; # Enable ACLs (access control lists) on linux; might be useful at some point. (»posix« is the same as »posixacl«, but this is the normalized form)
+                acltype = lib.mkOptionDefault "posix"; # Enable ACLs (access control lists) on Linux; might be useful at some point. (»posix« is the same as »posixacl«, but this is the normalized form)
                 xattr = lib.mkOptionDefault "sa"; # Allow extended attributes and store them as system attributes, recommended with »acltype=posix«.
                 dnodesize = lib.mkOptionDefault "auto"; # Recommenced with »xattr=sa«. (A dnode is roughly equal to inodes, storing file directory or meta data.)
                 #normalization = lib.mkOptionDefault "formD"; # Don't enforce utf-8, and thus don't normalize file names; instead accept any byte stream as file name.
@@ -94,9 +94,6 @@ in let module = {
                 canmount = lib.mkOptionDefault "off"; mountpoint = lib.mkOptionDefault "none"; # Assume the pool root is a "container", unless overwritten.
             }; }) cfg.pools;
 
-            # All pools that have at least one dataset that (explicitly or implicitly) has a key to be loaded from »/run/keystore-.../zfs/« have to be imported in the initramfs while the keystore is open (but only if the keystore is not disabled):
-            zfs.extraInitrdPools = lib.mkIf (config.boot.initrd.luks.devices?"keystore-${hash}") (lib.mapAttrsToList (name: _: lib.head (lib.splitString "/" name)) (lib.filterAttrs (name: { props, ... }: if props?keylocation then lib.fun.startsWith "file:///run/keystore-${hash}/" props.keylocation else config.${setup}.keystore.keys?"zfs/${name}") cfg.datasets));
-
             # Might as well set some defaults for all partitions required (though for all but one at least some of the values will need to be changed):
             disks.partitions = lib.fun.mapMergeUnique (name: { ${name} = { # (This also implicitly ensures that no partition is used twice for zpools.)
                 type = lib.mkDefault "bf00"; size = lib.mkOptionDefault null; order = lib.mkDefault 500;
@@ -104,13 +101,28 @@ in let module = {
         };
 
 
-    }) ({ ## Implement »cfg.extraInitrdPools«:
+    }) (let ## Implement »cfg.extraInitrdPools«:
 
-        boot.initrd.postDeviceCommands = (lib.mkAfter ''
+        # All pools that have at least one dataset that (explicitly or implicitly) has a key to be loaded from »/run/keystore-.../zfs/« have to be imported in the initramfs while the keystore is open (but only if the keystore is not disabled):
+        keystorePools = lib.optionals (config.boot.initrd.luks.devices?"keystore-${hash}") (lib.mapAttrsToList (name: _: lib.head (lib.splitString "/" name)) (lib.filterAttrs (name: { props, ... }: if props?keylocation then lib.fun.startsWith "file:///run/keystore-${hash}/" props.keylocation else config.${setup}.keystore.keys?"zfs/${name}") cfg.datasets));
+    in {
+        ${setup}.zfs.extraInitrdPools = keystorePools;
+
+        boot.initrd.postDeviceCommands = lib.mkIf (!config.boot.initrd.systemd.enable) (lib.mkAfter ''
             ${lib.concatStringsSep "\n" (map verbose.initrd-import-zpool cfg.extraInitrdPools)}
             ${verbose.initrd-load-keys}
         '');
 
+        boot.initrd.systemd.services = lib.mkIf (config.boot.initrd.systemd.enable) (lib.fun.mapMerge (pool: { "zfs-import-${pool}" = let
+            service = config.systemd.services."zfs-import-${pool}" or null;
+            addPrefix = deps: map (dep: if dep == "zfs-import.target" || dep == "sysusr-usr.mount" then dep else "sysroot-${dep}") deps;
+        in lib.mkMerge [ (lib.mkIf (service != null) (
+            (builtins.removeAttrs service [ "restartTriggers" "reloadTriggers" ])
+            // { requiredBy = addPrefix service.requiredBy; before = addPrefix service.before; }
+        )) (lib.mkIf (builtins.elem pool keystorePools) (
+            rec { after = [ "systemd-cryptsetup@keystore\\x2d${hash}.service" ]; wants = after; } # without this, the keystore's password prompt fails
+        )) ]; }) cfg.extraInitrdPools);
+
 
     }) (lib.mkIf (config.boot.resumeDevice == "") { ## Disallow hibernation without fixed »resumeDevice«:
 
@@ -156,35 +168,59 @@ in let module = {
         anyPool = filterBy: lib.any (pool: pool.${filterBy}) (lib.attrValues cfg.pools);
         poolNames = filterBy: lib.attrNames (lib.filterAttrs (name: pool: pool.${filterBy}) cfg.pools);
         filter = pool: "^${pool}($|[/])";
-        ensure-datasets = zfsPackage: extraUtils: pkgs.writeScript "ensure-datasets" ''
+        ensure-datasets = zfsPackage: extraUtils: (let
-            #!${pkgs.pkgsStatic.bash}/bin/bash
+            inherit (lib.fun.substituteImplicit { inherit pkgs; scripts = lib.attrValues { inherit (lib.self.setup-scripts) zfs utils; }; context = { inherit config; native = pkgs // { zfs = zfsPackage; } // (lib.optionalAttrs (extraUtils != null) (lib.genAttrs [
-            set -o pipefail -o nounset ; declare-command () { : ; } ; declare-flag () { : ; } ;
-            ${lib.fun.substituteImplicit { inherit pkgs; scripts = lib.attrValues { inherit (lib.self.setup-scripts) zfs utils; }; context = { inherit config; native = pkgs // { zfs = zfsPackage; } // (lib.optionalAttrs (extraUtils != null) (lib.genAttrs [
                 "kmod" # modprobe
                 "util-linux" # mount umount
                 "nix" "openssh" "jq" # (unused)
-            ] (_: extraUtils))); }; }}
+            ] (_: extraUtils))); }; }) script scripts vars;
+        in { script =  pkgs.writeScript "ensure-datasets" ''
+            #!${pkgs.pkgsStatic.bash}/bin/bash
+            set -o pipefail -o nounset ; declare-command () { : ; } ; declare-flag () { : ; } ;
+            ${script}
             ensure-datasets "$@"
-        '';
+        ''; inherit scripts vars; });
         ensure-datasets-for = filterBy: zfsPackage: extraUtils: ''( if [ ! "''${IN_NIXOS_ENTER:-}" ] && [ -e ${zfsPackage}/bin/zfs ] ; then
             ${lib.concatStrings (map (pool: ''
                 expected=${lib.escapeShellArg (builtins.toJSON (lib.mapAttrs (n: v: v.props // (if v.permissions != { } then { ":permissions" = v.permissions; } else { })) (lib.filterAttrs (path: _: path == pool || lib.fun.startsWith "${pool}/" path) cfg.datasets)))}
                 if [ "$(${zfsPackage}/bin/zfs get -H -o value nixos-${setup}:applied-datasets ${pool})" != "$expected" ] ; then
-                    ${ensure-datasets zfsPackage extraUtils} / ${lib.escapeShellArg (filter pool)} && ${zfsPackage}/bin/zfs set nixos-${setup}:applied-datasets="$expected" ${pool}
+                    ${(ensure-datasets zfsPackage extraUtils).script} / ${lib.escapeShellArg (filter pool)} && ${zfsPackage}/bin/zfs set nixos-${setup}:applied-datasets="$expected" ${pool}
                 fi
             '') (poolNames filterBy))}
         fi )'';
+        ensure-datasets-service = pool: initrd: if
+            (if initrd then !pool.autoApplyDuringBoot else !pool.autoApplyOnActivation)
+        then { } else { "zfs-ensure-${pool.name}" = let
+            expected = builtins.toJSON (lib.mapAttrs (n: v: v.props // (if v.permissions != { } then { ":permissions" = v.permissions; } else { })) (lib.filterAttrs (path: _: path == pool.name || lib.fun.startsWith "${pool.name}/" path) cfg.datasets));
+            #zfsPackage = if initrd then pkgs.runCommandLocal "root-link" { } ''ln -sT / $out'' else pkgs.runCommandLocal "booted-system-link" { } ''ln -sT /run/booted-system/sw $out'';
+            zfsPackage = if initrd then "/" else "/run/booted-system/sw";
+            #extraUtils = if initrd then pkgs.runCommandLocal "root-link" { } ''ln -sT / $out'' else null;
+            extraUtils = if initrd then "/" else null;
+        in rec {
+            after = [ "zfs-import-${pool.name}.service" ];
+            before = [ "zfs-import.target" "shutdown.target" ]; wantedBy = [ "zfs-import.target" ]; conflicts = [ "shutdown.target" ];
+            unitConfig.DefaultDependencies = false; # (not after basic.target or sysinit.target)
+            serviceConfig.Type = "oneshot";
+            script = ''
+                expected=${lib.escapeShellArg expected}
+                if [ "$(${zfsPackage}/bin/zfs get -H -o value nixos-${setup}:applied-datasets ${pool.name})" != "$expected" ] ; then
+                    ${(ensure-datasets zfsPackage extraUtils).script} ${if initrd then "/sysroot" else "/"} ${lib.escapeShellArg (filter pool.name)} && ${zfsPackage}/bin/zfs set nixos-${setup}:applied-datasets="$expected" ${pool.name}
+                fi
+            '';
+        } // (lib.optionalAttrs (!initrd) {
+            restartTriggers = [ expected ];
+        }); };
     in {
 
-        boot.initrd.postDeviceCommands = lib.mkIf (anyPool "autoApplyDuringBoot") (lib.mkOrder 2000 ''
+        boot.initrd.postDeviceCommands = lib.mkIf (!config.boot.initrd.systemd.enable) (lib.mkIf (anyPool "autoApplyDuringBoot") (lib.mkOrder 2000 ''
             ${ensure-datasets-for "autoApplyDuringBoot" extraUtils extraUtils}
-        '');
+        ''));
+        boot.initrd.systemd.services = lib.mkIf (config.boot.initrd.systemd.enable) (lib.fun.mapMerge (pool: ensure-datasets-service pool true) (lib.attrValues cfg.pools));
+        boot.initrd.systemd.storePaths = lib.mkIf (anyPool "autoApplyDuringBoot") (let deps = ensure-datasets "/" "/"; in [ "${pkgs.pkgsStatic.bash}/bin/bash" deps.script ] ++ deps.scripts ++ (lib.filter (v: lib.isStringLike v && lib.hasPrefix builtins.storeDir v) (lib.attrValues deps.vars))); # TODO: using pkgsStatic.bash may not be necessary here
         boot.initrd.supportedFilesystems = lib.mkIf (anyPool "autoApplyDuringBoot") [ "zfs" ];
         ${setup}.zfs.extraInitrdPools = (poolNames "autoApplyDuringBoot");
 
-        system.activationScripts.A_ensure-datasets = lib.mkIf (anyPool "autoApplyOnActivation") {
+        systemd.services = lib.fun.mapMerge (pool: ensure-datasets-service pool false) (lib.attrValues cfg.pools);
-            text = ensure-datasets-for "autoApplyOnActivation" (pkgs.runCommandLocal "booted-system-link" { } ''ln -sT /run/booted-system/sw $out'') null; # (want to use the version of ZFS that the kernel module uses, also it's convenient that this does not yet exist during activation at boot)
-        }; # these are sorted alphabetically, unless one gets "lifted up" by some other ending on it via its ».deps« field
 
 
     }) ])) (

modules/vm-exec.nix.md

@@ -88,6 +88,7 @@ in let hostModule = {
         virtualisation.graphics = false;
 
         # Instead of tearing down the initrd environment, adjust some mounts and run the »command« in the initrd:
+        boot.initrd.systemd.enable = lib.mkVMOverride false;
         boot.initrd.postMountCommands = ''
 
             for fs in tmp/shared tmp/xchg nix/store.lower nix/var/nix/db.lower ; do