nixos-installer/modules/fs/zfs.nix.md

/*

ZFS Pools and Datasets

This module primarily allows the specification of ZFS pools and datasets. The declared pools and datasets are complemented with some default and are then used by lib/setup-scripts/zfs.sh to create them during system installation, and can optionally later be kept up to date (with the config) at config activation time or during reboot. Additionally, this module sets some defaults for ZFS (but only in a "always better than nothing" style, so lib.mkForce null should never be necessary).

Implementation

#*/# end of MarkDown, beginning of NixOS module:
dirname: inputs: { config, pkgs, lib, ... }: let inherit (inputs.self) lib; in let
    cfg = config.${prefix}.fs.zfs;
    prefix = inputs.config.prefix;
    hash = builtins.substring 0 8 (builtins.hashString "sha256" config.networking.hostName);
in let module = {

    options.${prefix} = { fs.zfs = {
        enable = lib.mkEnableOption "NixOS managed ZFS pools and datasets";

        pools = lib.mkOption {
            description = "ZFS pools created during this host's installation.";
            type = lib.types.attrsOf (lib.types.nullOr (lib.types.submodule ({ name, ... }: { options = {
                name = lib.mkOption { description = "Attribute name as name of the pool."; type = lib.types.str; default = name; readOnly = true; };
                vdevArgs = lib.mkOption { description = "List of arguments that specify the virtual devices (vdevs) used when initially creating the pool. Can consist of the device type keywords and partition labels. The latter are prefixed with »/dev/mapper/« if a mapping with that name is configured or »/dev/disk/by-partlabel/« otherwise, and then the resulting argument sequence is is used verbatim in »zpool create«."; type = lib.types.listOf lib.types.str; default = [ name ]; example = [ "raidz1" "data1-..." "data2-..." "data3-..." "cache" "cache-..." ]; };
                props = lib.mkOption { description = "Zpool properties to pass when creating the pool. May also set »feature@...« and »compatibility«."; type = lib.types.attrsOf (lib.types.nullOr lib.types.str); default = { }; };
                createDuringInstallation = (lib.mkEnableOption "creation of this pool during system installation. If disabled, the pool needs to exist already or be created manually and the pools disk devices are expected to be present from the first boot onwards") // { default = true; };
                autoApplyDuringBoot = (lib.mkEnableOption "automatically re-applying changed dataset properties and create missing datasets in the initramfs phase during boot for this pool. This can be useful since the keystore is open but no datasets are mounted at that time") // { default = true; };
                autoApplyOnActivation = (lib.mkEnableOption "automatically re-applying changed dataset properties and create missing datasets on system activation for this pool. This may fail for some changes since datasets may be mounted and the keystore is usually closed at this time. Enable ».autoApplyDuringBoot« and reboot to address this") // { default = true; };
            }; config = {
                props.ashift = lib.mkOptionDefault "12"; # be explicit
                props.comment = lib.mkOptionDefault "hostname=${config.networking.hostName};"; # This is just nice to know without needing to inspect the datasets (»zpool import« shows the comment).
                props.cachefile = lib.mkOptionDefault "none"; # If it works on first boot without (stateful) cachefile, then it will also do so later.
            }; })));
            default = { };
            apply = lib.filterAttrs (k: v: v != null);
        };

        datasets = lib.mkOption {
            description = "ZFS datasets managed and mounted on this host.";
            type = lib.types.attrsOf (lib.types.nullOr (lib.types.submodule ({ name, ... }: { options = {
                name = lib.mkOption { description = "Attribute name as name of the pool."; type = lib.types.str; default = name; readOnly = true; };
                props = lib.mkOption { description = "ZFS properties to set on the dataset."; type = lib.types.attrsOf (lib.types.nullOr lib.types.str); default = { }; };
                mount = lib.mkOption { description = "Whether to create a »fileSystems« entry to mount the dataset. »noauto« creates an entry with that option set."; type = lib.types.enum [ true "noauto" false ]; default = false; };
                permissions = lib.mkOption { description = ''Permissions to set on the dataset via »zfs allow«. Attribute names should express propagation/who and match »/^[dl]?([ug]\d+|e)$/«, the values are the list of permissions granted.''; type = lib.types.attrsOf lib.types.commas; default = { }; };
                uid = lib.mkOption { description = "UID owning the dataset's root directory."; type = lib.types.ints.unsigned; default = 0; };
                gid = lib.mkOption { description = "GID owning the dataset's root directory."; type = lib.types.ints.unsigned; default = 0; };
                mode = lib.mkOption { description = "Permission mode of the dataset's root directory."; type = lib.types.str; default = "750"; };
            }; config = {
                props.canmount = lib.mkOptionDefault "off"; # (need to know this explicitly for each dataset)
            }; })));
            default = { };
            apply = lib.filterAttrs (k: v: v != null);
        };

        extraInitrdPools = lib.mkOption { description = "Additional pool that are imported in the initrd."; type = lib.types.listOf lib.types.str; default = [ ]; apply = lib.unique; };
    }; };

    config = let
    in lib.mkIf cfg.enable (lib.mkMerge [ ({

        # boot.(initrd.)supportedFilesystems = [ "zfs" ]; # NixOS should figure that out itself based on zfs being used in »config.fileSystems«.
        # boot.zfs.extraPools = [ ]; # Don't need to import pools that have at least one dataset listed in »config.fileSystems« / with ».mount != false«.
        boot.zfs.devNodes = lib.mkDefault ''/dev/disk/by-partlabel" -d "/dev/mapper''; # Do automatic imports (initrd & systemd) by-partlabel or mapped device, instead of by-id, since that is how the pools were created. (This option is meant to support only a single path, but since it is not properly escaped, this works to pass two paths.)
        services.zfs.autoScrub.enable = true;
        services.zfs.trim.enable = true; # (default)

        ## Implement »cfg.datasets.*.mount«:
        fileSystems = lib.wip.mapMerge (path: { props, mount, ... }: if mount != false then {
            "${props.mountpoint}" = { fsType = "zfs"; device = path; options = [ "zfsutil" ] ++ (lib.optionals (mount == "noauto") [ "noauto" ]); };
        } else { }) cfg.datasets;

        ## Load keys (only) for (all) datasets that are declared as encryption roots and aren't disabled:
        boot.zfs.requestEncryptionCredentials = lib.attrNames (lib.filterAttrs (name: { props, ... }: if props?keylocation then props.keylocation != "file:///dev/null" else config.${prefix}.fs.keystore.keys?"zfs/${name}") cfg.datasets);

        ${prefix} = {
            # Set default root dataset properties for every pool:
            fs.zfs.datasets = lib.mapAttrs (name: { ... }: { props = {
                # Properties to set at the root dataset of the root pool at its creation. All are inherited by default, but some can't be changed later.
                devices = lib.mkOptionDefault "off"; # Don't allow character or block devices on the file systems, where they might be owned by non-root users.
                setuid = lib.mkOptionDefault "off"; # Don't allow suid binaries (NixOS puts them in »/run/wrappers/«).
                compression = lib.mkOptionDefault "lz4"; # Seems to be the best compromise between compression and CPU load.
                atime = lib.mkOptionDefault "off"; relatime = lib.mkOptionDefault "on"; # Very much don't need access times at all.

                acltype = lib.mkOptionDefault "posix"; # Enable ACLs (access control lists) on linux; might be useful at some point. (»posix« is the same as »posixacl«, but this is the normalized form)
                xattr = lib.mkOptionDefault "sa"; # Allow extended attributes and store them as system attributes, recommended with »acltype=posix«.
                dnodesize = lib.mkOptionDefault "auto"; # Recommenced with »xattr=sa«. (A dnode is roughly equal to inodes, storing file directory or meta data.)
                #normalization = lib.mkOptionDefault "formD"; # Don't enforce utf-8, and thus don't normalize file names; instead accept any byte stream as file name.

                canmount = lib.mkOptionDefault "off"; mountpoint = lib.mkOptionDefault "none"; # Assume the pool root is a "container", unless overwritten.
            }; }) cfg.pools;

            # All pools that have at least one dataset that (explicitly or implicitly) has a key to be loaded from »/run/keystore-.../zfs/« have to be imported in the initramfs while the keystore is open:
            fs.zfs.extraInitrdPools = (lib.mapAttrsToList (name: _: lib.head (lib.splitString "/" name)) (lib.filterAttrs (name: { props, ... }: if props?keylocation then lib.wip.startsWith "file:///run/keystore-" props.keylocation else config.${prefix}.fs.keystore.keys?"zfs/${name}") cfg.datasets));

            # Might as well set some defaults for all partitions required (though for all but one at least some of the values will need to be changed):
            fs.disks.partitions = lib.wip.mapMergeUnique (name: { ${name} = { # (This also implicitly ensures that no partition is used twice for zpools.)
                type = lib.mkDefault "bf00"; size = lib.mkOptionDefault null; order = lib.mkDefault 500;
            }; }) (lib.wip.filterMismatching ''/|^(mirror|raidz[123]?|draid[123]?.*|spare|log|dedup|special|cache)$'' (lib.concatLists (lib.catAttrs "vdevArgs" (lib.attrValues cfg.pools))));
        };


    }) ({ ## Implement »cfg.extraInitrdPools«:

        boot.initrd.postDeviceCommands = (lib.mkAfter ''
            ${lib.concatStringsSep "\n" (map verbose.initrd-import-zpool cfg.extraInitrdPools)}
            ${verbose.initrd-load-keys}
        '');


    }) (lib.mkIf (config.boot.resumeDevice == "") { ## Disallow hibernation without fixed »resumeDevice«:

        boot.kernelParams = [ "nohibernate" "hibernate=no" ];
        assertions = [ { # Ensure that none is overriding the above:
            assertion = builtins.elem "nohibernate" config.boot.kernelParams;
            message = ''Hibernation with ZFS (and NixOS' initrd) without fixed »resumeDevice« can/will lead to pool corruption. Disallow it by setting »boot.kernelParams = [ "nohibernate" ]«'';
        } ];


    }) (lib.mkIf (config.boot.resumeDevice != "") { ## Make resuming after hibernation safe with ZFS:

        boot.kernelParams = [ "resume=${config.boot.resumeDevice}" ];
        assertions = [ { # Just making sure ...
            assertion = builtins.elem "resume=${config.boot.resumeDevice}" config.boot.kernelParams;
            message = "When using ZFS and not disabling hibernation, make sure to set the »resume=« kernel parameter!";
        } ];

        boot.initrd.postDeviceCommands = let
            inherit (config.system.build) extraUtils;
        in (lib.mkBefore ''
            # After hibernation, the pools MUST NOT be imported before resuming, as doing so can corrupt them.
            # NixOS' mess of an initrd script does resuming after mounting FSs (which seems very unnecessarily late, resuming from a "file" doesn't need the FS to be mounted, does it?).
            # This should generally run after all (also mapped) devices are created, but before any ZFS action, so do the hibernation resume here.
            # But also only support a fixed »resumeDevice«. No guessing:

            resumeInfo="$(udevadm info -q property "${config.boot.resumeDevice}" )"
            if [ "$(echo "$resumeInfo" | sed -n 's/^ID_FS_TYPE=//p')" = "swsuspend" ]; then
                resumeMajor="$(echo "$resumeInfo" | sed -n 's/^MAJOR=//p')"
                resumeMinor="$(echo "$resumeInfo" | sed -n 's/^MINOR=//p')"
                echo -n "Attempting to resume from hibernation (device $resumeMajor:$resumeMinor as ${config.boot.resumeDevice}) ..."
                echo "$resumeMajor:$resumeMinor" > /sys/power/resume 2> /dev/null || echo "failed to wake from hibernation, continuing normal boot!"
            fi
        '');
            #setsid ${extraUtils}/bin/ash -c "exec ${extraUtils}/bin/ash < /dev/$console >/dev/$console 2>/dev/$console"


    }) (let ## Implement »cfg.pools.*.autoApplyDuringBoot« and »cfg.pools.*.autoApplyOnActivation«:

        inherit (config.system.build) extraUtils;
        anyPool = filterBy: lib.any (pool: pool.${filterBy}) (lib.attrValues cfg.pools);
        poolNames = filterBy: lib.attrNames (lib.filterAttrs (name: pool: pool.${filterBy}) cfg.pools);
        filter = pool: "^${pool}($|[/])";
        ensure-datasets = zfsPackage: pkgs.writeShellScript "ensure-datasets" ''
            ${lib.wip.substituteImplicit { inherit pkgs; scripts = lib.attrValues { inherit (lib.wip.setup-scripts) zfs utils; }; context = { inherit config; native = pkgs // { zfs = zfsPackage; }; }; }}
            set -eu ; ensure-datasets "$@"
        '';
        ensure-datasets-for = filterBy: zfsPackage: ''( if [ ! "''${IN_NIXOS_ENTER:-}" ] && [ -e ${zfsPackage}/bin/zfs ] ; then
            ${lib.concatStrings (map (pool: ''
                expected=${lib.escapeShellArg (builtins.toJSON (lib.mapAttrs (n: v: v.props) (lib.filterAttrs (path: _: path == pool || lib.wip.startsWith "${pool}/" path) cfg.datasets)))}
                if [ "$(${zfsPackage}/bin/zfs get -H -o value nixos-${prefix}:applied-datasets ${pool})" != "$expected" ] ; then
                    ${ensure-datasets zfsPackage} / ${lib.escapeShellArg (filter pool)} && ${zfsPackage}/bin/zfs set nixos-${prefix}:applied-datasets="$expected" ${pool}
                fi
            '') (poolNames filterBy))}
        fi )'';
    in {

        boot.initrd.postDeviceCommands = lib.mkIf (anyPool "autoApplyDuringBoot") (lib.mkOrder 2000 ''
            ${ensure-datasets-for "autoApplyDuringBoot" extraUtils}
        '');
        boot.initrd.supportedFilesystems = lib.mkIf (anyPool "autoApplyDuringBoot") [ "zfs" ];
        ${prefix}.fs.zfs.extraInitrdPools = (poolNames "autoApplyDuringBoot");

        system.activationScripts.A_ensure-datasets = lib.mkIf (anyPool "autoApplyOnActivation") {
            text = ensure-datasets-for "autoApplyOnActivation" (pkgs.runCommandLocal "booted-system-link" { } ''ln -sT /run/booted-system/sw $out''); # (want to use the version of ZFS that the kernel module uses, also it's convenient that this does not yet exist during activation at boot)
        }; # these are sorted alphabetically, unless one gets "lifted up" by some other ending on it via its ».deps« field


    }) ]);

}; verbose = {

    # copied verbatim from https://github.com/NixOS/nixpkgs/blob/f989e13983fd1619f723b42ba271fe0b781dd24b/nixos/modules/tasks/filesystems/zfs.nix
    # It would be nice if this was done in a somewhat more composable way (why isn't this a function?) ...
    initrd-import-zpool = pool: ''
            echo -n "importing root ZFS pool \"${pool}\"..."
            # Loop across the import until it succeeds, because the devices needed may not be discovered yet.
            if ! poolImported "${pool}"; then
              for trial in `seq 1 60`; do
                poolReady "${pool}" > /dev/null && msg="$(poolImport "${pool}" 2>&1)" && break
                sleep 1
                echo -n .
              done
              echo
              if [[ -n "$msg" ]]; then
                echo "$msg";
              fi
              poolImported "${pool}" || poolImport "${pool}"  # Try one last time, e.g. to import a degraded pool.
            fi
    '';
    initrd-load-keys = let
        inherit (lib) isBool optionalString concatMapStrings; cfgZfs = config.boot.zfs;
    in ''
            ${if isBool cfgZfs.requestEncryptionCredentials
              then optionalString cfgZfs.requestEncryptionCredentials ''
                zfs load-key -a
              ''
              else concatMapStrings (fs: ''
                zfs load-key ${fs}
              '') cfgZfs.requestEncryptionCredentials}
    '';
}; in module