nixos/multipath: add multipath module

The multipath-tools package had existed in Nixpkgs for some time but
without a nixos module to configure/drive it. This module provides
attributes to drive the majority of multipath configuration options
and is being successfully used in stage-1 and stage-2 boot to mount
/nix from a multipath-serviced iSCSI volume.

Credit goes to @grahamc for early contributions to the module and
authoring the NixOS module test.

nixos/doc/manual/from_md/release-notes/rl-2111.section.xml

@@ -329,6 +329,13 @@
           controller support.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/opensvc/multipath-tools">multipath</link>,
+          the device mapper multipath (DM-MP) daemon. Available as
+          <link linkend="opt-services.multipath.enable">services.multipath</link>.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-21.11-incompatibilities">

nixos/doc/manual/release-notes/rl-2111.section.md

@@ -101,6 +101,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [joycond](https://github.com/DanielOgorchock/joycond), a service that uses `hid-nintendo` to provide nintendo joycond pairing and better nintendo switch pro controller support.
 
+- [multipath](https://github.com/opensvc/multipath-tools), the device mapper multipath (DM-MP) daemon. Available as [services.multipath](#opt-services.multipath.enable).
+
 ## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
 
 - The `security.wrappers` option now requires to always specify an owner, group and whether the setuid/setgid bit should be set.

nixos/modules/module-list.nix

@@ -779,6 +779,7 @@
   ./services/networking/mstpd.nix
   ./services/networking/mtprotoproxy.nix
   ./services/networking/mullvad-vpn.nix
+  ./services/networking/multipath.nix
   ./services/networking/murmur.nix
   ./services/networking/mxisd.nix
   ./services/networking/namecoind.nix

nixos/modules/services/networking/multipath.nix

@@ -0,0 +1,572 @@
+{ config, lib, pkgs, ... }: with lib;
+
+# See http://christophe.varoqui.free.fr/usage.html and
+# https://github.com/opensvc/multipath-tools/blob/master/multipath/multipath.conf.5
+
+let
+  cfg = config.services.multipath;
+
+  indentLines = n: str: concatStringsSep "\n" (
+    map (line: "${fixedWidthString n " " " "}${line}") (
+      filter ( x: x != "" ) ( splitString "\n" str )
+    )
+  );
+
+  addCheckDesc = desc: elemType: check: types.addCheck elemType check
+    // { description = "${elemType.description} (with check: ${desc})"; };
+  hexChars = stringToCharacters "0123456789abcdef";
+  isHexString = s: all (c: elem c hexChars) (stringToCharacters (toLower s));
+  hexStr = addCheckDesc "hexadecimal string" types.str isHexString;
+
+in {
+
+  options.services.multipath = with types; {
+
+    enable = mkEnableOption "the device mapper multipath (DM-MP) daemon";
+
+    package = mkOption {
+      type = package;
+      description = "multipath-tools package to use";
+      default = pkgs.multipath-tools;
+      defaultText = "pkgs.multipath-tools";
+    };
+
+    devices = mkOption {
+      default = [ ];
+      example = literalExpression ''
+        [
+          {
+            vendor = "\"COMPELNT\"";
+            product = "\"Compellent Vol\"";
+            path_checker = "tur";
+            no_path_retry = "queue";
+            max_sectors_kb = 256;
+          }, ...
+        ]
+      '';
+      description = ''
+        This option allows you to define arrays for use in multipath
+        groups.
+      '';
+      type = listOf (submodule {
+        options = {
+
+          vendor = mkOption {
+            type = str;
+            example = "COMPELNT";
+            description = "Regular expression to match the vendor name";
+          };
+
+          product = mkOption {
+            type = str;
+            example = "Compellent Vol";
+            description = "Regular expression to match the product name";
+          };
+
+          revision = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "Regular expression to match the product revision";
+          };
+
+          product_blacklist = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "Products with the given vendor matching this string are blacklisted";
+          };
+
+          alias_prefix = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "The user_friendly_names prefix to use for this device type, instead of the default mpath";
+          };
+
+          vpd_vendor = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "The vendor specific vpd page information, using the vpd page abbreviation";
+          };
+
+          hardware_handler = mkOption {
+            type = nullOr (enum [ "emc" "rdac" "hp_sw" "alua" "ana" ]);
+            default = null;
+            description = "The hardware handler to use for this device type";
+          };
+
+          # Optional arguments
+          path_grouping_policy = mkOption {
+            type = nullOr (enum [ "failover" "multibus" "group_by_serial" "group_by_prio" "group_by_node_name" ]);
+            default = null; # real default: "failover"
+            description = "The default path grouping policy to apply to unspecified multipaths";
+          };
+
+          uid_attribute = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "The udev attribute providing a unique path identifier (WWID)";
+          };
+
+          getuid_callout = mkOption {
+            type = nullOr str;
+            default = null;
+            description = ''
+              (Superseded by uid_attribute) The default program and args to callout
+              to obtain a unique path identifier. Should be specified with an absolute path.
+            '';
+          };
+
+          path_selector = mkOption {
+            type = nullOr (enum [
+              ''"round-robin 0"''
+              ''"queue-length 0"''
+              ''"service-time 0"''
+              ''"historical-service-time 0"''
+            ]);
+            default = null; # real default: "service-time 0"
+            description = "The default path selector algorithm to use; they are offered by the kernel multipath target";
+          };
+
+          path_checker = mkOption {
+            type = enum [ "readsector0" "tur" "emc_clariion" "hp_sw" "rdac" "directio" "cciss_tur" "none" ];
+            default = "tur";
+            description = "The default method used to determine the paths state";
+          };
+
+          prio = mkOption {
+            type = nullOr (enum [
+              "none" "const" "sysfs" "emc" "alua" "ontap" "rdac" "hp_sw" "hds"
+              "random" "weightedpath" "path_latency" "ana" "datacore" "iet"
+            ]);
+            default = null; # real default: "const"
+            description = "The name of the path priority routine";
+          };
+
+          prio_args = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "Arguments to pass to to the prio function";
+          };
+
+          features = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "Specify any device-mapper features to be used";
+          };
+
+          failback = mkOption {
+            type = nullOr str;
+            default = null; # real default: "manual"
+            description = "Tell multipathd how to manage path group failback. Quote integers as strings";
+          };
+
+          rr_weight = mkOption {
+            type = nullOr (enum [ "priorities" "uniform" ]);
+            default = null; # real default: "uniform"
+            description = ''
+              If set to priorities the multipath configurator will assign path weights
+              as "path prio * rr_min_io".
+            '';
+          };
+
+          no_path_retry = mkOption {
+            type = nullOr str;
+            default = null; # real default: "fail"
+            description = "Specify what to do when all paths are down. Quote integers as strings";
+          };
+
+          rr_min_io = mkOption {
+            type = nullOr int;
+            default = null; # real default: 1000
+            description = ''
+              Number of I/O requests to route to a path before switching to the next in the
+              same path group. This is only for Block I/O (BIO) based multipath and
+              only apply to round-robin path_selector.
+            '';
+          };
+
+          rr_min_io_rq = mkOption {
+            type = nullOr int;
+            default = null; # real default: 1
+            description = ''
+              Number of I/O requests to route to a path before switching to the next in the
+              same path group. This is only for Request based multipath and
+              only apply to round-robin path_selector.
+            '';
+          };
+
+          fast_io_fail_tmo = mkOption {
+            type = nullOr str;
+            default = null; # real default: 5
+            description = ''
+              Specify the number of seconds the SCSI layer will wait after a problem has been
+              detected on a FC remote port before failing I/O to devices on that remote port.
+              This should be smaller than dev_loss_tmo. Setting this to "off" will disable
+              the timeout. Quote integers as strings.
+            '';
+          };
+
+          dev_loss_tmo = mkOption {
+            type = nullOr str;
+            default = null; # real default: 600
+            description = ''
+              Specify the number of seconds the SCSI layer will wait after a problem has
+              been detected on a FC remote port before removing it from the system. This
+              can be set to "infinity" which sets it to the max value of 2147483647
+              seconds, or 68 years. It will be automatically adjusted to the overall
+              retry interval no_path_retry * polling_interval
+              if a number of retries is given with no_path_retry and the
+              overall retry interval is longer than the specified dev_loss_tmo value.
+              The Linux kernel will cap this value to 600 if fast_io_fail_tmo
+              is not set.
+            '';
+          };
+
+          flush_on_last_del = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "no"
+            description = ''
+              If set to "yes" multipathd will disable queueing when the last path to a
+              device has been deleted.
+            '';
+          };
+
+          user_friendly_names = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "no"
+            description = ''
+              If set to "yes", using the bindings file /etc/multipath/bindings
+              to assign a persistent and unique alias to the multipath, in the
+              form of mpath. If set to "no" use the WWID as the alias. In either
+              case this be will be overridden by any specific aliases in the
+              multipaths section.
+            '';
+          };
+
+          retain_attached_hw_handler = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "yes"
+            description = ''
+              (Obsolete for kernels >= 4.3) If set to "yes" and the SCSI layer has
+              already attached a hardware_handler to the device, multipath will not
+              force the device to use the hardware_handler specified by mutipath.conf.
+              If the SCSI layer has not attached a hardware handler, multipath will
+              continue to use its configured hardware handler.
+
+              Important Note: Linux kernel 4.3 or newer always behaves as if
+              "retain_attached_hw_handler yes" was set.
+            '';
+          };
+
+          detect_prio = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "yes"
+            description = ''
+              If set to "yes", multipath will try to detect if the device supports
+              SCSI-3 ALUA. If so, the device will automatically use the sysfs
+              prioritizer if the required sysf attributes access_state and
+              preferred_path are supported, or the alua prioritizer if not. If set
+              to "no", the prioritizer will be selected as usual.
+            '';
+          };
+
+          detect_checker = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "yes"
+            description = ''
+              If set to "yes", multipath will try to detect if the device supports
+              SCSI-3 ALUA. If so, the device will automatically use the tur checker.
+              If set to "no", the checker will be selected as usual.
+            '';
+          };
+
+          deferred_remove = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "no"
+            description = ''
+              If set to "yes", multipathd will do a deferred remove instead of a
+              regular remove when the last path device has been deleted. This means
+              that if the multipath device is still in use, it will be freed when
+              the last user closes it. If path is added to the multipath device
+              before the last user closes it, the deferred remove will be canceled.
+            '';
+          };
+
+          san_path_err_threshold = mkOption {
+            type = nullOr str;
+            default = null;
+            description = ''
+              If set to a value greater than 0, multipathd will watch paths and check
+              how many times a path has been failed due to errors.If the number of
+              failures on a particular path is greater then the san_path_err_threshold,
+              then the path will not reinstate till san_path_err_recovery_time. These
+              path failures should occur within a san_path_err_forget_rate checks, if
+              not we will consider the path is good enough to reinstantate.
+            '';
+          };
+
+          san_path_err_forget_rate = mkOption {
+            type = nullOr str;
+            default = null;
+            description = ''
+              If set to a value greater than 0, multipathd will check whether the path
+              failures has exceeded the san_path_err_threshold within this many checks
+              i.e san_path_err_forget_rate. If so we will not reinstante the path till
+              san_path_err_recovery_time.
+            '';
+          };
+
+          san_path_err_recovery_time = mkOption {
+            type = nullOr str;
+            default = null;
+            description = ''
+              If set to a value greater than 0, multipathd will make sure that when
+              path failures has exceeded the san_path_err_threshold within
+              san_path_err_forget_rate then the path will be placed in failed state
+              for san_path_err_recovery_time duration. Once san_path_err_recovery_time
+              has timeout we will reinstante the failed path. san_path_err_recovery_time
+              value should be in secs.
+            '';
+          };
+
+          marginal_path_err_sample_time = mkOption {
+            type = nullOr int;
+            default = null;
+            description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
+          };
+
+          marginal_path_err_rate_threshold = mkOption {
+            type = nullOr int;
+            default = null;
+            description = "The error rate threshold as a permillage (1/1000)";
+          };
+
+          marginal_path_err_recheck_gap_time = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
+          };
+
+          marginal_path_double_failed_time = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
+          };
+
+          delay_watch_checks = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "This option is deprecated, and mapped to san_path_err_forget_rate";
+          };
+
+          delay_wait_checks = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "This option is deprecated, and mapped to san_path_err_recovery_time";
+          };
+
+          skip_kpartx = mkOption {
+            type = nullOr (enum [ "yes" "no" ]);
+            default = null; # real default: "no"
+            description = "If set to yes, kpartx will not automatically create partitions on the device";
+          };
+
+          max_sectors_kb = mkOption {
+            type = nullOr int;
+            default = null;
+            description = "Sets the max_sectors_kb device parameter on all path devices and the multipath device to the specified value";
+          };
+
+          ghost_delay = mkOption {
+            type = nullOr int;
+            default = null;
+            description = "Sets the number of seconds that multipath will wait after creating a device with only ghost paths before marking it ready for use in systemd";
+          };
+
+          all_tg_pt = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "Set the 'all targets ports' flag when registering keys with mpathpersist";
+          };
+
+        };
+      });
+    };
+
+    defaults = mkOption {
+      type = nullOr str;
+      default = null;
+      description = ''
+        This section defines default values for attributes which are used
+        whenever no values are given in the appropriate device or multipath
+        sections.
+      '';
+    };
+
+    blacklist = mkOption {
+      type = nullOr str;
+      default = null;
+      description = ''
+        This section defines which devices should be excluded from the
+        multipath topology discovery.
+      '';
+    };
+
+    blacklist_exceptions = mkOption {
+      type = nullOr str;
+      default = null;
+      description = ''
+        This section defines which devices should be included in the
+        multipath topology discovery, despite being listed in the
+        blacklist section.
+      '';
+    };
+
+    overrides = mkOption {
+      type = nullOr str;
+      default = null;
+      description = ''
+        This section defines values for attributes that should override the
+        device-specific settings for all devices.
+      '';
+    };
+
+    extraConfig = mkOption {
+      type = nullOr str;
+      default = null;
+      description = "Lines to append to default multipath.conf";
+    };
+
+    extraConfigFile = mkOption {
+      type = nullOr str;
+      default = null;
+      description = "Append an additional file's contents to /etc/multipath.conf";
+    };
+
+    pathGroups = mkOption {
+      example = literalExpression ''
+        [
+          {
+            wwid = "360080e500043b35c0123456789abcdef";
+            alias = 10001234;
+            array = "bigarray.example.com";
+            fsType = "zfs"; # optional
+            options = "ro"; # optional
+          }, ...
+        ]
+      '';
+      description = ''
+        This option allows you to define multipath groups as described
+        in http://christophe.varoqui.free.fr/usage.html.
+      '';
+      type = listOf (submodule {
+        options = {
+
+          alias = mkOption {
+            type = int;
+            example = 1001234;
+            description = "The name of the multipath device";
+          };
+
+          wwid = mkOption {
+            type = hexStr;
+            example = "360080e500043b35c0123456789abcdef";
+            description = "The identifier for the multipath device";
+          };
+
+          array = mkOption {
+            type = str;
+            default = null;
+            example = "bigarray.example.com";
+            description = "The DNS name of the storage array";
+          };
+
+          fsType = mkOption {
+            type = nullOr str;
+            default = null;
+            example = "zfs";
+            description = "Type of the filesystem";
+          };
+
+          options = mkOption {
+            type = nullOr str;
+            default = null;
+            example = "ro";
+            description = "Options used to mount the file system";
+          };
+
+        };
+      });
+    };
+
+  };
+
+  config = mkIf cfg.enable {
+    environment.etc."multipath.conf".text =
+      let
+        inherit (cfg) defaults blacklist blacklist_exceptions overrides;
+
+        mkDeviceBlock = cfg: let
+          nonNullCfg = lib.filterAttrs (k: v: v != null) cfg;
+          attrs = lib.mapAttrsToList (name: value: "  ${name} ${toString value}") nonNullCfg;
+        in ''
+          device {
+          ${lib.concatStringsSep "\n" attrs}
+          }
+        '';
+        devices = lib.concatMapStringsSep "\n" mkDeviceBlock cfg.devices;
+
+        mkMultipathBlock = m: ''
+          multipath {
+            wwid ${m.wwid}
+            alias ${toString m.alias}
+          }
+        '';
+        multipaths = lib.concatMapStringsSep "\n" mkMultipathBlock cfg.pathGroups;
+
+      in ''
+        devices {
+        ${indentLines 2 devices}
+        }
+
+        ${optionalString (!isNull defaults) ''
+          defaults {
+          ${indentLines 2 defaults}
+            multipath_dir ${cfg.package}/lib/multipath
+          }
+        ''}
+        ${optionalString (!isNull blacklist) ''
+          blacklist {
+          ${indentLines 2 blacklist}
+          }
+        ''}
+        ${optionalString (!isNull blacklist_exceptions) ''
+          blacklist_exceptions {
+          ${indentLines 2 blacklist_exceptions}
+          }
+        ''}
+        ${optionalString (!isNull overrides) ''
+          overrides {
+          ${indentLines 2 overrides}
+          }
+        ''}
+        multipaths {
+        ${indentLines 2 multipaths}
+        }
+      '';
+
+    systemd.packages = [ cfg.package ];
+
+    environment.systemPackages = [ cfg.package ];
+    boot.kernelModules = [ "dm-multipath" "dm-service-time" ];
+
+    # We do not have systemd in stage-1 boot so must invoke `multipathd`
+    # with the `-1` argument which disables systemd calls. Invoke `multipath`
+    # to display the multipath mappings in the output of `journalctl -b`.
+    boot.initrd.kernelModules = [ "dm-multipath" "dm-service-time" ];
+    boot.initrd.postDeviceCommands = ''
+      modprobe -a dm-multipath dm-service-time
+      multipathd -s
+      (set -x && sleep 1 && multipath -ll)
+    '';
+  };
+}

nixos/modules/system/boot/stage-1.nix

@@ -137,6 +137,14 @@ let
         copy_bin_and_libs ${pkgs.e2fsprogs}/sbin/resize2fs
       ''}
 
+      # Copy multipath.
+      ${optionalString config.services.multipath.enable ''
+        copy_bin_and_libs ${config.services.multipath.package}/bin/multipath
+        copy_bin_and_libs ${config.services.multipath.package}/bin/multipathd
+        # Copy lib/multipath manually.
+        cp -rpv ${config.services.multipath.package}/lib/multipath $out/lib
+      ''}
+
       # Copy secrets if needed.
       #
       # TODO: move out to a separate script; see #85000.
@@ -199,6 +207,10 @@ let
       $out/bin/dmsetup --version 2>&1 | tee -a log | grep -q "version:"
       LVM_SYSTEM_DIR=$out $out/bin/lvm version 2>&1 | tee -a log | grep -q "LVM"
       $out/bin/mdadm --version
+      ${optionalString config.services.multipath.enable ''
+        ($out/bin/multipath || true) 2>&1 | grep -q 'need to be root'
+        ($out/bin/multipathd || true) 2>&1 | grep -q 'need to be root'
+      ''}
 
       ${config.boot.initrd.extraUtilsCommandsTest}
       fi
@@ -338,6 +350,18 @@ let
         { object = pkgs.kmod-debian-aliases;
           symlink = "/etc/modprobe.d/debian.conf";
         }
+      ] ++ lib.optionals config.services.multipath.enable [
+        { object = pkgs.runCommand "multipath.conf" {
+              src = config.environment.etc."multipath.conf".text;
+              preferLocalBuild = true;
+            } ''
+              target=$out
+              printf "$src" > $out
+              substituteInPlace $out \
+                --replace ${config.services.multipath.package}/lib ${extraUtils}/lib
+            '';
+          symlink = "/etc/multipath.conf";
+        }
       ] ++ (lib.mapAttrsToList
         (symlink: options:
           {

nixos/tests/iscsi-multipath-root.nix

@@ -0,0 +1,267 @@
+import ./make-test-python.nix (
+  { pkgs, lib, ... }:
+  let
+    initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
+    targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
+  in
+  {
+    name = "iscsi";
+    meta = {
+      maintainers = pkgs.lib.teams.deshaw.members;
+    };
+
+    nodes = {
+      target = { config, pkgs, lib, ... }: {
+        virtualisation.vlans = [ 1 2 ];
+        services.target = {
+          enable = true;
+          config = {
+            fabric_modules = [ ];
+            storage_objects = [
+              {
+                dev = "/dev/vdb";
+                name = "test";
+                plugin = "block";
+                write_back = true;
+                wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
+              }
+            ];
+            targets = [
+              {
+                fabric = "iscsi";
+                tpgs = [
+                  {
+                    enable = true;
+                    attributes = {
+                      authentication = 0;
+                      generate_node_acls = 1;
+                    };
+                    luns = [
+                      {
+                        alias = "94dfe06967";
+                        alua_tg_pt_gp_name = "default_tg_pt_gp";
+                        index = 0;
+                        storage_object = "/backstores/block/test";
+                      }
+                    ];
+                    node_acls = [
+                      {
+                        mapped_luns = [
+                          {
+                            alias = "d42f5bdf8a";
+                            index = 0;
+                            tpg_lun = 0;
+                            write_protect = false;
+                          }
+                        ];
+                        node_wwn = initiatorName;
+                      }
+                    ];
+                    portals = [
+                      {
+                        ip_address = "0.0.0.0";
+                        iser = false;
+                        offload = false;
+                        port = 3260;
+                      }
+                    ];
+                    tag = 1;
+                  }
+                ];
+                wwn = targetName;
+              }
+            ];
+          };
+        };
+
+        networking.firewall.allowedTCPPorts = [ 3260 ];
+        networking.firewall.allowedUDPPorts = [ 3260 ];
+
+        virtualisation.memorySize = 2048;
+        virtualisation.emptyDiskImages = [ 2048 ];
+      };
+
+      initiatorAuto = { nodes, config, pkgs, ... }: {
+        virtualisation.vlans = [ 1 2 ];
+
+        services.multipath = {
+          enable = true;
+          defaults = ''
+            find_multipaths yes
+            user_friendly_names yes
+          '';
+          pathGroups = [
+            {
+              alias = 123456;
+              wwid = "3600140592b17c3f6b404168b082ceeb7";
+            }
+          ];
+        };
+
+        services.openiscsi = {
+          enable = true;
+          enableAutoLoginOut = true;
+          discoverPortal = "target";
+          name = initiatorName;
+        };
+
+        environment.systemPackages = with pkgs; [
+          xfsprogs
+        ];
+
+        environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
+
+        nix.binaryCaches = lib.mkForce [ ];
+        nix.extraOptions = ''
+          hashed-mirrors =
+          connect-timeout = 1
+        '';
+      };
+
+      initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
+        boot.initrd.network.enable = true;
+        boot.loader.grub.enable = false;
+
+        boot.kernelParams = lib.mkOverride 5 (
+          [
+            "boot.shell_on_fail"
+            "console=tty1"
+            "ip=192.168.1.1:::255.255.255.0::ens9:none"
+            "ip=192.168.2.1:::255.255.255.0::ens10:none"
+          ]
+        );
+
+        # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
+        virtualisation.writableStore = false;
+        virtualisation.vlans = [ 1 2 ];
+
+        services.multipath = {
+          enable = true;
+          defaults = ''
+            find_multipaths yes
+            user_friendly_names yes
+          '';
+          pathGroups = [
+            {
+              alias = 123456;
+              wwid = "3600140592b17c3f6b404168b082ceeb7";
+            }
+          ];
+        };
+
+        fileSystems = lib.mkOverride 5 {
+          "/" = {
+            fsType = "xfs";
+            device = "/dev/mapper/123456";
+            options = [ "_netdev" ];
+          };
+        };
+
+        boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
+
+        boot.iscsi-initiator = {
+          discoverPortal = "target";
+          name = initiatorName;
+          target = targetName;
+          extraIscsiCommands = ''
+            iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
+          '';
+        };
+      };
+
+    };
+
+    testScript = { nodes, ... }: ''
+      target.start()
+      target.wait_for_unit("iscsi-target.service")
+
+      initiatorAuto.start()
+
+      initiatorAuto.wait_for_unit("iscsid.service")
+      initiatorAuto.wait_for_unit("iscsi.service")
+      initiatorAuto.get_unit_info("iscsi")
+
+      # Expecting this to fail since we should already know about 192.168.1.3
+      initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
+      # Expecting this to succeed since we don't yet know about 192.168.2.3
+      initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
+
+      # /dev/sda is provided by iscsi on target
+      initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
+
+      initiatorAuto.succeed("mkfs.xfs /dev/sda")
+      initiatorAuto.succeed("mkdir /mnt")
+
+      # Start by verifying /dev/sda and /dev/sdb are both the same disk
+      initiatorAuto.succeed("mount /dev/sda /mnt")
+      initiatorAuto.succeed("touch /mnt/hi")
+      initiatorAuto.succeed("umount /mnt")
+
+      initiatorAuto.succeed("mount /dev/sdb /mnt")
+      initiatorAuto.succeed("test -e /mnt/hi")
+      initiatorAuto.succeed("umount /mnt")
+
+      initiatorAuto.succeed("systemctl restart multipathd")
+      initiatorAuto.succeed("multipath -ll | systemd-cat")
+
+      # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
+      initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
+      initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
+      initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
+      initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
+      initiatorAuto.succeed(
+        "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
+      )
+      initiatorAuto.succeed("umount /mnt")
+      initiatorAuto.shutdown()
+
+      initiatorRootDisk.start()
+      initiatorRootDisk.wait_for_unit("multi-user.target")
+      initiatorRootDisk.wait_for_unit("iscsid")
+
+      # Log in over both nodes
+      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
+      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
+      initiatorRootDisk.succeed("systemctl restart multipathd")
+      initiatorRootDisk.succeed("multipath -ll | systemd-cat")
+
+      # Verify we can write and sync the root disk
+      initiatorRootDisk.succeed("mkdir /scratch")
+      initiatorRootDisk.succeed("touch /scratch/both-up")
+      initiatorRootDisk.succeed("sync /scratch")
+
+      # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
+      initiatorRootDisk.succeed("ip link set ens9 down")
+      initiatorRootDisk.succeed("touch /scratch/ens9-down")
+      initiatorRootDisk.succeed("sync /scratch")
+      initiatorRootDisk.succeed("ip link set ens9 up")
+
+      # todo: better way to wait until multipath notices the link is back
+      initiatorRootDisk.succeed("sleep 5")
+      initiatorRootDisk.succeed("touch /scratch/both-down")
+      initiatorRootDisk.succeed("sync /scratch")
+
+      # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
+      initiatorRootDisk.succeed("ip link set ens10 down")
+      initiatorRootDisk.succeed("touch /scratch/ens10-down")
+      initiatorRootDisk.succeed("sync /scratch")
+      initiatorRootDisk.succeed("ip link set ens10 up")
+      initiatorRootDisk.succeed("touch /scratch/ens10-down")
+      initiatorRootDisk.succeed("sync /scratch")
+
+      initiatorRootDisk.succeed("ip link set ens9 up")
+      initiatorRootDisk.succeed("ip link set ens10 up")
+      initiatorRootDisk.shutdown()
+
+      # Verify we can boot with the target's eth1 down, forcing
+      # it to multipath via the second link
+      target.succeed("ip link set eth1 down")
+      initiatorRootDisk.start()
+      initiatorRootDisk.wait_for_unit("multi-user.target")
+      initiatorRootDisk.wait_for_unit("iscsid")
+      initiatorRootDisk.succeed("test -e /scratch/both-up")
+    '';
+  }
+)
+
+