make-disk-image: change to be less VM-centric

This changes much of the make-disk-image.nix logic (and thus most NixOS image building) to use LKL to set up the target directory structure rather than a Linux VM. The only work we still do in a VM is less IO-heavy stuff that while still time-consuming, is less of the overall load. The goal is to kill more of that stuff, but that will require deeper changes to NixOS activation scripts and switch-to-configuration.pl, and I don't want to bite off too much at once.
7 years ago · f1708a9d7d
parent 5c7f4669a7
commit f1708a9d7d
3 changed files with 125 additions and 86 deletions
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@ -33,42 +33,124 @@

 , name ? "nixos-disk-image"

-  # This prevents errors while checking nix-store validity, see
-  # https://github.com/NixOS/nix/issues/1134
-, fixValidity ? true
-
 , format ? "raw"
 }:

 with lib;

-pkgs.vmTools.runInLinuxVM (
+let
+  # Copied from https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/installer/cd-dvd/channel.nix
+  # TODO: factor out more cleanly
+
+  # Do not include these things:
+  #   - The '.git' directory
+  #   - Result symlinks from nix-build ('result', 'result-2', 'result-bin', ...)
+  #   - VIM/Emacs swap/backup files ('.swp', '.swo', '.foo.swp', 'foo~', ...)
+  filterFn = path: type: let basename = baseNameOf (toString path); in
+    if type == "directory" then basename != ".git"
+    else if type == "symlink" then builtins.match "^result(|-.*)$" basename == null
+    else builtins.match "^((|\..*)\.sw[a-z]|.*~)$" basename == null;
+
+  nixpkgs = builtins.filterSource filterFn pkgs.path;
+
+  channelSources = pkgs.runCommand "nixos-${config.system.nixosVersion}" {} ''
+    mkdir -p $out
+    cp -prd ${nixpkgs} $out/nixos
+    chmod -R u+w $out/nixos
+    if [ ! -e $out/nixos/nixpkgs ]; then
+      ln -s . $out/nixos/nixpkgs
+    fi
+    rm -rf $out/nixos/.git
+    echo -n ${config.system.nixosVersionSuffix} > $out/nixos/.version-suffix
+  '';
+
+  metaClosure = pkgs.writeText "meta" ''
+    ${config.system.build.toplevel}
+    ${config.nix.package.out}
+    ${channelSources}
+  '';
+
+  prepareImageInputs = with pkgs; [ rsync utillinux parted e2fsprogs lkl fakeroot config.system.build.nixos-prepare-root ] ++ stdenv.initialPath;
+
+  # I'm preserving the line below because I'm going to search for it across nixpkgs to consolidate
+  # image building logic. The comment right below this now appears in 4 different places in nixpkgs :)
+  # !!! should use XML.
+  sources = map (x: x.source) contents;
+  targets = map (x: x.target) contents;
+
+  prepareImage = ''
+    export PATH=${pkgs.lib.makeSearchPathOutput "bin" "bin" prepareImageInputs}
+
+    mkdir $out
+    diskImage=nixos.raw
+    truncate -s ${toString diskSize}M $diskImage
+
+    ${if partitioned then ''
+      parted $diskImage -- mklabel msdos mkpart primary ext4 1M -1s
+      offset=$((2048*512))
+    '' else ''
+      offset=0
+    ''}
+
+    mkfs.${fsType} -F -L nixos -E offset=$offset $diskImage
+  
+    root="$PWD/root"
+    mkdir -p $root
+
+    # Copy arbitrary other files into the image
+    # Semi-shamelessly copied from make-etc.sh. I (@copumpkin) shall factor this stuff out as part of
+    # https://github.com/NixOS/nixpkgs/issues/23052.
+    set -f
+    sources_=(${concatStringsSep " " sources})
+    targets_=(${concatStringsSep " " targets})
+    set +f
+
+    for ((i = 0; i < ''${#targets_[@]}; i++)); do
+      source="''${sources_[$i]}"
+      target="''${targets_[$i]}"
+
+      if [[ "$source" =~ '*' ]]; then
+        # If the source name contains '*', perform globbing.
+        mkdir -p $root/$target
+        for fn in $source; do
+          rsync -a --no-o --no-g "$fn" $root/$target/
+        done
+      else
+        mkdir -p $root/$(dirname $target)
+        if ! [ -e $root/$target ]; then
+          rsync -a --no-o --no-g $source $root/$target
+        else
+          echo "duplicate entry $target -> $source"
+          exit 1
+        fi
+      fi
+    done
+
+    # TODO: Nix really likes to chown things it creates to its current user...
+    fakeroot nixos-prepare-root $root ${channelSources} ${config.system.build.toplevel} closure
+
+    echo "copying staging root to image..."
+    cptofs ${pkgs.lib.optionalString partitioned "-P 1"} -t ${fsType} -i $diskImage $root/* /
+  '';
+in pkgs.vmTools.runInLinuxVM (
  pkgs.runCommand name
-    { preVM =
-        ''
-          mkdir $out
-          diskImage=$out/nixos.${if format == "qcow2" then "qcow2" else "img"}
-          ${pkgs.vmTools.qemu}/bin/qemu-img create -f ${format} $diskImage "${toString diskSize}M"
-          mv closure xchg/
-        '';
-      buildInputs = with pkgs; [ utillinux perl e2fsprogs parted rsync ];
-
-      # I'm preserving the line below because I'm going to search for it across nixpkgs to consolidate
-      # image building logic. The comment right below this now appears in 4 different places in nixpkgs :)
-      # !!! should use XML.
-      sources = map (x: x.source) contents;
-      targets = map (x: x.target) contents;
-
-      exportReferencesGraph =
-        [ "closure" config.system.build.toplevel ];
-      inherit postVM;
+    { preVM = prepareImage;
+      buildInputs = with pkgs; [ utillinux e2fsprogs ];
+      exportReferencesGraph = [ "closure" metaClosure ];
+      postVM = ''
+        ${if format == "raw" then ''
+          mv $diskImage $out/nixos.img
+          diskImage=$out/nixos.img
+        '' else ''
+          ${pkgs.qemu}/bin/qemu-img convert -f raw -O qcow2 $diskImage $out/nixos.qcow2
+          diskImage=$out/nixos.qcow2
+        ''}
+        ${postVM}
+      '';
      memSize = 1024;
    }
    ''
      ${if partitioned then ''
-        # Create a single / partition.
-        parted /dev/vda mklabel msdos
-        parted /dev/vda -- mkpart primary ext2 1M -1s
        . /sys/class/block/vda1/uevent
        mknod /dev/vda1 b $MAJOR $MINOR
        rootDisk=/dev/vda1
@ -76,74 +158,34 @@ pkgs.vmTools.runInLinuxVM (
        rootDisk=/dev/vda
      ''}

-      # Create an empty filesystem and mount it.
-      mkfs.${fsType} -L nixos $rootDisk
-      mkdir /mnt
-      mount $rootDisk /mnt
-
-      # Register the paths in the Nix database.
-      printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
-          ${config.nix.package.out}/bin/nix-store --load-db --option build-users-group ""
-
-      ${if fixValidity then ''
-        # Add missing size/hash fields to the database. FIXME:
-        # exportReferencesGraph should provide these directly.
-        ${config.nix.package.out}/bin/nix-store --verify --check-contents --option build-users-group ""
-      '' else ""}
-
-      # In case the bootloader tries to write to /dev/sda…
+      # Some tools assume these exist
      ln -s vda /dev/xvda
      ln -s vda /dev/sda

-      # Install the closure onto the image
-      USER=root ${config.system.build.nixos-install}/bin/nixos-install \
-        --closure ${config.system.build.toplevel} \
-        --no-channel-copy \
-        --no-root-passwd \
-        ${optionalString (!installBootLoader) "--no-bootloader"}
+      mountPoint=/mnt
+      mkdir $mountPoint
+      mount $rootDisk $mountPoint

-      # Install a configuration.nix.
+      # Install a configuration.nix
      mkdir -p /mnt/etc/nixos
      ${optionalString (configFile != null) ''
        cp ${configFile} /mnt/etc/nixos/configuration.nix
      ''}

-      # Remove /etc/machine-id so that each machine cloning this image will get its own id
-      rm -f /mnt/etc/machine-id
-
-      # Copy arbitrary other files into the image
-      # Semi-shamelessly copied from make-etc.sh. I (@copumpkin) shall factor this stuff out as part of
-      # https://github.com/NixOS/nixpkgs/issues/23052.
-      set -f
-      sources_=($sources)
-      targets_=($targets)
-      set +f
-
-      for ((i = 0; i < ''${#targets_[@]}; i++)); do
-        source="''${sources_[$i]}"
-        target="''${targets_[$i]}"
+      mount --rbind /dev  $mountPoint/dev
+      mount --rbind /proc $mountPoint/proc
+      mount --rbind /sys  $mountPoint/sys

-        if [[ "$source" =~ '*' ]]; then
+      # Set up core system link, GRUB, etc.
+      NIXOS_INSTALL_BOOTLOADER=1 chroot $mountPoint /nix/var/nix/profiles/system/bin/switch-to-configuration boot

-          # If the source name contains '*', perform globbing.
-          mkdir -p /mnt/$target
-          for fn in $source; do
-            rsync -a --no-o --no-g "$fn" /mnt/$target/
-          done
+      # TODO: figure out if I should activate, but for now I won't
+      # chroot $mountPoint /nix/var/nix/profiles/system/activate

-        else
-
-          mkdir -p /mnt/$(dirname $target)
-          if ! [ -e /mnt/$target ]; then
-            rsync -a --no-o --no-g $source /mnt/$target
-          else
-            echo "duplicate entry $target -> $source"
-            exit 1
-          fi
-        fi
-      done
+      # The above scripts will generate a random machine-id and we don't want to bake a single ID into all our images
+      rm -f $mountPoint/etc/machine-id

-      umount /mnt
+      umount -R /mnt

      # Make sure resize2fs works. Note that resize2fs has stricter criteria for resizing than a normal
      # mount, so the `-c 0` and `-i 0` don't affect it. Setting it to `now` doesn't produce deterministic
--- a/nixos/maintainers/scripts/ec2/amazon-image.nix
+++ b/nixos/maintainers/scripts/ec2/amazon-image.nix
@ -6,10 +6,7 @@ let
  cfg = config.amazonImage;
 in {

-  imports =
-    [ ../../../modules/installer/cd-dvd/channel.nix
-      ../../../modules/virtualisation/amazon-image.nix
-    ];
+  imports = [ ../../../modules/virtualisation/amazon-image.nix ];

  options.amazonImage = {
    contents = mkOption {
--- a/nixos/modules/installer/tools/nixos-prepare-root.sh
+++ b/nixos/modules/installer/tools/nixos-prepare-root.sh
@ -70,7 +70,7 @@ for i in $closures; do
            rsync -a $j $mountPoint/nix/store/
        done

-        nix-store --register-validity < $i
+        nix-store --option build-users-group root --register-validity < $i
    fi
 done