Merge pull request #171647 from hercules-ci/testers

`testers`: docs and move

doc/builders/testers.chapter.md

@@ -80,3 +80,49 @@ tests.fetchgit = invalidateFetcherByDrvHash fetchgit {
   sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
 };
 ```
+
+## `nixosTest` {#tester-nixosTest}
+
+Run a NixOS VM network test using this evaluation of Nixpkgs.
+
+NOTE: This function is primarily for external use. NixOS itself uses `make-test-python.nix` directly. Packages defined in Nixpkgs [reuse NixOS tests via `nixosTests`, plural](#ssec-nixos-tests-linking).
+
+It is mostly equivalent to the function `import ./make-test-python.nix` from the
+[NixOS manual](https://nixos.org/nixos/manual/index.html#sec-nixos-tests),
+except that the current application of Nixpkgs (`pkgs`) will be used, instead of
+letting NixOS invoke Nixpkgs anew.
+
+If a test machine needs to set NixOS options under `nixpkgs`, it must set only the
+`nixpkgs.pkgs` option.
+
+### Parameter
+
+A [NixOS VM test network](https://nixos.org/nixos/manual/index.html#sec-nixos-tests), or path to it. Example:
+
+```nix
+{
+  name = "my-test";
+  nodes = {
+    machine1 = { lib, pkgs, nodes, ... }: {
+      environment.systemPackages = [ pkgs.hello ];
+      services.foo.enable = true;
+    };
+    # machine2 = ...;
+  };
+  testScript = ''
+    start_all()
+    machine1.wait_for_unit("foo.service")
+    machine1.succeed("hello | foo-send")
+  '';
+}
+```
+
+### Result
+
+A derivation that runs the VM test.
+
+Notable attributes:
+
+ * `nodes`: the evaluated NixOS configurations. Useful for debugging and exploring the configuration.
+
+ * `driverInteractive`: a script that launches an interactive Python session in the context of the `testScript`.

nixos/doc/manual/from_md/release-notes/rl-2111.section.xml

@@ -569,8 +569,9 @@
       <listitem>
         <para>
           The NixOS VM test framework,
-          <literal>pkgs.nixosTest</literal>/<literal>make-test-python.nix</literal>,
-          now requires detaching commands such as
+          <literal>pkgs.nixosTest</literal>/<literal>make-test-python.nix</literal>
+          (<literal>pkgs.testers.nixosTest</literal> since 22.05), now
+          requires detaching commands such as
           <literal>succeed(&quot;foo &amp;&quot;)</literal> and
           <literal>succeed(&quot;foo | xclip -i&quot;)</literal> to
           close stdout. This can be done with a redirect such as

nixos/doc/manual/release-notes/rl-2111.section.md

@@ -166,7 +166,7 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 ## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
 
-- The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix`, now requires detaching commands such as `succeed("foo &")` and `succeed("foo | xclip -i")` to close stdout.
+- The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix` (`pkgs.testers.nixosTest` since 22.05), now requires detaching commands such as `succeed("foo &")` and `succeed("foo | xclip -i")` to close stdout.
   This can be done with a redirect such as `succeed("foo >&2 &")`. This breaking change was necessitated by a race condition causing tests to fail or hang.
   It applies to all methods that invoke commands on the nodes, including `execute`, `succeed`, `fail`, `wait_until_succeeds`, `wait_until_fails`.
 

nixos/lib/build-vms.nix

@@ -38,7 +38,7 @@ rec {
           { key = "no-revision";
             # Make the revision metadata constant, in order to avoid needless retesting.
             # The human version (e.g. 21.05-pre) is left as is, because it is useful
-            # for external modules that test with e.g. nixosTest and rely on that
+            # for external modules that test with e.g. testers.nixosTest and rely on that
             # version number.
             config.system.nixos.revision = mkForce "constant-nixos-revision";
           }

nixos/lib/testing-python.nix

@@ -119,6 +119,7 @@ rec {
         passthru = passthru // {
           inherit nodes;
         };
+        meta.mainProgram = "nixos-test-driver";
       }
       ''
         mkdir -p $out/bin

pkgs/build-support/docker/examples.nix

@@ -97,7 +97,7 @@ rec {
   };
   # Same example, but re-fetches every time the fetcher implementation changes.
   # NOTE: Only use this for testing, or you'd be wasting a lot of time, network and space.
-  testNixFromDockerHub = pkgs.invalidateFetcherByDrvHash pullImage {
+  testNixFromDockerHub = pkgs.testers.invalidateFetcherByDrvHash pullImage {
     imageName = "nixos/nix";
     imageDigest = "sha256:85299d86263a3059cf19f419f9d286cc9f06d3c13146a8ebbb21b3437f598357";
     sha256 = "19fw0n3wmddahzr20mhdqv6jkjn1kanh6n2mrr08ai53dr8ph5n7";

pkgs/build-support/fetchfirefoxaddon/tests.nix

@@ -1,7 +1,7 @@
-{ invalidateFetcherByDrvHash, fetchFirefoxAddon, fetchurl, ... }:
+{ testers, fetchFirefoxAddon, fetchurl, ... }:
 
 {
-  simple = invalidateFetcherByDrvHash fetchFirefoxAddon {
+  simple = testers.invalidateFetcherByDrvHash fetchFirefoxAddon {
     name = "image-search-options";
     # Chosen because its only 147KB
     url = "https://addons.mozilla.org/firefox/downloads/file/3059971/image_search_options-3.0.12-fx.xpi";
@@ -14,7 +14,7 @@
         sha256 = "sha256-H73YWX/DKxvhEwKpWOo7orAQ7c/rQywpljeyxYxv0Gg=";
       };
     in
-    invalidateFetcherByDrvHash fetchFirefoxAddon {
+    testers.invalidateFetcherByDrvHash fetchFirefoxAddon {
       name = "image-search-options";
       src = image-search-options;
     };

pkgs/build-support/fetchgit/tests.nix

@@ -1,14 +1,14 @@
-{ invalidateFetcherByDrvHash, fetchgit, ... }:
+{ testers, fetchgit, ... }:
 
 {
-  simple = invalidateFetcherByDrvHash fetchgit {
+  simple = testers.invalidateFetcherByDrvHash fetchgit {
     name = "nix-source";
     url = "https://github.com/NixOS/nix";
     rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
     sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
   };
 
-  sparseCheckout = invalidateFetcherByDrvHash fetchgit {
+  sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit {
     name = "nix-source";
     url = "https://github.com/NixOS/nix";
     rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";

pkgs/build-support/fetchpatch/tests.nix

@@ -1,18 +1,18 @@
-{ invalidateFetcherByDrvHash, fetchpatch, ... }:
+{ testers, fetchpatch, ... }:
 
 {
-  simple = invalidateFetcherByDrvHash fetchpatch {
+  simple = testers.invalidateFetcherByDrvHash fetchpatch {
     url = "https://github.com/facebook/zstd/pull/2724/commits/e1f85dbca3a0ed5ef06c8396912a0914db8dea6a.patch";
     sha256 = "sha256-PuYAqnJWAE+L9bsroOnnBGJhERW8LHrGSLtIEkKU9vg=";
   };
 
-  relative = invalidateFetcherByDrvHash fetchpatch {
+  relative = testers.invalidateFetcherByDrvHash fetchpatch {
     url = "https://github.com/boostorg/math/commit/7d482f6ebc356e6ec455ccb5f51a23971bf6ce5b.patch";
     relative = "include";
     sha256 = "sha256-KlmIbixcds6GyKYt1fx5BxDIrU7msrgDdYo9Va/KJR4=";
   };
 
-  full = invalidateFetcherByDrvHash fetchpatch {
+  full = testers.invalidateFetcherByDrvHash fetchpatch {
     url = "https://github.com/boostorg/math/commit/7d482f6ebc356e6ec455ccb5f51a23971bf6ce5b.patch";
     relative = "test";
     stripLen = 1;

pkgs/build-support/node/fetch-yarn-deps/tests/default.nix

@@ -1,15 +1,15 @@
-{ invalidateFetcherByDrvHash, fetchYarnDeps, ... }:
+{ testers, fetchYarnDeps, ... }:
 
 {
-  simple = invalidateFetcherByDrvHash fetchYarnDeps {
+  simple = testers.invalidateFetcherByDrvHash fetchYarnDeps {
     yarnLock = ./simple.lock;
     sha256 = "sha256-Erdkw2E8wWT09jFNLXGkrdwKl0HuSZWnUDJUrV95vSE=";
   };
-  gitDep = invalidateFetcherByDrvHash fetchYarnDeps {
+  gitDep = testers.invalidateFetcherByDrvHash fetchYarnDeps {
     yarnLock = ./git.lock;
     sha256 = "sha256-lAqN4LpoE+jgsQO1nDtuORwcVEO7ogEV53jCu2jFJUI=";
   };
-  githubDep = invalidateFetcherByDrvHash fetchYarnDeps {
+  githubDep = testers.invalidateFetcherByDrvHash fetchYarnDeps {
     yarnLock = ./github.lock;
     sha256 = "sha256-Tsfgyjxz8x6gNmfN0xR7G/NQNoEs4svxRN/N+26vfJU=";
   };

pkgs/build-support/testers/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, lib, callPackage, runCommand }:
+{ pkgs, lib, callPackage, runCommand, stdenv }:
 # Documentation is in doc/builders/testers.chapter.md
 {
   testEqualDerivation = callPackage ./test-equal-derivation.nix { };
@@ -15,4 +15,49 @@
         echo "$output" >&2 && exit 1
       fi
     '';
+
+  # See doc/builders/testers.chapter.md or
+  # https://nixos.org/manual/nixpkgs/unstable/#tester-invalidateFetcherByDrvHash
+  invalidateFetcherByDrvHash = f: args:
+    let
+      drvPath = (f args).drvPath;
+      # It's safe to discard the context, because we don't access the path.
+      salt = builtins.unsafeDiscardStringContext (lib.substring 0 12 (baseNameOf drvPath));
+      # New derivation incorporating the original drv hash in the name
+      salted = f (args // { name = "${args.name or "source"}-salted-${salt}"; });
+      # Make sure we did change the derivation. If the fetcher ignores `name`,
+      # `invalidateFetcherByDrvHash` doesn't work.
+      checked =
+        if salted.drvPath == drvPath
+        then throw "invalidateFetcherByDrvHash: Adding the derivation hash to the fixed-output derivation name had no effect. Make sure the fetcher's name argument ends up in the derivation name. Otherwise, the fetcher will not be re-run when its implementation changes. This is important for testing."
+        else salted;
+    in checked;
+
+  # See doc/builders/testers.chapter.md or
+  # https://nixos.org/manual/nixpkgs/unstable/#tester-invalidateFetcherByDrvHash
+  nixosTest =
+    let
+      /* The nixos/lib/testing-python.nix module, preapplied with arguments that
+       * make sense for this evaluation of Nixpkgs.
+       */
+      nixosTesting =
+        (import ../../../nixos/lib/testing-python.nix {
+          inherit (stdenv.hostPlatform) system;
+          inherit pkgs;
+          extraConfigurations = [(
+            { lib, ... }: {
+              config.nixpkgs.pkgs = lib.mkDefault pkgs;
+            }
+          )];
+        });
+    in
+      test:
+        let
+          loadedTest = if builtins.typeOf test == "path"
+            then import test
+            else test;
+          calledTest = lib.toFunction loadedTest pkgs;
+        in
+          nixosTesting.makeTest calledTest;
+
 }

pkgs/build-support/testers/test/README.md

@@ -0,0 +1,8 @@
+# Tests _for the testers_
+
+    cd nixpkgs
+    nix-build -A tests.testers
+
+Tests generally derive their own correctness from simplicity, which in the
+case of testers (themselves functions) does not always work out.
+Hence the need for tests that test the testers.

pkgs/build-support/testers/test/default.nix

@@ -0,0 +1,27 @@
+{ testers, lib, pkgs, ... }:
+let
+  pkgs-with-overlay = pkgs.extend(final: prev: {
+    proof-of-overlay-hello = prev.hello;
+  });
+
+  dummyVersioning = {
+    revision = "test";
+    versionSuffix = "test";
+    label = "test";
+  };
+
+in
+lib.recurseIntoAttrs {
+  # Check that the wiring of nixosTest is correct.
+  # Correct operation of the NixOS test driver should be asserted elsewhere.
+  nixosTest-example = pkgs-with-overlay.testers.nixosTest ({ lib, pkgs, figlet, ... }: {
+    name = "nixosTest-test";
+    nodes.machine = { pkgs, ... }: {
+      system.nixos = dummyVersioning;
+      environment.systemPackages = [ pkgs.proof-of-overlay-hello figlet ];
+    };
+    testScript = ''
+      machine.succeed("hello | figlet >/dev/console")
+    '';
+  });
+}

pkgs/build-support/trivial-builders/test/references.nix

@@ -1,4 +1,4 @@
-{ lib, nixosTest, pkgs, writeText, hello, figlet, stdenvNoCC }:
+{ lib, testers, pkgs, writeText, hello, figlet, stdenvNoCC }:
 
 # -------------------------------------------------------------------------- #
 #
@@ -22,7 +22,7 @@ let
       lib.attrValues (import file { inherit pkgs; })
     );
 in
-nixosTest {
+testers.nixosTest {
   name = "nixpkgs-trivial-builders";
   nodes.machine = { ... }: {
     virtualisation.writableStore = true;

pkgs/test/default.nix

@@ -67,6 +67,8 @@ with pkgs;
 
   writers = callPackage ../build-support/writers/test.nix {};
 
+  testers = callPackage ../build-support/testers/test/default.nix {};
+
   dhall = callPackage ./dhall { };
 
   makeWrapper = callPackage ./make-wrapper {};

pkgs/test/nixos-functions/default.nix

@@ -1,7 +1,7 @@
 /*
 
 This file is a test that makes sure that the `pkgs.nixos` and
-`pkgs.nixosTest` functions work. It's far from a perfect test suite,
+`pkgs.testers.nixosTest` functions work. It's far from a perfect test suite,
 but better than not checking them at all on hydra.
 
 To run this test:
@@ -27,17 +27,5 @@ in lib.optionalAttrs stdenv.hostPlatform.isLinux (
       system.stateVersion = lib.trivial.release;
     }).toplevel;
 
-    nixosTest-test = pkgs.nixosTest ({ lib, pkgs, figlet, ... }: {
-      name = "nixosTest-test";
-      nodes.machine = { pkgs, ... }: {
-        system.nixos = dummyVersioning;
-        environment.systemPackages = [ pkgs.hello figlet ];
-        system.stateVersion = lib.trivial.release;
-      };
-      testScript = ''
-        machine.succeed("hello | figlet >/dev/console")
-      '';
-    });
-
   }
 )

pkgs/top-level/aliases.nix

@@ -876,6 +876,7 @@ mapAliases ({
   nix_2_5 = nixVersions.nix_2_5;
   nix_2_6 = nixVersions.nix_2_6;
   nixopsUnstable = nixops_unstable; # Added 2022-03-03
+  nixosTest = testers.nixosTest; # Added 2022-05-05
   nmap-unfree = nmap; # Added 2021-04-06
   nmap-graphical = throw "nmap graphical support has been removed due to its python2 dependency"; # Added 2022-04-26
   nmap_graphical = throw "nmap graphical support has been removed due to its python2 dependency"; # Modified 2022-04-26
@@ -1283,6 +1284,7 @@ mapAliases ({
   terraform_1_0_0 = throw "terraform_1_0_0 has been renamed to terraform_1"; # Added 2021-06-15
   tesseract_4 = throw "'tesseract_4' has been renamed to/replaced by 'tesseract4'"; # Converted to throw 2022-02-22
   testVersion = testers.testVersion; # Added 2022-04-20
+  invalidateFetcherByDrvHash = testers.invalidateFetcherByDrvHash; # Added 2022-05-05
   tex-gyre-bonum-math = throw "'tex-gyre-bonum-math' has been renamed to/replaced by 'tex-gyre-math.bonum'"; # Converted to throw 2022-02-22
   tex-gyre-pagella-math = throw "'tex-gyre-pagella-math' has been renamed to/replaced by 'tex-gyre-math.pagella'"; # Converted to throw 2022-02-22
   tex-gyre-schola-math = throw "'tex-gyre-schola-math' has been renamed to/replaced by 'tex-gyre-math.schola'"; # Converted to throw 2022-02-22

pkgs/top-level/all-packages.nix

@@ -750,23 +750,6 @@ with pkgs;
 
   installShellFiles = callPackage ../build-support/install-shell-files {};
 
-  # See doc/builders/testers.chapter.md or
-  # https://nixos.org/manual/nixpkgs/unstable/#tester-invalidateFetcherByDrvHash
-  invalidateFetcherByDrvHash = f: args:
-    let
-      drvPath = (f args).drvPath;
-      # It's safe to discard the context, because we don't access the path.
-      salt = builtins.unsafeDiscardStringContext (lib.substring 0 12 (baseNameOf drvPath));
-      # New derivation incorporating the original drv hash in the name
-      salted = f (args // { name = "${args.name or "source"}-salted-${salt}"; });
-      # Make sure we did change the derivation. If the fetcher ignores `name`,
-      # `invalidateFetcherByDrvHash` doesn't work.
-      checked =
-        if salted.drvPath == drvPath
-        then throw "invalidateFetcherByDrvHash: Adding the derivation hash to the fixed-output derivation name had no effect. Make sure the fetcher's name argument ends up in the derivation name. Otherwise, the fetcher will not be re-run when its implementation changes. This is important for testing."
-        else salted;
-    in checked;
-
   lazydocker = callPackage ../tools/misc/lazydocker { };
 
   ld-is-cc-hook = makeSetupHook { name = "ld-is-cc-hook"; }
@@ -34173,58 +34156,6 @@ with pkgs;
       };
   };
 
-  /*
-   * Run a NixOS VM network test using this evaluation of Nixpkgs.
-   *
-   * It is mostly equivalent to `import ./make-test-python.nix` from the
-   * NixOS manual[1], except that your `pkgs` will be used instead of
-   * letting NixOS invoke Nixpkgs again. If a test machine needs to
-   * set NixOS options under `nixpkgs`, it must set only the
-   * `nixpkgs.pkgs` option. For the details, see the Nixpkgs
-   * `pkgs.nixos` documentation.
-   *
-   * Parameter:
-   *   A NixOS VM test network, or path to it. Example:
-   *
-   *      { lib, ... }:
-   *      { name = "my-test";
-   *        nodes = {
-   *          machine-1 = someNixOSConfiguration;
-   *          machine-2 = ...;
-   *        }
-   *      }
-   *
-   * Result:
-   *   A derivation that runs the VM test.
-   *
-   * [1]: For writing NixOS tests, see
-   *      https://nixos.org/nixos/manual/index.html#sec-nixos-tests
-   */
-  nixosTest =
-    let
-      /* The nixos/lib/testing-python.nix module, preapplied with arguments that
-       * make sense for this evaluation of Nixpkgs.
-       */
-      nixosTesting =
-        (import ../../nixos/lib/testing-python.nix {
-          inherit (stdenv.hostPlatform) system;
-          inherit pkgs;
-          extraConfigurations = [(
-            { lib, ... }: {
-              config.nixpkgs.pkgs = lib.mkDefault pkgs;
-            }
-          )];
-        });
-    in
-      test:
-        let
-          loadedTest = if builtins.typeOf test == "path"
-            then import test
-            else test;
-          calledTest = lib.toFunction loadedTest pkgs;
-        in
-          nixosTesting.makeTest calledTest;
-
   nixosOptionsDoc = attrs:
     (import ../../nixos/lib/make-options-doc)
     ({ inherit pkgs lib; } // attrs);