diff --git a/doc/builders/testers.chapter.md b/doc/builders/testers.chapter.md index 2c30c8bd240..c6fb71de018 100644 --- a/doc/builders/testers.chapter.md +++ b/doc/builders/testers.chapter.md @@ -80,3 +80,49 @@ tests.fetchgit = invalidateFetcherByDrvHash fetchgit { sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; ``` + +## `nixosTest` {#tester-nixosTest} + +Run a NixOS VM network test using this evaluation of Nixpkgs. + +NOTE: This function is primarily for external use. NixOS itself uses `make-test-python.nix` directly. Packages defined in Nixpkgs [reuse NixOS tests via `nixosTests`, plural](#ssec-nixos-tests-linking). + +It is mostly equivalent to the function `import ./make-test-python.nix` from the +[NixOS manual](https://nixos.org/nixos/manual/index.html#sec-nixos-tests), +except that the current application of Nixpkgs (`pkgs`) will be used, instead of +letting NixOS invoke Nixpkgs anew. + +If a test machine needs to set NixOS options under `nixpkgs`, it must set only the +`nixpkgs.pkgs` option. + +### Parameter + +A [NixOS VM test network](https://nixos.org/nixos/manual/index.html#sec-nixos-tests), or path to it. Example: + +```nix +{ + name = "my-test"; + nodes = { + machine1 = { lib, pkgs, nodes, ... }: { + environment.systemPackages = [ pkgs.hello ]; + services.foo.enable = true; + }; + # machine2 = ...; + }; + testScript = '' + start_all() + machine1.wait_for_unit("foo.service") + machine1.succeed("hello | foo-send") + ''; +} +``` + +### Result + +A derivation that runs the VM test. + +Notable attributes: + + * `nodes`: the evaluated NixOS configurations. Useful for debugging and exploring the configuration. + + * `driverInteractive`: a script that launches an interactive Python session in the context of the `testScript`. diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index b61a0268dee..d9ebbe74d54 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -569,8 +569,9 @@ The NixOS VM test framework, - pkgs.nixosTest/make-test-python.nix, - now requires detaching commands such as + pkgs.nixosTest/make-test-python.nix + (pkgs.testers.nixosTest since 22.05), now + requires detaching commands such as succeed("foo &") and succeed("foo | xclip -i") to close stdout. This can be done with a redirect such as diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 310d32cfdd7..e673d6721a3 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -166,7 +166,7 @@ In addition to numerous new and upgraded packages, this release has the followin ## Backward Incompatibilities {#sec-release-21.11-incompatibilities} -- The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix`, now requires detaching commands such as `succeed("foo &")` and `succeed("foo | xclip -i")` to close stdout. +- The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix` (`pkgs.testers.nixosTest` since 22.05), now requires detaching commands such as `succeed("foo &")` and `succeed("foo | xclip -i")` to close stdout. This can be done with a redirect such as `succeed("foo >&2 &")`. This breaking change was necessitated by a race condition causing tests to fail or hang. It applies to all methods that invoke commands on the nodes, including `execute`, `succeed`, `fail`, `wait_until_succeeds`, `wait_until_fails`. diff --git a/nixos/lib/build-vms.nix b/nixos/lib/build-vms.nix index 05d9ce89dbd..18af49db177 100644 --- a/nixos/lib/build-vms.nix +++ b/nixos/lib/build-vms.nix @@ -38,7 +38,7 @@ rec { { key = "no-revision"; # Make the revision metadata constant, in order to avoid needless retesting. # The human version (e.g. 21.05-pre) is left as is, because it is useful - # for external modules that test with e.g. nixosTest and rely on that + # for external modules that test with e.g. testers.nixosTest and rely on that # version number. config.system.nixos.revision = mkForce "constant-nixos-revision"; } diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index cd2bb2f9d4d..4ab3cf53045 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -119,6 +119,7 @@ rec { passthru = passthru // { inherit nodes; }; + meta.mainProgram = "nixos-test-driver"; } '' mkdir -p $out/bin diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 169edb171db..9b9a21a1469 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -97,7 +97,7 @@ rec { }; # Same example, but re-fetches every time the fetcher implementation changes. # NOTE: Only use this for testing, or you'd be wasting a lot of time, network and space. - testNixFromDockerHub = pkgs.invalidateFetcherByDrvHash pullImage { + testNixFromDockerHub = pkgs.testers.invalidateFetcherByDrvHash pullImage { imageName = "nixos/nix"; imageDigest = "sha256:85299d86263a3059cf19f419f9d286cc9f06d3c13146a8ebbb21b3437f598357"; sha256 = "19fw0n3wmddahzr20mhdqv6jkjn1kanh6n2mrr08ai53dr8ph5n7"; diff --git a/pkgs/build-support/fetchfirefoxaddon/tests.nix b/pkgs/build-support/fetchfirefoxaddon/tests.nix index c407d0e74b8..fd70d0f82ac 100644 --- a/pkgs/build-support/fetchfirefoxaddon/tests.nix +++ b/pkgs/build-support/fetchfirefoxaddon/tests.nix @@ -1,7 +1,7 @@ -{ invalidateFetcherByDrvHash, fetchFirefoxAddon, fetchurl, ... }: +{ testers, fetchFirefoxAddon, fetchurl, ... }: { - simple = invalidateFetcherByDrvHash fetchFirefoxAddon { + simple = testers.invalidateFetcherByDrvHash fetchFirefoxAddon { name = "image-search-options"; # Chosen because its only 147KB url = "https://addons.mozilla.org/firefox/downloads/file/3059971/image_search_options-3.0.12-fx.xpi"; @@ -14,7 +14,7 @@ sha256 = "sha256-H73YWX/DKxvhEwKpWOo7orAQ7c/rQywpljeyxYxv0Gg="; }; in - invalidateFetcherByDrvHash fetchFirefoxAddon { + testers.invalidateFetcherByDrvHash fetchFirefoxAddon { name = "image-search-options"; src = image-search-options; }; diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index c558fb6efa4..b9ab66d9353 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -1,14 +1,14 @@ -{ invalidateFetcherByDrvHash, fetchgit, ... }: +{ testers, fetchgit, ... }: { - simple = invalidateFetcherByDrvHash fetchgit { + simple = testers.invalidateFetcherByDrvHash fetchgit { name = "nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; - sparseCheckout = invalidateFetcherByDrvHash fetchgit { + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "nix-source"; url = "https://github.com/NixOS/nix"; rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; diff --git a/pkgs/build-support/fetchpatch/tests.nix b/pkgs/build-support/fetchpatch/tests.nix index ff2b81bf3a1..a42b7cd7d14 100644 --- a/pkgs/build-support/fetchpatch/tests.nix +++ b/pkgs/build-support/fetchpatch/tests.nix @@ -1,18 +1,18 @@ -{ invalidateFetcherByDrvHash, fetchpatch, ... }: +{ testers, fetchpatch, ... }: { - simple = invalidateFetcherByDrvHash fetchpatch { + simple = testers.invalidateFetcherByDrvHash fetchpatch { url = "https://github.com/facebook/zstd/pull/2724/commits/e1f85dbca3a0ed5ef06c8396912a0914db8dea6a.patch"; sha256 = "sha256-PuYAqnJWAE+L9bsroOnnBGJhERW8LHrGSLtIEkKU9vg="; }; - relative = invalidateFetcherByDrvHash fetchpatch { + relative = testers.invalidateFetcherByDrvHash fetchpatch { url = "https://github.com/boostorg/math/commit/7d482f6ebc356e6ec455ccb5f51a23971bf6ce5b.patch"; relative = "include"; sha256 = "sha256-KlmIbixcds6GyKYt1fx5BxDIrU7msrgDdYo9Va/KJR4="; }; - full = invalidateFetcherByDrvHash fetchpatch { + full = testers.invalidateFetcherByDrvHash fetchpatch { url = "https://github.com/boostorg/math/commit/7d482f6ebc356e6ec455ccb5f51a23971bf6ce5b.patch"; relative = "test"; stripLen = 1; diff --git a/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix b/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix index a781dad8307..19451466f24 100644 --- a/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix +++ b/pkgs/build-support/node/fetch-yarn-deps/tests/default.nix @@ -1,15 +1,15 @@ -{ invalidateFetcherByDrvHash, fetchYarnDeps, ... }: +{ testers, fetchYarnDeps, ... }: { - simple = invalidateFetcherByDrvHash fetchYarnDeps { + simple = testers.invalidateFetcherByDrvHash fetchYarnDeps { yarnLock = ./simple.lock; sha256 = "sha256-Erdkw2E8wWT09jFNLXGkrdwKl0HuSZWnUDJUrV95vSE="; }; - gitDep = invalidateFetcherByDrvHash fetchYarnDeps { + gitDep = testers.invalidateFetcherByDrvHash fetchYarnDeps { yarnLock = ./git.lock; sha256 = "sha256-lAqN4LpoE+jgsQO1nDtuORwcVEO7ogEV53jCu2jFJUI="; }; - githubDep = invalidateFetcherByDrvHash fetchYarnDeps { + githubDep = testers.invalidateFetcherByDrvHash fetchYarnDeps { yarnLock = ./github.lock; sha256 = "sha256-Tsfgyjxz8x6gNmfN0xR7G/NQNoEs4svxRN/N+26vfJU="; }; diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 8b79843b833..3ab97760e72 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -1,4 +1,4 @@ -{ pkgs, lib, callPackage, runCommand }: +{ pkgs, lib, callPackage, runCommand, stdenv }: # Documentation is in doc/builders/testers.chapter.md { testEqualDerivation = callPackage ./test-equal-derivation.nix { }; @@ -15,4 +15,49 @@ echo "$output" >&2 && exit 1 fi ''; + + # See doc/builders/testers.chapter.md or + # https://nixos.org/manual/nixpkgs/unstable/#tester-invalidateFetcherByDrvHash + invalidateFetcherByDrvHash = f: args: + let + drvPath = (f args).drvPath; + # It's safe to discard the context, because we don't access the path. + salt = builtins.unsafeDiscardStringContext (lib.substring 0 12 (baseNameOf drvPath)); + # New derivation incorporating the original drv hash in the name + salted = f (args // { name = "${args.name or "source"}-salted-${salt}"; }); + # Make sure we did change the derivation. If the fetcher ignores `name`, + # `invalidateFetcherByDrvHash` doesn't work. + checked = + if salted.drvPath == drvPath + then throw "invalidateFetcherByDrvHash: Adding the derivation hash to the fixed-output derivation name had no effect. Make sure the fetcher's name argument ends up in the derivation name. Otherwise, the fetcher will not be re-run when its implementation changes. This is important for testing." + else salted; + in checked; + + # See doc/builders/testers.chapter.md or + # https://nixos.org/manual/nixpkgs/unstable/#tester-invalidateFetcherByDrvHash + nixosTest = + let + /* The nixos/lib/testing-python.nix module, preapplied with arguments that + * make sense for this evaluation of Nixpkgs. + */ + nixosTesting = + (import ../../../nixos/lib/testing-python.nix { + inherit (stdenv.hostPlatform) system; + inherit pkgs; + extraConfigurations = [( + { lib, ... }: { + config.nixpkgs.pkgs = lib.mkDefault pkgs; + } + )]; + }); + in + test: + let + loadedTest = if builtins.typeOf test == "path" + then import test + else test; + calledTest = lib.toFunction loadedTest pkgs; + in + nixosTesting.makeTest calledTest; + } diff --git a/pkgs/build-support/testers/test/README.md b/pkgs/build-support/testers/test/README.md new file mode 100644 index 00000000000..2d6b4bdc43f --- /dev/null +++ b/pkgs/build-support/testers/test/README.md @@ -0,0 +1,8 @@ +# Tests _for the testers_ + + cd nixpkgs + nix-build -A tests.testers + +Tests generally derive their own correctness from simplicity, which in the +case of testers (themselves functions) does not always work out. +Hence the need for tests that test the testers. diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix new file mode 100644 index 00000000000..30e778cf652 --- /dev/null +++ b/pkgs/build-support/testers/test/default.nix @@ -0,0 +1,27 @@ +{ testers, lib, pkgs, ... }: +let + pkgs-with-overlay = pkgs.extend(final: prev: { + proof-of-overlay-hello = prev.hello; + }); + + dummyVersioning = { + revision = "test"; + versionSuffix = "test"; + label = "test"; + }; + +in +lib.recurseIntoAttrs { + # Check that the wiring of nixosTest is correct. + # Correct operation of the NixOS test driver should be asserted elsewhere. + nixosTest-example = pkgs-with-overlay.testers.nixosTest ({ lib, pkgs, figlet, ... }: { + name = "nixosTest-test"; + nodes.machine = { pkgs, ... }: { + system.nixos = dummyVersioning; + environment.systemPackages = [ pkgs.proof-of-overlay-hello figlet ]; + }; + testScript = '' + machine.succeed("hello | figlet >/dev/console") + ''; + }); +} diff --git a/pkgs/build-support/trivial-builders/test/references.nix b/pkgs/build-support/trivial-builders/test/references.nix index 989722121cb..3e1eb16eecd 100644 --- a/pkgs/build-support/trivial-builders/test/references.nix +++ b/pkgs/build-support/trivial-builders/test/references.nix @@ -1,4 +1,4 @@ -{ lib, nixosTest, pkgs, writeText, hello, figlet, stdenvNoCC }: +{ lib, testers, pkgs, writeText, hello, figlet, stdenvNoCC }: # -------------------------------------------------------------------------- # # @@ -22,7 +22,7 @@ let lib.attrValues (import file { inherit pkgs; }) ); in -nixosTest { +testers.nixosTest { name = "nixpkgs-trivial-builders"; nodes.machine = { ... }: { virtualisation.writableStore = true; diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 63aaf6bb72e..2ab03bda8b7 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -67,6 +67,8 @@ with pkgs; writers = callPackage ../build-support/writers/test.nix {}; + testers = callPackage ../build-support/testers/test/default.nix {}; + dhall = callPackage ./dhall { }; makeWrapper = callPackage ./make-wrapper {}; diff --git a/pkgs/test/nixos-functions/default.nix b/pkgs/test/nixos-functions/default.nix index 5e9857134de..bdd5e3c6d8b 100644 --- a/pkgs/test/nixos-functions/default.nix +++ b/pkgs/test/nixos-functions/default.nix @@ -1,7 +1,7 @@ /* This file is a test that makes sure that the `pkgs.nixos` and -`pkgs.nixosTest` functions work. It's far from a perfect test suite, +`pkgs.testers.nixosTest` functions work. It's far from a perfect test suite, but better than not checking them at all on hydra. To run this test: @@ -27,17 +27,5 @@ in lib.optionalAttrs stdenv.hostPlatform.isLinux ( system.stateVersion = lib.trivial.release; }).toplevel; - nixosTest-test = pkgs.nixosTest ({ lib, pkgs, figlet, ... }: { - name = "nixosTest-test"; - nodes.machine = { pkgs, ... }: { - system.nixos = dummyVersioning; - environment.systemPackages = [ pkgs.hello figlet ]; - system.stateVersion = lib.trivial.release; - }; - testScript = '' - machine.succeed("hello | figlet >/dev/console") - ''; - }); - } ) diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 9a693bffe6c..ac387f67ef8 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -876,6 +876,7 @@ mapAliases ({ nix_2_5 = nixVersions.nix_2_5; nix_2_6 = nixVersions.nix_2_6; nixopsUnstable = nixops_unstable; # Added 2022-03-03 + nixosTest = testers.nixosTest; # Added 2022-05-05 nmap-unfree = nmap; # Added 2021-04-06 nmap-graphical = throw "nmap graphical support has been removed due to its python2 dependency"; # Added 2022-04-26 nmap_graphical = throw "nmap graphical support has been removed due to its python2 dependency"; # Modified 2022-04-26 @@ -1283,6 +1284,7 @@ mapAliases ({ terraform_1_0_0 = throw "terraform_1_0_0 has been renamed to terraform_1"; # Added 2021-06-15 tesseract_4 = throw "'tesseract_4' has been renamed to/replaced by 'tesseract4'"; # Converted to throw 2022-02-22 testVersion = testers.testVersion; # Added 2022-04-20 + invalidateFetcherByDrvHash = testers.invalidateFetcherByDrvHash; # Added 2022-05-05 tex-gyre-bonum-math = throw "'tex-gyre-bonum-math' has been renamed to/replaced by 'tex-gyre-math.bonum'"; # Converted to throw 2022-02-22 tex-gyre-pagella-math = throw "'tex-gyre-pagella-math' has been renamed to/replaced by 'tex-gyre-math.pagella'"; # Converted to throw 2022-02-22 tex-gyre-schola-math = throw "'tex-gyre-schola-math' has been renamed to/replaced by 'tex-gyre-math.schola'"; # Converted to throw 2022-02-22 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ce04e52db14..126c6d4ba39 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -750,23 +750,6 @@ with pkgs; installShellFiles = callPackage ../build-support/install-shell-files {}; - # See doc/builders/testers.chapter.md or - # https://nixos.org/manual/nixpkgs/unstable/#tester-invalidateFetcherByDrvHash - invalidateFetcherByDrvHash = f: args: - let - drvPath = (f args).drvPath; - # It's safe to discard the context, because we don't access the path. - salt = builtins.unsafeDiscardStringContext (lib.substring 0 12 (baseNameOf drvPath)); - # New derivation incorporating the original drv hash in the name - salted = f (args // { name = "${args.name or "source"}-salted-${salt}"; }); - # Make sure we did change the derivation. If the fetcher ignores `name`, - # `invalidateFetcherByDrvHash` doesn't work. - checked = - if salted.drvPath == drvPath - then throw "invalidateFetcherByDrvHash: Adding the derivation hash to the fixed-output derivation name had no effect. Make sure the fetcher's name argument ends up in the derivation name. Otherwise, the fetcher will not be re-run when its implementation changes. This is important for testing." - else salted; - in checked; - lazydocker = callPackage ../tools/misc/lazydocker { }; ld-is-cc-hook = makeSetupHook { name = "ld-is-cc-hook"; } @@ -34173,58 +34156,6 @@ with pkgs; }; }; - /* - * Run a NixOS VM network test using this evaluation of Nixpkgs. - * - * It is mostly equivalent to `import ./make-test-python.nix` from the - * NixOS manual[1], except that your `pkgs` will be used instead of - * letting NixOS invoke Nixpkgs again. If a test machine needs to - * set NixOS options under `nixpkgs`, it must set only the - * `nixpkgs.pkgs` option. For the details, see the Nixpkgs - * `pkgs.nixos` documentation. - * - * Parameter: - * A NixOS VM test network, or path to it. Example: - * - * { lib, ... }: - * { name = "my-test"; - * nodes = { - * machine-1 = someNixOSConfiguration; - * machine-2 = ...; - * } - * } - * - * Result: - * A derivation that runs the VM test. - * - * [1]: For writing NixOS tests, see - * https://nixos.org/nixos/manual/index.html#sec-nixos-tests - */ - nixosTest = - let - /* The nixos/lib/testing-python.nix module, preapplied with arguments that - * make sense for this evaluation of Nixpkgs. - */ - nixosTesting = - (import ../../nixos/lib/testing-python.nix { - inherit (stdenv.hostPlatform) system; - inherit pkgs; - extraConfigurations = [( - { lib, ... }: { - config.nixpkgs.pkgs = lib.mkDefault pkgs; - } - )]; - }); - in - test: - let - loadedTest = if builtins.typeOf test == "path" - then import test - else test; - calledTest = lib.toFunction loadedTest pkgs; - in - nixosTesting.makeTest calledTest; - nixosOptionsDoc = attrs: (import ../../nixos/lib/make-options-doc) ({ inherit pkgs lib; } // attrs);