From a4e708522768a30cd5120a65c979c87abd525336 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 20 Apr 2021 13:46:53 +0200 Subject: [PATCH 01/13] stdenv.mkDerivation: Allow overriding of recursive definitions See updated manual for further explanation. --- doc/stdenv/meta.chapter.md | 30 +++++++++++ doc/stdenv/stdenv.chapter.md | 52 +++++++++++++++++++ pkgs/stdenv/generic/make-derivation.nix | 67 +++++++++++++++++++++++-- 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/doc/stdenv/meta.chapter.md b/doc/stdenv/meta.chapter.md index d3e1dd5b27d..5f3e12ba004 100644 --- a/doc/stdenv/meta.chapter.md +++ b/doc/stdenv/meta.chapter.md @@ -175,6 +175,36 @@ The NixOS tests are available as `nixosTests` in parameters of derivations. For NixOS tests run in a VM, so they are slower than regular package tests. For more information see [NixOS module tests](https://nixos.org/manual/nixos/stable/#sec-nixos-tests). +Alternatively, you can specify other derivations as tests. You can make use of +the optional parameter (here: `self`) to inject the correct package without +relying on non-local definitions, even in the presence of `overrideAttrs`. This +definition of `tests` does not rely on the original `mypkg` or overrides it in +all places. + +```nix +# my-package/default.nix +{ stdenv, callPackage }: +stdenv.mkDerivation (self: { + # ... + passthru.tests.example = callPackage ./example.nix { my-package = self; }; +}) +``` + +```nix +# my-package/example.nix +{ runCommand, lib, my-package, ... }: +runCommand "my-package-test" { + nativeBuildInputs = [ my-package ]; + src = lib.sources.sourcesByRegex ./. [ ".*.in" ".*.expected" ]; +} '' + my-package --help + my-package example.actual + diff -U3 --color=auto example.expected example.actual + mkdir $out +'' +``` + + ### `timeout` {#var-meta-timeout} A timeout (in seconds) for building the derivation. If the derivation takes longer than this time to build, it can fail due to breaking the timeout. However, all computers do not have the same computing power, hence some builders may decide to apply a multiplicative factor to this value. When filling this value in, try to keep it approximately consistent with other values already present in `nixpkgs`. diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index 40f295b178b..2994f7020b1 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -317,6 +317,58 @@ The script will be usually run from the root of the Nixpkgs repository but you s For information about how to run the updates, execute `nix-shell maintainers/scripts/update.nix`. +### Recursive attributes in `mkDerivation` + +If you pass a function to `mkDerivation`, it will receive as its argument the final output of the same `mkDerivation` call. For example: + +```nix +mkDerivation (self: { + pname = "hello"; + withFeature = true; + configureFlags = + lib.optionals self.withFeature ["--with-feature"]; +}) +``` + +Note that this does not use the `rec` keyword to reuse `withFeature` in `configureFlags`. +Instead, the definition references `self`, allowing users to change `withFeature` +consistently with `overrideAttrs`. + +Let's look at a more elaborate example to understand the differences between +various bindings: + +```nix +# `pkg` is the _original_ definition (for illustration purposes) +let pkg = + mkDerivation (self: { # self is the final package + # ... + + # An example attribute + packages = []; + + # `passthru.tests` is a commonly defined attribute. + passthru.tests.simple = f self; + + # An example of an attribute containing a function + passthru.appendPackages = packages': + self.overrideAttrs (newSelf: super: { + packages = super.packages ++ packages'; + }); + + # For illustration purposes; referenced as + # `(pkg.overrideAttrs(x)).self` etc in the text below. + passthru.self = self; + passthru.original = pkg; + }); +in pkg +``` + +Unlike the `pkg` binding in the above example, the `self` parameter always references the final package. For instance `(pkg.overrideAttrs(x)).self` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as `pkg`. + +This is also different from `mkDerivation rec { ..... }`, which binds the recursive references immediately, so it allows you to reference original _inputs_ only. + +See also the section about [`passthru.tests`](#var-meta-tests). + ## Phases {#sec-stdenv-phases} `stdenv.mkDerivation` sets the Nix [derivation](https://nixos.org/manual/nix/stable/expressions/derivations.html#derivations)'s builder to a script that loads the stdenv `setup.sh` bash library and calls `genericBuild`. Most packaging functions rely on this default builder. diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index d1b93874a25..2a47ea99383 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -9,8 +9,68 @@ let # to build it. This is a bit confusing for cross compilation. inherit (stdenv) hostPlatform; }; + + makeOverlayable = mkDerivationSimple: # TODO(@robert): turn mkDerivationSimple into let binding. + fnOrAttrs: + if builtins.isFunction fnOrAttrs + then makeDerivationExtensible mkDerivationSimple fnOrAttrs + else makeDerivationExtensibleConst mkDerivationSimple fnOrAttrs; + + # Based off lib.makeExtensible, with modifications: + # - lib.fix' -> lib.fix ∘ mkDerivationSimple; then inline fix + # - convert `f` to an overlay + # - inline overrideAttrs and make it positional instead of // to reduce allocs + makeDerivationExtensible = mkDerivationSimple: rattrs: + let + r = mkDerivationSimple + (f0: + let + f = self: super: + # Convert f0 to an overlay. Legacy is: + # overrideAttrs (super: {}) + # We want to introduce self. We follow the convention of overlays: + # overrideAttrs (self: super: {}) + # Which means the first parameter can be either self or super. + # This is surprising, but far better than the confusion that would + # arise from flipping an overlay's parameters in some cases. + let x = f0 super; + in + if builtins.isFunction x + then + # Can't reuse `x`, because `self` comes first. + # Looks inefficient, but `f0 super` was a cheap thunk. + f0 self super + else x; + in + makeDerivationExtensible mkDerivationSimple + (self: let super = rattrs self; in super // f self super)) + (rattrs r); + in r; + + # makeDerivationExtensibleConst == makeDerivationExtensible (_: attrs), + # but pre-evaluated for a slight improvement in performance. + makeDerivationExtensibleConst = mkDerivationSimple: attrs: + mkDerivationSimple (f0: + let + f = self: super: + let x = f0 super; + in + if builtins.isFunction x + then + # Can't reuse `x`, because `self` comes first. + # Looks inefficient, but `f0 super` was a cheap thunk. + f0 self super + else x; + in + makeDerivationExtensible mkDerivationSimple (self: attrs // f self attrs)) + attrs; + in +# TODO(@roberth): inline makeOverlayable; reindenting whole rest of this file. +makeOverlayable (overrideAttrs: + + # `mkDerivation` wraps the builtin `derivation` function to # produce derivations that use this stdenv and its shell. # @@ -70,6 +130,7 @@ in , # TODO(@Ericson2314): Make always true and remove strictDeps ? if config.strictDepsByDefault then true else stdenv.hostPlatform != stdenv.buildPlatform + , meta ? {} , passthru ? {} , pos ? # position used in error messages and for meta.position @@ -381,8 +442,6 @@ in lib.extendDerivation validity.handled ({ - overrideAttrs = f: stdenv.mkDerivation (attrs // (f attrs)); - # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations @@ -408,10 +467,12 @@ lib.extendDerivation args = [ "-c" "export > $out" ]; }); - inherit meta passthru; + inherit meta passthru overrideAttrs; } // # Pass through extra attributes that are not inputs, but # should be made available to Nix expressions using the # derivation (e.g., in assertions). passthru) (derivation derivationArg) + +) From 2afc03a084cf736cc76f629a45a720c3c44eb27d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 20 Apr 2021 13:46:53 +0200 Subject: [PATCH 02/13] hello: Define a passthru test via new mkDerivation self arg --- pkgs/applications/misc/hello/default.nix | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 60482a84c9b..8d36b344a22 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -1,4 +1,5 @@ { lib +, runCommand , stdenv , fetchurl , nixos @@ -6,12 +7,12 @@ , hello }: -stdenv.mkDerivation rec { +stdenv.mkDerivation (self: { pname = "hello"; version = "2.12"; src = fetchurl { - url = "mirror://gnu/hello/${pname}-${version}.tar.gz"; + url = "mirror://gnu/hello/${self.pname}-${self.version}.tar.gz"; sha256 = "1ayhp9v4m4rdhjmnl2bq3cibrbqqkgjbl3s7yk2nhlh8vj3ay16g"; }; @@ -27,6 +28,13 @@ stdenv.mkDerivation rec { (nixos { environment.noXlibs = true; }).pkgs.hello; }; + passthru.tests.run = runCommand "hello-test-run" { + nativeBuildInputs = [ self ]; + } '' + diff -U3 --color=auto <(hello) <(echo 'Hello, world!') + touch $out + ''; + meta = with lib; { description = "A program that produces a familiar, friendly greeting"; longDescription = '' @@ -34,9 +42,9 @@ stdenv.mkDerivation rec { It is fully customizable. ''; homepage = "https://www.gnu.org/software/hello/manual/"; - changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${version}"; + changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${self.version}"; license = licenses.gpl3Plus; maintainers = [ maintainers.eelco ]; platforms = platforms.all; }; -} +}) From 40ab3b8738b4bb10ae849712c6bdc0fdf7564e3e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 26 Apr 2021 14:47:18 +0200 Subject: [PATCH 03/13] hello: Use callPackage for test --- pkgs/applications/misc/hello/default.nix | 11 +++-------- pkgs/applications/misc/hello/test.nix | 8 ++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 pkgs/applications/misc/hello/test.nix diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 8d36b344a22..0ea6570b320 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -1,5 +1,5 @@ -{ lib -, runCommand +{ callPackage +, lib , stdenv , fetchurl , nixos @@ -28,12 +28,7 @@ stdenv.mkDerivation (self: { (nixos { environment.noXlibs = true; }).pkgs.hello; }; - passthru.tests.run = runCommand "hello-test-run" { - nativeBuildInputs = [ self ]; - } '' - diff -U3 --color=auto <(hello) <(echo 'Hello, world!') - touch $out - ''; + passthru.tests.run = callPackage ./test.nix { hello = self; }; meta = with lib; { description = "A program that produces a familiar, friendly greeting"; diff --git a/pkgs/applications/misc/hello/test.nix b/pkgs/applications/misc/hello/test.nix new file mode 100644 index 00000000000..7acded2a16f --- /dev/null +++ b/pkgs/applications/misc/hello/test.nix @@ -0,0 +1,8 @@ +{ runCommand, hello }: + +runCommand "hello-test-run" { + nativeBuildInputs = [ hello ]; +} '' + diff -U3 --color=auto <(hello) <(echo 'Hello, world!') + touch $out +'' From 2f21bc2fdb4349ca89f3c8db9742cdaa15702a52 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 May 2021 16:41:31 +0200 Subject: [PATCH 04/13] doc/stdenv/meta: tests -> passthru.tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the text understandable without knowing that these are identical. Co-authored-by: Daniël de Kok --- doc/stdenv/meta.chapter.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/stdenv/meta.chapter.md b/doc/stdenv/meta.chapter.md index 5f3e12ba004..64924dceb71 100644 --- a/doc/stdenv/meta.chapter.md +++ b/doc/stdenv/meta.chapter.md @@ -178,6 +178,7 @@ NixOS tests run in a VM, so they are slower than regular package tests. For more Alternatively, you can specify other derivations as tests. You can make use of the optional parameter (here: `self`) to inject the correct package without relying on non-local definitions, even in the presence of `overrideAttrs`. This +means `(mypkg.overrideAttrs f).passthru.tests` will be as expected, as long as the definition of `tests` does not rely on the original `mypkg` or overrides it in all places. From 6d7efb3a16ddbc58d0c4688cbe8213b337388a0c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 29 Jul 2021 13:09:27 +0200 Subject: [PATCH 05/13] stdenv.mkDerivation: Make self more overlay-like; use self.public `self` is now arguments, like `super`. The final package is in `self.public`. --- doc/stdenv/meta.chapter.md | 2 +- doc/stdenv/stdenv.chapter.md | 15 +++-- pkgs/applications/misc/hello/default.nix | 2 +- pkgs/stdenv/generic/make-derivation.nix | 78 ++++++++++++------------ 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/doc/stdenv/meta.chapter.md b/doc/stdenv/meta.chapter.md index 64924dceb71..e57669e261b 100644 --- a/doc/stdenv/meta.chapter.md +++ b/doc/stdenv/meta.chapter.md @@ -187,7 +187,7 @@ all places. { stdenv, callPackage }: stdenv.mkDerivation (self: { # ... - passthru.tests.example = callPackage ./example.nix { my-package = self; }; + passthru.tests.example = callPackage ./example.nix { my-package = self.public; } }) ``` diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index 2994f7020b1..b54d68cb5dd 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -319,7 +319,7 @@ For information about how to run the updates, execute `nix-shell maintainers/scr ### Recursive attributes in `mkDerivation` -If you pass a function to `mkDerivation`, it will receive as its argument the final output of the same `mkDerivation` call. For example: +If you pass a function to `mkDerivation`, it will receive as its argument the final arguments, considering use of `overrideAttrs`. For example: ```nix mkDerivation (self: { @@ -331,9 +331,14 @@ mkDerivation (self: { ``` Note that this does not use the `rec` keyword to reuse `withFeature` in `configureFlags`. +The `rec` keyword works at the syntax level and is unaware of overriding. + Instead, the definition references `self`, allowing users to change `withFeature` consistently with `overrideAttrs`. +`self` also contains the attribute `public`, which represents the final package, +including the output paths, etc. + Let's look at a more elaborate example to understand the differences between various bindings: @@ -347,11 +352,11 @@ let pkg = packages = []; # `passthru.tests` is a commonly defined attribute. - passthru.tests.simple = f self; + passthru.tests.simple = f self.public; # An example of an attribute containing a function passthru.appendPackages = packages': - self.overrideAttrs (newSelf: super: { + self.public.overrideAttrs (newSelf: super: { packages = super.packages ++ packages'; }); @@ -363,9 +368,7 @@ let pkg = in pkg ``` -Unlike the `pkg` binding in the above example, the `self` parameter always references the final package. For instance `(pkg.overrideAttrs(x)).self` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as `pkg`. - -This is also different from `mkDerivation rec { ..... }`, which binds the recursive references immediately, so it allows you to reference original _inputs_ only. +Unlike the `pkg` binding in the above example, the `self` parameter always references the final attributes. For instance `(pkg.overrideAttrs(x)).self.public` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as `pkg`. See also the section about [`passthru.tests`](#var-meta-tests). diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 0ea6570b320..75887348730 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -28,7 +28,7 @@ stdenv.mkDerivation (self: { (nixos { environment.noXlibs = true; }).pkgs.hello; }; - passthru.tests.run = callPackage ./test.nix { hello = self; }; + passthru.tests.run = callPackage ./test.nix { hello = self.public; }; meta = with lib; { description = "A program that produces a familiar, friendly greeting"; diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 2a47ea99383..ac661d9429f 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -17,52 +17,50 @@ let else makeDerivationExtensibleConst mkDerivationSimple fnOrAttrs; # Based off lib.makeExtensible, with modifications: - # - lib.fix' -> lib.fix ∘ mkDerivationSimple; then inline fix - # - convert `f` to an overlay - # - inline overrideAttrs and make it positional instead of // to reduce allocs makeDerivationExtensible = mkDerivationSimple: rattrs: let - r = mkDerivationSimple - (f0: - let - f = self: super: - # Convert f0 to an overlay. Legacy is: - # overrideAttrs (super: {}) - # We want to introduce self. We follow the convention of overlays: - # overrideAttrs (self: super: {}) - # Which means the first parameter can be either self or super. - # This is surprising, but far better than the confusion that would - # arise from flipping an overlay's parameters in some cases. - let x = f0 super; - in - if builtins.isFunction x - then - # Can't reuse `x`, because `self` comes first. - # Looks inefficient, but `f0 super` was a cheap thunk. - f0 self super - else x; - in - makeDerivationExtensible mkDerivationSimple - (self: let super = rattrs self; in super // f self super)) - (rattrs r); - in r; + args = rattrs (args // { inherit public; }); + public = + mkDerivationSimple + (f0: + let + f = self: super: + # Convert f0 to an overlay. Legacy is: + # overrideAttrs (super: {}) + # We want to introduce self. We follow the convention of overlays: + # overrideAttrs (self: super: {}) + # Which means the first parameter can be either self or super. + # This is surprising, but far better than the confusion that would + # arise from flipping an overlay's parameters in some cases. + let x = f0 super; + in + if builtins.isFunction x + then + # Can't reuse `x`, because `self` comes first. + # Looks inefficient, but `f0 super` was a cheap thunk. + f0 self super + else x; + in + makeDerivationExtensible mkDerivationSimple + (self: let super = rattrs self; in super // f self super)) + args; + in public; # makeDerivationExtensibleConst == makeDerivationExtensible (_: attrs), # but pre-evaluated for a slight improvement in performance. makeDerivationExtensibleConst = mkDerivationSimple: attrs: - mkDerivationSimple (f0: - let - f = self: super: - let x = f0 super; - in - if builtins.isFunction x - then - # Can't reuse `x`, because `self` comes first. - # Looks inefficient, but `f0 super` was a cheap thunk. - f0 self super - else x; - in - makeDerivationExtensible mkDerivationSimple (self: attrs // f self attrs)) + mkDerivationSimple + (f0: + let + f = self: super: + let x = f0 super; + in + if builtins.isFunction x + then + f0 self super + else x; + in + makeDerivationExtensible mkDerivationSimple (self: attrs // f self attrs)) attrs; in From 41b3688ba1222b61d19e6f810c9a1867eef69141 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 23 Dec 2021 19:46:32 +0100 Subject: [PATCH 06/13] make-derivation.nix: Remove unnecessary TODO The goal was to keep the number of calls to a minimum, but that would come at the cost of clobbering git blame and possibly not improving readability. --- pkgs/stdenv/generic/make-derivation.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index ac661d9429f..afb6eed632a 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -10,7 +10,7 @@ let inherit (stdenv) hostPlatform; }; - makeOverlayable = mkDerivationSimple: # TODO(@robert): turn mkDerivationSimple into let binding. + makeOverlayable = mkDerivationSimple: fnOrAttrs: if builtins.isFunction fnOrAttrs then makeDerivationExtensible mkDerivationSimple fnOrAttrs @@ -65,7 +65,6 @@ let in -# TODO(@roberth): inline makeOverlayable; reindenting whole rest of this file. makeOverlayable (overrideAttrs: From d629ba27d963664282253e6bf32d0f1a38af796a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 23 Dec 2021 20:54:57 +0100 Subject: [PATCH 07/13] Use finalAttrs instead of self for mkDerivation "overlay" --- doc/stdenv/meta.chapter.md | 13 ++++++++----- doc/stdenv/stdenv.chapter.md | 20 ++++++++++---------- pkgs/applications/misc/hello/default.nix | 8 ++++---- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/doc/stdenv/meta.chapter.md b/doc/stdenv/meta.chapter.md index e57669e261b..ca8dd1d824f 100644 --- a/doc/stdenv/meta.chapter.md +++ b/doc/stdenv/meta.chapter.md @@ -176,18 +176,21 @@ The NixOS tests are available as `nixosTests` in parameters of derivations. For NixOS tests run in a VM, so they are slower than regular package tests. For more information see [NixOS module tests](https://nixos.org/manual/nixos/stable/#sec-nixos-tests). Alternatively, you can specify other derivations as tests. You can make use of -the optional parameter (here: `self`) to inject the correct package without -relying on non-local definitions, even in the presence of `overrideAttrs`. This -means `(mypkg.overrideAttrs f).passthru.tests` will be as expected, as long as the +the optional parameter to inject the correct package without +relying on non-local definitions, even in the presence of `overrideAttrs`. +Here that's `finalAttrs.public`, but you could choose a different name if +`finalAttrs` already exists in your scope. + +`(mypkg.overrideAttrs f).passthru.tests` will be as expected, as long as the definition of `tests` does not rely on the original `mypkg` or overrides it in all places. ```nix # my-package/default.nix { stdenv, callPackage }: -stdenv.mkDerivation (self: { +stdenv.mkDerivation (finalAttrs: { # ... - passthru.tests.example = callPackage ./example.nix { my-package = self.public; } + passthru.tests.example = callPackage ./example.nix { my-package = finalAttrs.public; } }) ``` diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index b54d68cb5dd..26f242dbae4 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -322,21 +322,21 @@ For information about how to run the updates, execute `nix-shell maintainers/scr If you pass a function to `mkDerivation`, it will receive as its argument the final arguments, considering use of `overrideAttrs`. For example: ```nix -mkDerivation (self: { +mkDerivation (finalAttrs: { pname = "hello"; withFeature = true; configureFlags = - lib.optionals self.withFeature ["--with-feature"]; + lib.optionals finalAttrs.withFeature ["--with-feature"]; }) ``` Note that this does not use the `rec` keyword to reuse `withFeature` in `configureFlags`. The `rec` keyword works at the syntax level and is unaware of overriding. -Instead, the definition references `self`, allowing users to change `withFeature` +Instead, the definition references `finalAttrs`, allowing users to change `withFeature` consistently with `overrideAttrs`. -`self` also contains the attribute `public`, which represents the final package, +`finalAttrs` also contains the attribute `public`, which represents the final package, including the output paths, etc. Let's look at a more elaborate example to understand the differences between @@ -345,30 +345,30 @@ various bindings: ```nix # `pkg` is the _original_ definition (for illustration purposes) let pkg = - mkDerivation (self: { # self is the final package + mkDerivation (finalAttrs: { # ... # An example attribute packages = []; # `passthru.tests` is a commonly defined attribute. - passthru.tests.simple = f self.public; + passthru.tests.simple = f finalAttrs.public; # An example of an attribute containing a function passthru.appendPackages = packages': - self.public.overrideAttrs (newSelf: super: { + finalAttrs.public.overrideAttrs (newSelf: super: { packages = super.packages ++ packages'; }); # For illustration purposes; referenced as - # `(pkg.overrideAttrs(x)).self` etc in the text below. - passthru.self = self; + # `(pkg.overrideAttrs(x)).finalAttrs` etc in the text below. + passthru.finalAttrs = finalAttrs; passthru.original = pkg; }); in pkg ``` -Unlike the `pkg` binding in the above example, the `self` parameter always references the final attributes. For instance `(pkg.overrideAttrs(x)).self.public` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as `pkg`. +Unlike the `pkg` binding in the above example, the `finalAttrs` parameter always references the final attributes. For instance `(pkg.overrideAttrs(x)).finalAttrs.public` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as the original `pkg`. See also the section about [`passthru.tests`](#var-meta-tests). diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 75887348730..bd7bcd8a3b0 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -7,12 +7,12 @@ , hello }: -stdenv.mkDerivation (self: { +stdenv.mkDerivation (finalAttrs: { pname = "hello"; version = "2.12"; src = fetchurl { - url = "mirror://gnu/hello/${self.pname}-${self.version}.tar.gz"; + url = "mirror://gnu/hello/${finalAttrs.pname}-${finalAttrs.version}.tar.gz"; sha256 = "1ayhp9v4m4rdhjmnl2bq3cibrbqqkgjbl3s7yk2nhlh8vj3ay16g"; }; @@ -28,7 +28,7 @@ stdenv.mkDerivation (self: { (nixos { environment.noXlibs = true; }).pkgs.hello; }; - passthru.tests.run = callPackage ./test.nix { hello = self.public; }; + passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.public; }; meta = with lib; { description = "A program that produces a familiar, friendly greeting"; @@ -37,7 +37,7 @@ stdenv.mkDerivation (self: { It is fully customizable. ''; homepage = "https://www.gnu.org/software/hello/manual/"; - changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${self.version}"; + changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${finalAttrs.version}"; license = licenses.gpl3Plus; maintainers = [ maintainers.eelco ]; platforms = platforms.all; From 1bbb5a14c89427241ec2d7d6c55724332dd59803 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 23 Dec 2021 21:03:23 +0100 Subject: [PATCH 08/13] doc/using/overrides: Update for overlay style mkDerivation overrideAttrs --- doc/using/overrides.chapter.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/using/overrides.chapter.md b/doc/using/overrides.chapter.md index 66e5103531a..6e69423fe5f 100644 --- a/doc/using/overrides.chapter.md +++ b/doc/using/overrides.chapter.md @@ -39,14 +39,18 @@ The function `overrideAttrs` allows overriding the attribute set passed to a `st Example usage: ```nix -helloWithDebug = pkgs.hello.overrideAttrs (oldAttrs: rec { +helloWithDebug = pkgs.hello.overrideAttrs (finalAttrs: previousAttrs: { separateDebugInfo = true; }); ``` In the above example, the `separateDebugInfo` attribute is overridden to be true, thus building debug info for `helloWithDebug`, while all other attributes will be retained from the original `hello` package. -The argument `oldAttrs` is conventionally used to refer to the attr set originally passed to `stdenv.mkDerivation`. +The argument `previousAttrs` is conventionally used to refer to the attr set originally passed to `stdenv.mkDerivation`. + +The argument `finalAttrs` refers to the final attributes passed to `mkDerivation`, plus the `public` attribute which is the result of `mkDerivation` — the derivation or package. + +If only a one-argument function is written, the argument has the meaning of `previousAttrs`. ::: {.note} Note that `separateDebugInfo` is processed only by the `stdenv.mkDerivation` function, not the generated, raw Nix derivation. Thus, using `overrideDerivation` will not work in this case, as it overrides only the attributes of the final derivation. It is for this reason that `overrideAttrs` should be preferred in (almost) all cases to `overrideDerivation`, i.e. to allow using `stdenv.mkDerivation` to process input arguments, as well as the fact that it is easier to use (you can use the same attribute names you see in your Nix code, instead of the ones generated (e.g. `buildInputs` vs `nativeBuildInputs`), and it involves less typing). From 37ab5b43963f51f939b9c031a0dd824d82b41259 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 11 Jan 2022 09:07:58 +0100 Subject: [PATCH 09/13] mkDerivation: Add error hint for infinite recursion --- pkgs/stdenv/generic/make-derivation.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index afb6eed632a..3e3a7aa790d 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -19,7 +19,14 @@ let # Based off lib.makeExtensible, with modifications: makeDerivationExtensible = mkDerivationSimple: rattrs: let + # NOTE: The following is a hint that will be printed by the Nix cli when + # encountering an infinite recursion. It must not be formatted into + # separate lines, because Nix would only show the last line of the comment. + + # An infinite recursion here can be caused by having the attribute names of expression `e` in `.overrideAttrs(finalAttrs: previousAttrs: e)` depend on `finalAttrs`. Only the attribute values of `e` can depend on `finalAttrs`. args = rattrs (args // { inherit public; }); + # ^^^^ + public = mkDerivationSimple (f0: From 2e0bd527623626dac4fa9937f94682ecf761af3e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 7 Feb 2022 10:01:09 +0100 Subject: [PATCH 10/13] rl-2205: Add entry for overlay-style mkDerivation overriding --- .../from_md/release-notes/rl-2205.section.xml | 27 +++++++++++++++++++ .../manual/release-notes/rl-2205.section.md | 15 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 2046e2449cc..c91fb4d90f0 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -43,6 +43,33 @@ Shell. + + + stdenv.mkDerivation now supports a + self-referencing finalAttrs: parameter + containing the final mkDerivation arguments + including overrides. drv.overrideAttrs now + supports two parameters + finalAttrs: previousAttrs:. This allows + packaging configuration to be overridden in a consistent + manner by providing an alternative to + rec {} syntax. + + + Additionally, passthru can now reference + finalAttrs.public containing the final + package, including attributes such as the output paths and + overrideAttrs. + + + New language integrations can be simplified by overriding a + prototype package containing the + language-specific logic. This removes the need for a extra + layer of overriding for the generic builder + arguments, thus removing a usability problem and source of + error. + + PHP 8.1 is now available diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index 9674eb66a4c..6dec64695d2 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -17,6 +17,21 @@ In addition to numerous new and upgraded packages, this release has the followin - GNOME has been upgraded to 42. Please take a look at their [Release Notes](https://release.gnome.org/42/) for details. Notably, it replaces gedit with GNOME Text Editor, GNOME Terminal with GNOME Console (formerly King’s Cross), and GNOME Screenshot with a tool built into the Shell. +- `stdenv.mkDerivation` now supports a self-referencing `finalAttrs:` parameter + containing the final `mkDerivation` arguments including overrides. + `drv.overrideAttrs` now supports two parameters `finalAttrs: previousAttrs:`. + This allows packaging configuration to be overridden in a consistent manner by + providing an alternative to `rec {}` syntax. + + Additionally, `passthru` can now reference `finalAttrs.public` containing + the final package, including attributes such as the output paths and + `overrideAttrs`. + + New language integrations can be simplified by overriding a "prototype" + package containing the language-specific logic. This removes the need for a + extra layer of overriding for the "generic builder" arguments, thus removing a + usability problem and source of error. + - PHP 8.1 is now available - Mattermost has been updated to extended support release 6.3, as the previously packaged extended support release 5.37 is [reaching its end of life](https://docs.mattermost.com/upgrade/extended-support-release.html). From ca83dd1ae73a94bee5c67ed21b2f6be41b195f40 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 9 Feb 2022 12:10:37 +0100 Subject: [PATCH 11/13] stdenv.md: Clarify overrideAttrs sentence I weirded my English. --- doc/stdenv/stdenv.chapter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index 26f242dbae4..fb0bdd5e2d7 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -319,7 +319,7 @@ For information about how to run the updates, execute `nix-shell maintainers/scr ### Recursive attributes in `mkDerivation` -If you pass a function to `mkDerivation`, it will receive as its argument the final arguments, considering use of `overrideAttrs`. For example: +If you pass a function to `mkDerivation`, it will receive as its argument the final arguments, including the overrides when reinvoked via `overrideAttrs`. For example: ```nix mkDerivation (finalAttrs: { From f066dddaa5cb6aea4429cd9386d4aa6f3963a889 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 9 Feb 2022 12:35:35 +0100 Subject: [PATCH 12/13] hello: Make pname overridable without breaking src.url --- pkgs/applications/misc/hello/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index bd7bcd8a3b0..676d10d06ce 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -12,7 +12,7 @@ stdenv.mkDerivation (finalAttrs: { version = "2.12"; src = fetchurl { - url = "mirror://gnu/hello/${finalAttrs.pname}-${finalAttrs.version}.tar.gz"; + url = "mirror://gnu/hello/hello-${finalAttrs.version}.tar.gz"; sha256 = "1ayhp9v4m4rdhjmnl2bq3cibrbqqkgjbl3s7yk2nhlh8vj3ay16g"; }; From 0e00acafe9dc8bbd9e6b49d4341b6bf49b6defb8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 15 Mar 2022 12:22:16 +0100 Subject: [PATCH 13/13] stdenv.mkDerivation: public -> finalPackage --- doc/stdenv/meta.chapter.md | 4 ++-- doc/stdenv/stdenv.chapter.md | 9 ++++----- doc/using/overrides.chapter.md | 2 +- .../doc/manual/from_md/release-notes/rl-2205.section.xml | 6 +++--- nixos/doc/manual/release-notes/rl-2205.section.md | 2 +- pkgs/applications/misc/hello/default.nix | 2 +- pkgs/stdenv/generic/make-derivation.nix | 6 +++--- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/doc/stdenv/meta.chapter.md b/doc/stdenv/meta.chapter.md index ca8dd1d824f..c1bb3f8863f 100644 --- a/doc/stdenv/meta.chapter.md +++ b/doc/stdenv/meta.chapter.md @@ -178,7 +178,7 @@ NixOS tests run in a VM, so they are slower than regular package tests. For more Alternatively, you can specify other derivations as tests. You can make use of the optional parameter to inject the correct package without relying on non-local definitions, even in the presence of `overrideAttrs`. -Here that's `finalAttrs.public`, but you could choose a different name if +Here that's `finalAttrs.finalPackage`, but you could choose a different name if `finalAttrs` already exists in your scope. `(mypkg.overrideAttrs f).passthru.tests` will be as expected, as long as the @@ -190,7 +190,7 @@ all places. { stdenv, callPackage }: stdenv.mkDerivation (finalAttrs: { # ... - passthru.tests.example = callPackage ./example.nix { my-package = finalAttrs.public; } + passthru.tests.example = callPackage ./example.nix { my-package = finalAttrs.finalPackage; }; }) ``` diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index fb0bdd5e2d7..d5d27cbf086 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -336,8 +336,7 @@ The `rec` keyword works at the syntax level and is unaware of overriding. Instead, the definition references `finalAttrs`, allowing users to change `withFeature` consistently with `overrideAttrs`. -`finalAttrs` also contains the attribute `public`, which represents the final package, -including the output paths, etc. +`finalAttrs` also contains the attribute `finalPackage`, which includes the output paths, etc. Let's look at a more elaborate example to understand the differences between various bindings: @@ -352,11 +351,11 @@ let pkg = packages = []; # `passthru.tests` is a commonly defined attribute. - passthru.tests.simple = f finalAttrs.public; + passthru.tests.simple = f finalAttrs.finalPackage; # An example of an attribute containing a function passthru.appendPackages = packages': - finalAttrs.public.overrideAttrs (newSelf: super: { + finalAttrs.finalPackage.overrideAttrs (newSelf: super: { packages = super.packages ++ packages'; }); @@ -368,7 +367,7 @@ let pkg = in pkg ``` -Unlike the `pkg` binding in the above example, the `finalAttrs` parameter always references the final attributes. For instance `(pkg.overrideAttrs(x)).finalAttrs.public` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as the original `pkg`. +Unlike the `pkg` binding in the above example, the `finalAttrs` parameter always references the final attributes. For instance `(pkg.overrideAttrs(x)).finalAttrs.finalPackage` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as the original `pkg`. See also the section about [`passthru.tests`](#var-meta-tests). diff --git a/doc/using/overrides.chapter.md b/doc/using/overrides.chapter.md index 6e69423fe5f..a97a39354a9 100644 --- a/doc/using/overrides.chapter.md +++ b/doc/using/overrides.chapter.md @@ -48,7 +48,7 @@ In the above example, the `separateDebugInfo` attribute is overridden to be true The argument `previousAttrs` is conventionally used to refer to the attr set originally passed to `stdenv.mkDerivation`. -The argument `finalAttrs` refers to the final attributes passed to `mkDerivation`, plus the `public` attribute which is the result of `mkDerivation` — the derivation or package. +The argument `finalAttrs` refers to the final attributes passed to `mkDerivation`, plus the `finalPackage` attribute which is equal to the result of `mkDerivation` or subsequent `overrideAttrs` calls. If only a one-argument function is written, the argument has the meaning of `previousAttrs`. diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index c91fb4d90f0..3f0db0068d0 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -57,9 +57,9 @@ Additionally, passthru can now reference - finalAttrs.public containing the final - package, including attributes such as the output paths and - overrideAttrs. + finalAttrs.finalPackage containing the + final package, including attributes such as the output paths + and overrideAttrs. New language integrations can be simplified by overriding a diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index 6dec64695d2..1fba4fbbe2c 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -23,7 +23,7 @@ In addition to numerous new and upgraded packages, this release has the followin This allows packaging configuration to be overridden in a consistent manner by providing an alternative to `rec {}` syntax. - Additionally, `passthru` can now reference `finalAttrs.public` containing + Additionally, `passthru` can now reference `finalAttrs.finalPackage` containing the final package, including attributes such as the output paths and `overrideAttrs`. diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 676d10d06ce..c82de2ae027 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -28,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: { (nixos { environment.noXlibs = true; }).pkgs.hello; }; - passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.public; }; + passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.finalPackage; }; meta = with lib; { description = "A program that produces a familiar, friendly greeting"; diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 3e3a7aa790d..6d9f6a9e336 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -24,10 +24,10 @@ let # separate lines, because Nix would only show the last line of the comment. # An infinite recursion here can be caused by having the attribute names of expression `e` in `.overrideAttrs(finalAttrs: previousAttrs: e)` depend on `finalAttrs`. Only the attribute values of `e` can depend on `finalAttrs`. - args = rattrs (args // { inherit public; }); + args = rattrs (args // { inherit finalPackage; }); # ^^^^ - public = + finalPackage = mkDerivationSimple (f0: let @@ -51,7 +51,7 @@ let makeDerivationExtensible mkDerivationSimple (self: let super = rattrs self; in super // f self super)) args; - in public; + in finalPackage; # makeDerivationExtensibleConst == makeDerivationExtensible (_: attrs), # but pre-evaluated for a slight improvement in performance.