Merge pull request #19 from oxalica/feature/profiles

Refactor component resolution and aggregation

README.md

@@ -22,7 +22,7 @@ You can put the code below into your `~/.config/nixpkgs/overlays.nix`.
 ```
 Then the provided attribute paths are available in nix command.
 ```bash
-$ nix-env -iA rust-bin.stable.latest.rust # Do anything you like.
+$ nix-env -iA rust-bin.stable.latest.default # Do anything you like.
 ```
 
 Alternatively, you can install it into nix channels.
@@ -68,7 +68,7 @@ Here's an example of using it in nixos configuration.
           ./configuration.nix # Your system configuration.
           ({ pkgs, ... }: {
             nixpkgs.overlays = [ rust-overlay.overlay ];
-            environment.systemPackages = [ pkgs.rust-bin.stable.latest.rust ];
+            environment.systemPackages = [ pkgs.rust-bin.stable.latest.default ];
           })
         ];
       };
@@ -87,7 +87,7 @@ Here's an example of using it in nixos configuration.
     distRoot = "https://static.rust-lang.org/dist";
 
     # Select a toolchain and aggregate components by rustup's `rust-toolchain` file format.
-    # See: https://github.com/ebroto/rustup/blob/c2db7dac6b38c99538eec472db9d23d18f918409/README.md#the-toolchain-file
+    # See: https://rust-lang.github.io/rustup/overrides.html#the-toolchain-file
     fromRustupToolchain = { channel, components ? [], targets ? [] }: «derivation»;
     # Same as `fromRustupToolchain` but read from a `rust-toolchain` file (legacy one-line string or in TOML).
     fromRustupToolchainFile = rust-toolchain-file-path: «derivation»;
@@ -101,8 +101,18 @@ Here's an example of using it in nixos configuration.
     stable = {
       # The latest stable toolchain.
       latest = {
-        # Aggregate all default components. (recommended)
+        # [Experimental]
+        # Profiles, predefined component sets.
+        # See: https://rust-lang.github.io/rustup/concepts/profiles.html
+        minimal = «derivation»;  # Only `cargo`, `rustc` and `rust-std`.
+        default = «derivation»;  # The default profile of `rustup`. Good for general use.
+        complete = «derivation»; # Do not use it. It almost always fails.
+
+        # Pre-aggregated package provided by upstream, the most commonly used package in `mozilla-overlay`.
+        # It consists of an uncertain number of components, usually more than the `default` profile of `rustup`
+        # but less than `complete` profile.
         rust = «derivation»;
+
         # Individial components.
         rustc = «derivation»;
         cargo = «derivation»;
@@ -144,31 +154,38 @@ Here's an example of using it in nixos configuration.
 
 Some examples (assume `nixpkgs` had the overlay applied):
 
-- Latest stable/beta/nightly rust with all default components:
+- Latest stable/beta/nightly rust with almost all components (provided the same as `mozilla-overlay`):
   `nixpkgs.rust-bin.{stable,beta,nightly}.latest.rust`
+- *\[Experimental\]*
+  Latest stable/beta/nightly rust with `default` or `minimal` profile (provided the same as default behavior of `rustup install`).
+  `nixpkgs.rust-bin.{stable,beta,nightly}.latest.{default,minimal}`
+
+  Note: `default` profile on `nightly` may not always be available due to absense of required components.
+  You can check availability on [Rustup packages availibility](https://rust-lang.github.io/rustup-components-history/).
+
 - A specific version of stable rust:
-  `nixpkgs.rust-bin.stable."1.48.0".rust`
+  `nixpkgs.rust-bin.stable."1.48.0".default`
 - A specific date of beta rust:
-  `nixpkgs.rust-bin.nightly."2021-01-01".rust`
-- A specific version of stable rust:
-  `nixpkgs.rust-bin.stable."1.48.0".rust`
+  `nixpkgs.rust-bin.beta."2021-01-01".default`
 - A specific date of nightly rust:
-  `nixpkgs.rust-bin.nightly."2020-12-31".rust`
+  `nixpkgs.rust-bin.nightly."2020-12-31".default`
 - Latest stable rust with additional component `rust-src` and extra target
   `arm-unknown-linux-gnueabihf`:
 
   ```nix
-  nixpkgs.rust-bin.stable.latest.rust.override {
+  nixpkgs.rust-bin.stable.latest.default.override {
     extensions = [ "rust-src" ];
     targets = [ "arm-unknown-linux-gnueabihf" ];
   }
   ```
-- If you already have a [`rust-toolchain` file for rustup](https://github.com/ebroto/rustup/blob/c2db7dac6b38c99538eec472db9d23d18f918409/README.md#the-toolchain-file),
+
+- If you already have a [`rust-toolchain` file for rustup][rust-toolchain],
   you can simply use `fromRustupToolchainFile` to get the customized toolchain derivation.
 
   ```nix
   nixpkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain
   ```
+
 - *\[Experimental\]*
   Toolchain with specific rustc git revision.
   This is useful for development of rust components like [MIRI](https://github.com/rust-lang/miri).
@@ -182,8 +199,10 @@ Some examples (assume `nixpkgs` had the overlay applied):
     };
   }
   ```
+
 - See more examples in directory `examples`.
 
 For more details, see also the source code of `./rust-overlay.nix`.
 
 [mozilla]: https://github.com/mozilla/nixpkgs-mozilla
+[rust-toolchain]: https://rust-lang.github.io/rustup/overrides.html#the-toolchain-file

examples/cross-aarch64/.cargo/config

@@ -1,3 +1,6 @@
+[build]
+target-dir = "target"
+
 [target.aarch64-unknown-linux-gnu]
 linker = "aarch64-unknown-linux-gnu-gcc"
 runner = "qemu-aarch64"

examples/cross-aarch64/.gitignore

@@ -0,0 +1 @@
+/target

examples/cross-aarch64/shell.nix

@@ -6,7 +6,7 @@ mkShell {
   nativeBuildInputs = [
     # Manual `buildPackages` is required here. See: https://github.com/NixOS/nixpkgs/issues/49526
     # build = host = x86_64, target = aarch64
-    buildPackages.rust-bin.stable.latest.rust
+    buildPackages.rust-bin.stable.latest.minimal
     buildPackages.pkg-config
     # build = host = target = x86_64, just to avoid re-build.
     pkgsBuildBuild.qemu

flake.nix

@@ -41,27 +41,41 @@
     };
     defaultPackage = packages.rust;
 
-    # FIXME: We can only directly provide derivations here without nested set.
-    # Currently we only provide stable releases. Some nightly versions have components missing
-    # on some platforms, which makes `nix flake check` to be failed.
-    packages =
-      lib.mapAttrs' (version: comps: {
-        name = "rust-${lib.replaceStrings ["."] ["-"] version}";
-        value = comps.rust;
-      }) pkgs.rust-bin.stable //
-      lib.mapAttrs' (version: comps: {
-        name = "rust-nightly-${version}";
-        value = comps.rust;
-      }) (removeAttrs pkgs.rust-bin.nightly [ "2018-11-01" "2020-09-12" ]) // # FIXME: `rust` is not available.
-      lib.mapAttrs' (version: comps: {
-        name = "rust-beta-${version}";
-        value = comps.rust;
-      }) (removeAttrs pkgs.rust-bin.beta [ "2018-11-09" ]) // # FIXME: `rust` is not available.
-      {
-        rust = packages.rust-latest;
-        rust-nightly = packages.rust-nightly-latest;
-        rust-beta = packages.rust-beta-latest;
-      };
+    packages = let
+      inherit (builtins) tryEval;
+
+      defaultPkg = comps:
+        if comps ? default then
+          if (tryEval comps.default.drvPath).success then
+            comps.default
+          else if (tryEval comps.minimal.drvPath).success then
+            comps.minimal
+          else
+            null
+        else if (tryEval comps.rust.drvPath).success then
+          comps.rust
+        else
+          null;
+
+      result =
+        lib.mapAttrs' (version: comps: {
+          name = "rust-${lib.replaceStrings ["."] ["-"] version}";
+          value = defaultPkg comps;
+        }) pkgs.rust-bin.stable //
+        lib.mapAttrs' (version: comps: {
+          name = "rust-nightly-${version}";
+          value = defaultPkg comps;
+        }) pkgs.rust-bin.nightly //
+        lib.mapAttrs' (version: comps: {
+          name = "rust-beta-${version}";
+          value = defaultPkg comps;
+        }) pkgs.rust-bin.beta //
+        {
+          rust = result.rust-latest;
+          rust-nightly = result.rust-nightly-latest;
+          rust-beta = result.rust-beta-latest;
+        };
+    in lib.filterAttrs (name: drv: drv != null) result;
 
     checks = let
       inherit (pkgs) rust-bin rustChannelOf;
@@ -74,7 +88,7 @@
         message = "`${lhs}` != `${rhs}`";
       };
       assertUrl = drv: url: let
-        srcUrl = lib.head (lib.head drv.paths).src.urls;
+        srcUrl = lib.head drv.src.urls;
       in assertEq srcUrl url;
 
       assertions = {
@@ -86,49 +100,68 @@
         url-kind-beta = assertUrl beta."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-beta-${rustTarget}.tar.xz";
         url-fix = assertUrl nightly."2019-01-10".rustc "https://static.rust-lang.org/dist/2019-01-10/rustc-nightly-${rustTarget}.tar.xz";
 
+      # Check only tier 1 targets.
+      } // lib.optionalAttrs (lib.elem system [ "aarch64-linux" "x86_64-linux" ]) {
+
+        name-stable = assertEq stable."1.48.0".rustc.name "rustc-1.48.0";
+        name-beta = assertEq beta."2021-01-01".rustc.name "rustc-1.50.0-beta.2-2021-01-01";
+        name-nightly = assertEq nightly."2021-01-01".rustc.name "rustc-1.51.0-nightly-2021-01-01";
+        name-stable-profile-default = assertEq stable."1.51.0".default.name "rust-default-1.51.0";
+        name-stable-profile-minimal = assertEq stable."1.51.0".minimal.name "rust-minimal-1.51.0";
+
         rename-available = assertEq stable."1.48.0".rustfmt stable."1.48.0".rustfmt-preview;
         rename-unavailable = {
           assertion = !(stable."1.30.0" ? rustfmt);
           message = "1.30.0 has rustfmt still in preview state";
         };
 
-        latest-stable = assertEq pkgs.latest.rustChannels.stable.rust stable.latest.rust;
-        latest-beta = assertEq pkgs.latest.rustChannels.beta.rust beta.latest.rust;
-        latest-nightly = assertEq pkgs.latest.rustChannels.nightly.rust nightly.latest.rust;
-
-        rust-channel-of-stable = assertEq (rustChannelOf { channel = "stable"; }).rust stable.latest.rust;
-        rust-channel-of-beta = assertEq (rustChannelOf { channel = "beta"; }).rust beta.latest.rust;
-        rust-channel-of-nightly = assertEq (rustChannelOf { channel = "nightly"; }).rust nightly.latest.rust;
-        rust-channel-of-version = assertEq (rustChannelOf { channel = "1.48.0"; }).rust stable."1.48.0".rust;
-        rust-channel-of-nightly-date = assertEq (rustChannelOf { channel = "nightly"; date = "2021-01-01"; }).rust nightly."2021-01-01".rust;
-        rust-channel-of-beta-date = assertEq (rustChannelOf { channel = "beta"; date = "2021-01-01"; }).rust beta."2021-01-01".rust;
-
-        rustup-toolchain-stable = assertEq (fromRustupToolchain { channel = "stable"; }) stable.latest.rust;
-        rustup-toolchain-beta = assertEq (fromRustupToolchain { channel = "beta"; }) beta.latest.rust;
-        rustup-toolchain-nightly = assertEq (fromRustupToolchain { channel = "nightly"; }) nightly.latest.rust;
-        rustup-toolchain-version = assertEq (fromRustupToolchain { channel = "1.48.0"; }) stable."1.48.0".rust;
-        rustup-toolchain-nightly-date = assertEq (fromRustupToolchain { channel = "nightly-2021-01-01"; }) nightly."2021-01-01".rust;
-        rustup-toolchain-beta-date = assertEq (fromRustupToolchain { channel = "beta-2021-01-01"; }) beta."2021-01-01".rust;
+        latest-stable-legacy = assertEq pkgs.latest.rustChannels.stable.rustc stable.latest.rustc;
+        latest-beta-legacy = assertEq pkgs.latest.rustChannels.beta.rustc beta.latest.rustc;
+        latest-nightly-legacy = assertEq pkgs.latest.rustChannels.nightly.rustc nightly.latest.rustc;
+
+        rust-channel-of-stable = assertEq (rustChannelOf { channel = "stable"; }).rustc stable.latest.rustc;
+        rust-channel-of-beta = assertEq (rustChannelOf { channel = "beta"; }).rustc beta.latest.rustc;
+        rust-channel-of-nightly = assertEq (rustChannelOf { channel = "nightly"; }).rustc nightly.latest.rustc;
+        rust-channel-of-version = assertEq (rustChannelOf { channel = "1.48.0"; }).rustc stable."1.48.0".rustc;
+        rust-channel-of-nightly-date = assertEq (rustChannelOf { channel = "nightly"; date = "2021-01-01"; }).rustc nightly."2021-01-01".rustc;
+        rust-channel-of-beta-date = assertEq (rustChannelOf { channel = "beta"; date = "2021-01-01"; }).rustc beta."2021-01-01".rustc;
+
+        rustup-toolchain-stable = assertEq (fromRustupToolchain { channel = "stable"; }) stable.latest.default;
+        rustup-toolchain-beta = assertEq (fromRustupToolchain { channel = "beta"; }) beta.latest.default;
+        # rustup-toolchain-nightly = assertEq (fromRustupToolchain { channel = "nightly"; }) nightly.latest.default; # Not always available
+        rustup-toolchain-version = assertEq (fromRustupToolchain { channel = "1.51.0"; }) stable."1.51.0".default;
+        rustup-toolchain-nightly-date = assertEq (fromRustupToolchain { channel = "nightly-2021-01-01"; }) nightly."2021-01-01".default;
+        rustup-toolchain-beta-date = assertEq (fromRustupToolchain { channel = "beta-2021-01-01"; }) beta."2021-01-01".default;
         rustup-toolchain-customization = assertEq
           (fromRustupToolchain {
-            channel = "1.48.0";
+            channel = "1.51.0";
             components = [ "rustfmt" "rustc-dev" ];
             targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ];
           })
-          (stable."1.48.0".rust.override {
+          (stable."1.51.0".default.override {
             extensions = [ "rustfmt" "rustc-dev" ];
             targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ];
           });
 
+        rustup-toolchain-profile-missing = assertEq (builtins.tryEval (fromRustupToolchain { channel = "1.51.0"; profile = "non_existent"; })).success false;
+        rustup-toolchain-profile-too-early = assertEq (builtins.tryEval (fromRustupToolchain { channel = "1.29.0"; profile = "minimal"; })).success false;
+        rustup-toolchain-profile-fallback = assertEq (fromRustupToolchain { channel = "1.29.0"; }) stable."1.29.0".rust;
+
         rustup-toolchain-file-toml = assertEq
           (fromRustupToolchainFile ./tests/rust-toolchain-toml)
-          (nightly."2020-07-10".rust.override {
+          (nightly."2021-03-25".default.override {
             extensions = [ "rustfmt" "rustc-dev" ];
             targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ];
           });
         rustup-toolchain-file-legacy = assertEq
           (fromRustupToolchainFile ./tests/rust-toolchain-legacy)
-          nightly."2020-07-10".rust;
+          nightly."2021-03-25".default;
+        rustup-toolchain-file-minimal = assertEq
+          (fromRustupToolchainFile ./tests/rust-toolchain-minimal)
+          (nightly."2021-03-25".minimal.override {
+            extensions = [ "rustfmt" "rustc-dev" ];
+            targets = [ "aarch64-unknown-linux-gnu" ];
+          });
       };
 
       checkDrvs = {};

manifest.nix

@@ -4,6 +4,7 @@ with builtins;
 let
   targets = import ./manifests/targets.nix // { _ = "*"; };
   renamesList = import ./manifests/renames.nix;
+  profilesList = import ./manifests/profiles.nix;
 
   inherit (final.rust-bin) distRoot;
 
@@ -43,15 +44,23 @@ let
   # Uncompress the compressed manifest to the original one
   # (not complete but has enough information to make up the toolchain).
   uncompressManifest = channel: version: {
-    v, # rustc version
-    d, # date
-    r, # rename index
+    v,        # Rustc version
+    d,        # Date
+    r,        # Renames index
+    p ? null, # Profiles index
     ...
   }@manifest: rec {
+
+    # Version used for derivation.
+    version = if builtins.match ".*(nightly|beta).*" v != null
+      then "${v}-${d}"  # 1.51.0-nightly-2021-01-01, 1.52.0-beta.2-2021-03-27
+      else v;           # 1.51.0
+
     date = d;
     renames = mapAttrs (from: to: { inherit to; }) (elemAt renamesList r);
+
     pkg =
-      mapAttrs (pkgName: { u ? null /* url version */, ... }@hashes: {
+      mapAttrs (pkgName: { u ? null /* Version appears in URL */, ... }@hashes: {
         # We use rustc version for all components to reduce manifest size.
         # This version is just used for component derivation name.
         version = "${v} (000000000 ${d})"; # "<version> (<commit-hash> yyyy-mm-dd)"
@@ -61,7 +70,7 @@ let
             pkgNameStripped = removeSuffix "-preview" pkgName;
             targetTail = if targetIdx == "_" then "" else "-" + target;
             urlVersion =
-              if u != null then u                     # Use specified url version if exists.
+              if u != null then u                     # Use specified version for URL if exists.
               else if channel == "stable" then v      # For stable channel, default to be rustc version.
               else channel;                           # Otherwise, for beta/nightly channel, default to be "beta"/"nightly".
           in {
@@ -71,7 +80,18 @@ let
               xz_hash = hash;
             } // (if pkgName == "rust" then rustPkgExtra pkg target else {});
           }) (removeAttrs hashes ["u"]);
-      }) (removeAttrs manifest ["v" "d" "r"]);
+      }) (removeAttrs manifest ["v" "d" "r" "p"]);
+
+    profiles = if p == null
+      then {}
+      # `rust-mingw` is in each profile but doesn't support platforms other than Windows.
+      else mapAttrs (name: remove "rust-mingw") (elemAt profilesList p);
+
+    targetComponentsList = [
+      "rust-std"
+      "rustc-dev"
+      "rustc-docs"
+    ];
   };
 
   uncompressManifestSet = channel: set: let