Merge commit 'ccc467eff80b2fbb8000cf425e999ef14fbe200c'

infra/libkookie/overlays/rust/.github/workflows/ci.yaml

@@ -88,10 +88,11 @@ jobs:
     name: Check build Rust
     if: ${{ github.event.workflow_run.conclusion != 'failure' }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         channel: [stable, beta]
-        build_pkg: [rust, default]
+        build_pkg: [minimal, default]
     runs-on: ${{ matrix.os }}
     env:
       build_expr: (import <nixpkgs> { overlays = [ (import ./.) ]; }).rust-bin.${{ matrix.channel }}.latest.${{ matrix.build_pkg }}
@@ -135,7 +136,7 @@ jobs:
         nix-shell -p "$build_expr" --pure --command "rustc test.rs -o test"
         [[ "$(./test)" == "hello, world" ]]
     - name: Check `clippy` works
-      if: steps.cache-drv-status.outputs.cache-hit != 'true'
+      if: matrix.build_pkg == 'default' && steps.cache-drv-status.outputs.cache-hit != 'true'
       run: |
         echo 'fn main() { loop { break; } }' >./test.rs
         nix-shell -p "$build_expr" --pure --command 'clippy-driver test.rs -W clippy::never-loop' 2>&1 | tee msg

infra/libkookie/overlays/rust/README.md

@@ -134,7 +134,7 @@ Running `nix develop` will create a shell with the default nightly Rust toolchai
   rust-bin.beta.latest.minimal
   ```
 
-  It provices the same components as which installed by `rustup install`'s `default` or `minimal` profiles.
+  It provides the same components as which installed by `rustup install`'s `default` or `minimal` profiles.
 
   Almost always, `default` is what you want for development.
 

infra/libkookie/overlays/rust/examples/cross-aarch64/shell.nix

@@ -1,18 +1,22 @@
-with import <nixpkgs> {
+(import <nixpkgs> {
   crossSystem = "aarch64-linux";
   overlays = [ (import ../..) ];
-};
+}).callPackage (
+{ mkShell, rust-bin, pkg-config, openssl, pkgsBuildBuild }:
 mkShell {
   nativeBuildInputs = [
     # Manual `buildPackages` is required here. See: https://github.com/NixOS/nixpkgs/issues/49526
     # build = host = x86_64, target = aarch64
-    buildPackages.rust-bin.stable.latest.minimal
+    rust-bin.stable.latest.minimal
-    buildPackages.pkg-config
+    pkg-config
-    # build = host = target = x86_64, just to avoid re-build.
+
+    # build = host = target = x86_64
+    # qemu itself is multi-platform and `target` doesn't matter for it.
+    # Use build system's to avoid rebuild.
     pkgsBuildBuild.qemu
   ];
   buildInputs = [
     # build = x86_64, host = target = aarch64
     openssl
   ];
-}
+}) {}

infra/libkookie/overlays/rust/manifest.nix

@@ -84,8 +84,7 @@ let
 
     profiles = if p == null
       then {}
-      # `rust-mingw` is in each profile but doesn't support platforms other than Windows.
+      else elemAt profilesList p;
-      else mapAttrs (name: remove "rust-mingw") (elemAt profilesList p);
 
     targetComponentsList = [
       "rust-std"

infra/libkookie/overlays/rust/manifests/beta/default.nix

@@ -173,5 +173,15 @@
   "2021-08-22" = import ./2021/2021-08-22.nix;
   "2021-08-28" = import ./2021/2021-08-28.nix;
   "2021-08-29" = import ./2021/2021-08-29.nix;
-  latest = "2021-08-29";
+  "2021-09-05" = import ./2021/2021-09-05.nix;
+  "2021-09-08" = import ./2021/2021-09-08.nix;
+  "2021-09-12" = import ./2021/2021-09-12.nix;
+  "2021-09-18" = import ./2021/2021-09-18.nix;
+  "2021-10-06" = import ./2021/2021-10-06.nix;
+  "2021-10-15" = import ./2021/2021-10-15.nix;
+  "2021-10-17" = import ./2021/2021-10-17.nix;
+  "2021-10-19" = import ./2021/2021-10-19.nix;
+  "2021-10-22" = import ./2021/2021-10-22.nix;
+  "2021-11-02" = import ./2021/2021-11-02.nix;
+  latest = "2021-11-02";
 }

infra/libkookie/overlays/rust/manifests/nightly/default.nix

@@ -1066,5 +1066,81 @@
   "2021-09-01" = import ./2021/2021-09-01.nix;
   "2021-09-02" = import ./2021/2021-09-02.nix;
   "2021-09-03" = import ./2021/2021-09-03.nix;
-  latest = "2021-09-03";
+  "2021-09-04" = import ./2021/2021-09-04.nix;
+  "2021-09-05" = import ./2021/2021-09-05.nix;
+  "2021-09-06" = import ./2021/2021-09-06.nix;
+  "2021-09-07" = import ./2021/2021-09-07.nix;
+  "2021-09-08" = import ./2021/2021-09-08.nix;
+  "2021-09-09" = import ./2021/2021-09-09.nix;
+  "2021-09-10" = import ./2021/2021-09-10.nix;
+  "2021-09-11" = import ./2021/2021-09-11.nix;
+  "2021-09-12" = import ./2021/2021-09-12.nix;
+  "2021-09-13" = import ./2021/2021-09-13.nix;
+  "2021-09-14" = import ./2021/2021-09-14.nix;
+  "2021-09-15" = import ./2021/2021-09-15.nix;
+  "2021-09-16" = import ./2021/2021-09-16.nix;
+  "2021-09-17" = import ./2021/2021-09-17.nix;
+  "2021-09-18" = import ./2021/2021-09-18.nix;
+  "2021-09-19" = import ./2021/2021-09-19.nix;
+  "2021-09-20" = import ./2021/2021-09-20.nix;
+  "2021-09-22" = import ./2021/2021-09-22.nix;
+  "2021-09-23" = import ./2021/2021-09-23.nix;
+  "2021-09-24" = import ./2021/2021-09-24.nix;
+  "2021-09-25" = import ./2021/2021-09-25.nix;
+  "2021-09-26" = import ./2021/2021-09-26.nix;
+  "2021-09-27" = import ./2021/2021-09-27.nix;
+  "2021-09-28" = import ./2021/2021-09-28.nix;
+  "2021-09-29" = import ./2021/2021-09-29.nix;
+  "2021-09-30" = import ./2021/2021-09-30.nix;
+  "2021-10-01" = import ./2021/2021-10-01.nix;
+  "2021-10-02" = import ./2021/2021-10-02.nix;
+  "2021-10-03" = import ./2021/2021-10-03.nix;
+  "2021-10-04" = import ./2021/2021-10-04.nix;
+  "2021-10-05" = import ./2021/2021-10-05.nix;
+  "2021-10-06" = import ./2021/2021-10-06.nix;
+  "2021-10-07" = import ./2021/2021-10-07.nix;
+  "2021-10-08" = import ./2021/2021-10-08.nix;
+  "2021-10-09" = import ./2021/2021-10-09.nix;
+  "2021-10-10" = import ./2021/2021-10-10.nix;
+  "2021-10-11" = import ./2021/2021-10-11.nix;
+  "2021-10-12" = import ./2021/2021-10-12.nix;
+  "2021-10-13" = import ./2021/2021-10-13.nix;
+  "2021-10-14" = import ./2021/2021-10-14.nix;
+  "2021-10-15" = import ./2021/2021-10-15.nix;
+  "2021-10-16" = import ./2021/2021-10-16.nix;
+  "2021-10-17" = import ./2021/2021-10-17.nix;
+  "2021-10-18" = import ./2021/2021-10-18.nix;
+  "2021-10-19" = import ./2021/2021-10-19.nix;
+  "2021-10-20" = import ./2021/2021-10-20.nix;
+  "2021-10-21" = import ./2021/2021-10-21.nix;
+  "2021-10-22" = import ./2021/2021-10-22.nix;
+  "2021-10-23" = import ./2021/2021-10-23.nix;
+  "2021-10-24" = import ./2021/2021-10-24.nix;
+  "2021-10-25" = import ./2021/2021-10-25.nix;
+  "2021-10-26" = import ./2021/2021-10-26.nix;
+  "2021-10-27" = import ./2021/2021-10-27.nix;
+  "2021-10-28" = import ./2021/2021-10-28.nix;
+  "2021-10-29" = import ./2021/2021-10-29.nix;
+  "2021-10-30" = import ./2021/2021-10-30.nix;
+  "2021-10-31" = import ./2021/2021-10-31.nix;
+  "2021-11-01" = import ./2021/2021-11-01.nix;
+  "2021-11-02" = import ./2021/2021-11-02.nix;
+  "2021-11-03" = import ./2021/2021-11-03.nix;
+  "2021-11-04" = import ./2021/2021-11-04.nix;
+  "2021-11-05" = import ./2021/2021-11-05.nix;
+  "2021-11-06" = import ./2021/2021-11-06.nix;
+  "2021-11-07" = import ./2021/2021-11-07.nix;
+  "2021-11-08" = import ./2021/2021-11-08.nix;
+  "2021-11-09" = import ./2021/2021-11-09.nix;
+  "2021-11-10" = import ./2021/2021-11-10.nix;
+  "2021-11-11" = import ./2021/2021-11-11.nix;
+  "2021-11-12" = import ./2021/2021-11-12.nix;
+  "2021-11-13" = import ./2021/2021-11-13.nix;
+  "2021-11-14" = import ./2021/2021-11-14.nix;
+  "2021-11-15" = import ./2021/2021-11-15.nix;
+  "2021-11-16" = import ./2021/2021-11-16.nix;
+  "2021-11-17" = import ./2021/2021-11-17.nix;
+  "2021-11-18" = import ./2021/2021-11-18.nix;
+  "2021-11-19" = import ./2021/2021-11-19.nix;
+  latest = "2021-11-19";
 }

infra/libkookie/overlays/rust/manifests/stable/default.nix

@@ -37,5 +37,8 @@
   "1.52.1" = import ./1.52.1.nix;
   "1.53.0" = import ./1.53.0.nix;
   "1.54.0" = import ./1.54.0.nix;
-  latest = "1.54.0";
+  "1.55.0" = import ./1.55.0.nix;
+  "1.56.0" = import ./1.56.0.nix;
+  "1.56.1" = import ./1.56.1.nix;
+  latest = "1.56.1";
 }

infra/libkookie/overlays/rust/rust-overlay.nix

@@ -9,16 +9,27 @@ let
   # Manifest selector.
   selectManifest = { channel, date ? null }: let
     inherit (self.rust-bin) manifests;
-    inherit (builtins) match elemAt;
+    inherit (builtins) match elemAt compareVersions attrNames foldl' filter;
+    inherit (self.lib) hasPrefix;
 
     assertWith = cond: msg: body: if cond then body else throw msg;
 
-    asVersion = match "[0-9]+\\.[0-9]+\\.[0-9]+" channel;
+    # https://rust-lang.github.io/rustup/concepts/toolchains.html#toolchain-specification
+    # <channel> = stable|beta|nightly|<major.minor>|<major.minor.patch>
+
+    asVersion = match "[0-9]+\\.[0-9]+(\\.[0-9]+)?" channel;
     asNightlyDate = let m = match "nightly-([0-9]+-[0-9]+-[0-9]+)" channel; in
       if m == null then null else elemAt m 0;
     asBetaDate = let m = match "beta-([0-9]+-[0-9]+-[0-9]+)" channel; in
       if m == null then null else elemAt m 0;
 
+    maxWith = zero: f: foldl' (lhs: rhs: if lhs == zero || f lhs rhs < 0 then rhs else lhs) zero;
+
+    latestStableWithMajorMinor =
+      maxWith "" compareVersions
+        (filter (hasPrefix (channel + "."))
+          (attrNames manifests.stable));
+
   in
     # "stable"
     if channel == "stable" then
@@ -30,10 +41,15 @@ let
     # "beta"
     else if channel == "beta" then
       manifests.beta.${if date != null then date else "latest"} or (throw "Beta ${date} is not available")
-    # "1.49.0"
+    # "1.49.0" or "1.49"
     else if asVersion != null then
-      assertWith (date == null) "Stable version with specific date is not supported"
+      assertWith (date == null) "Stable version with specific date is not supported" (
-        manifests.stable.${channel} or (throw "Stable ${channel} is not available")
+        # "1.49"
+        if asVersion == [ null ] then
+          manifests.stable.${latestStableWithMajorMinor} or (throw "No stable ${channel}.* is available")
+        # "1.49.0"
+        else
+          manifests.stable.${channel} or (throw "Stable ${channel} is not available"))
     # "beta-2021-01-01"
     else if asBetaDate != null then
       assertWith (date == null) "Cannot specify date in both `channel` and `date`"
@@ -185,6 +201,14 @@ let
 
       nativeBuildInputs = [ self.gnutar ];
 
+      # Ourselves have offset -1. In order to make these offset -1 dependencies of downstream derivation,
+      # they are offset 0 propagated.
+      propagatedBuildInputs =
+        self.lib.optional (pname == "rustc") [ self.stdenv.cc self.buildPackages.stdenv.cc ];
+      # This goes downstream packages' buildInputs.
+      depsTargetTargetPropagated =
+        self.lib.optional (pname == "rustc" && self.stdenv.targetPlatform.isDarwin) self.libiconv;
+
       installPhase = ''
         runHook preInstall
         installerVersion=$(< ./rust-installer-version)
@@ -298,24 +322,8 @@ let
             --replace "@cargo_miri@" "$out/bin/.cargo-miri-wrapped" \
             --replace "@out@" "$out"
         fi
-
-        # `symlinkJoin` (`runCommand`) doesn't handle propagated dependencies.
-        # Need to do it manually.
-        mkdir -p "$out/nix-support"
-        echo "$propagatedBuildInputs" > "$out/nix-support/propagated-build-inputs"
-        if [[ -n "$propagatedNativeBuildInputs" ]]; then
-          echo "$propagatedNativeBuildInputs" > "$out/nix-support/propagated-native-build-inputs"
-        fi
       '';
 
-      # FIXME: If these propagated dependencies go components, darwin build will fail with "`-liconv` not found".
-
-      # use propagatedNativebuildinputs here because we want these dependencies to end up
-      # as buildPlatform dependencies on the derivation using the compiler even though
-      # they are targetPlatform dependencies from the compiler's perspective
-      propagatedNativeBuildInputs = [ self.stdenv.cc self.targetPackages.stdenv.cc ] ++
-        self.lib.optional (self.stdenv.targetPlatform.isDarwin) self.targetPackages.libiconv;
-
       meta.platforms = self.lib.platforms.all;
     };
 
@@ -326,49 +334,88 @@ let
   # `name` is only used for error message.
   #
   # Returns a list of component derivations, or throw if failed.
-  resolveComponents = { name, componentSet, targetComponentsList, extensions, targets, targetExtensions }:
+  resolveComponents =
+    { name
+    , componentSet
+    , targetComponentsList
+    , profileComponents
+    , extensions
+    , targets
+    , targetExtensions
+    }:
     let
       inherit (self.lib) flatten elem isString filter any remove concatStringsSep concatMapStrings attrNames;
       rustHostPlatform = self.rust.toRustTarget self.stdenv.hostPlatform;
 
-      collectComponentTargets = compName: comp:
+      collectComponentTargets = allowMissing: compName: comp:
+        # Fail fast when missing extension.
+        if isString comp then
+          comp
         # Platform irrelevent components like `rust-src`.
-        if comp ? "*" then
+        else if comp ? "*" then
           comp."*"
         # Components for target platform like `rust-std`.
         else if elem compName targetComponentsList then
-          collectTargetComponentTargets compName comp
+          collectTargetComponentTargets allowMissing compName comp
         # Components for host platform like `rustc`.
         else
-          comp.${rustHostPlatform} or "Host component `${compName}` doesn't support target `${rustHostPlatform}`";
+          comp.${rustHostPlatform} or (
+            if allowMissing then []
+            else "Host component `${compName}` doesn't support target `${rustHostPlatform}`");
 
-      collectTargetComponentTargets = compName: comp:
+      collectTargetComponentTargets = allowMissing: compName: comp:
         let selected = remove null (map (tgt: comp.${tgt} or null) targets); in
-        if selected == []
+        if !allowMissing -> selected != [] then
-          then throw "Extension `${compName}` doesn't support any of targets: ${concatStringsSep ", " targets}"
+          selected
-          else selected;
+        else
+          "Extension `${compName}` doesn't support any of targets: ${concatStringsSep ", " targets}";
 
-      collectComponents = name: collectComponentTargets name (componentSet.${name} or "Missing extension `${name}`");
+      collectComponents = allowMissing: name:
-      collectTargetComponents = name: collectTargetComponentTargets name (componentSet.${name} or "Missing target extension `${name}`");
+        collectComponentTargets allowMissing name (componentSet.${name} or "Missing extension `${name}`");
+      collectTargetComponents = name:
+        collectTargetComponentTargets false name (componentSet.${name} or "Missing target extension `${name}`");
 
+      # Profile components can be skipped silently when missing.
+      # Eg. `rust-mingw` on non-Windows platforms, or `rust-docs` on non-tier1 platforms.
       result =
-        flatten (map collectComponents extensions) ++
+        flatten (map (collectComponents true) profileComponents) ++
+        flatten (map (collectComponents false) extensions) ++
         flatten (map collectTargetComponents targetExtensions);
 
       isTargetUnused = target:
         !any (name: componentSet ? ${name}.${target})
-          (filter (name: elem name targetComponentsList) extensions ++ targetExtensions);
+          (filter (name: elem name targetComponentsList)
+            (profileComponents ++ extensions)
+          ++ targetExtensions);
 
       errors = filter isString result ++
         map (tgt: "Target `${tgt}` is not supported by any components or extensions")
           (filter isTargetUnused targets);
 
+      allComponents = attrNames componentSet;
+      availableComponentsFor = host:
+        let
+          comps =
+            filter
+              (name: componentSet.${name} ? ${host})
+              allComponents;
+        in
+          if comps == [] then "<empty>" else comps;
+
+      notes = [
+        "note: all extensions are: ${toString allComponents}"
+      ] ++ map (host:
+        "note: extensions available for ${host} are: ${toString (availableComponentsFor host)}"
+      ) targets ++ [
+        "note: Check here to see whether a component is available for rustup:
+          https://rust-lang.github.io/rustup-components-history"
+      ];
+
     in
       if errors == [] then result
       else throw ''
         Component resolution failed for ${name}
-        - note: available extensions are ${concatStringsSep ", " (attrNames componentSet)}
+        ${concatMapStrings (msg: "- ${msg}\n") (errors ++ notes)}
-        ${concatMapStrings (msg: "- ${msg}\n") errors}
       '';
 
   # Genereate the toolchain set from a parsed manifest.
@@ -454,20 +501,20 @@ let
     ) (removeAttrs manifest.pkg ["rust"]) //
     mapAttrs (name: { to }: componentSet.${to}) manifest.renames;
 
-    mkProfile = name: componentNames:
+    mkProfile = name: profileComponents:
       makeOverridable ({ extensions, targets, targetExtensions }:
         aggregateComponents {
           pname = "rust-${name}";
           version = manifest.version;
           components = resolveComponents {
             name = "rust-${name}-${manifest.version}";
-            inherit componentSet;
+            inherit componentSet profileComponents;
             inherit (manifest) targetComponentsList;
-            extensions = componentNames ++ extensions;
+            extensions = extensions;
-            targets = [
+            targets = self.lib.unique ([
               (toRustTarget self.stdenv.hostPlatform) # Build script requires host std.
               (toRustTarget self.stdenv.targetPlatform)
-            ] ++ targets;
+            ] ++ targets);
             inherit targetExtensions;
           };
         }