Merge pull request #2 from oxalica/version-and-renames

Tweak manifest format and support renames

README.md

@@ -147,8 +147,13 @@ Some examples (assume `nixpkgs` had the overlay applied):
     targets = [ "arm-unknown-linux-gnueabihf" ];
   }
   ```
+- If you already have a [`rust-toolchain` file for rustup](https://github.com/ebroto/rustup/blob/c2db7dac6b38c99538eec472db9d23d18f918409/README.md#the-toolchain-file),
+  you can simply use `fromRustupToolchainFile` to get the customized toolchain derivation.
 
-For detail about `override` , see the source code of `./rust-overlay.nix`,
-or README of [nixpkgs-mozilla][mozilla].
+  ```nix
+  nixpkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain
+  ```
+
+For more details, see also the source code of `./rust-overlay.nix`.
 
 [mozilla]: https://github.com/mozilla/nixpkgs-mozilla

fetch.py

@@ -23,6 +23,7 @@ DIST_ROOT = 'https://static.rust-lang.org/dist'
 NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'}
 MANIFEST_TMP_PATH = Path('manifest.tmp')
 TARGETS_PATH = Path('manifests/targets.nix')
+RENAMES_PATH = Path('manifests/renames.nix')
 
 RE_STABLE_VERSION = re.compile(r'^\d+\.\d+\.\d+$')
 
@@ -66,6 +67,25 @@ def compress_target(target: str) -> str:
         f.write('}\n')
     return f'_{idx}'
 
+renames_map = dict((line.strip(), i) for i, line in enumerate(RENAMES_PATH.read_text().strip().split('\n')[1:-1]))
+def compress_renames(renames: dict) -> int:
+    serialized = '{ ' + ''.join(
+        f'{escape_nix_key(k)} = {escape_nix_string(v["to"])}; '
+        for k, v in sorted(renames.items())
+    ) + '}'
+
+    if serialized in renames_map:
+        return renames_map[serialized]
+    idx = len(renames_map)
+    renames_map[serialized] = idx
+
+    with open(str(RENAMES_PATH), 'w') as f:
+        f.write('[\n')
+        for _, ser in sorted((idx, ser) for ser, idx in renames_map.items()):
+            f.write('  ' + ser + '\n')
+        f.write(']\n')
+    return idx
+
 def retry_with(f):
     i = 0
     while True:
@@ -81,17 +101,20 @@ def retry_with(f):
 def translate_dump_manifest(manifest: str, f, nightly=False):
     manifest = toml.loads(manifest)
     date = manifest['date']
-    version = manifest['pkg']['rustc']['version'].split()[0]
+    rustc_version = manifest['pkg']['rustc']['version'].split()[0]
+    renames_idx = compress_renames(manifest['renames'])
     strip_tail = '-preview'
 
     f.write('{')
-    f.write(f'date={escape_nix_string(date)};')
+    f.write(f'v={escape_nix_string(rustc_version)};')
+    f.write(f'd={escape_nix_string(date)};')
+    f.write(f'r={renames_idx};')
     for pkg_name in sorted(manifest['pkg'].keys()):
         pkg = manifest['pkg'][pkg_name]
         pkg_name_stripped = pkg_name[:-len(strip_tail)] if pkg_name.endswith(strip_tail) else pkg_name
         pkg_targets = sorted(pkg['target'].keys())
 
-        url_version = version
+        url_version = rustc_version
         for target_name in pkg_targets:
             target = pkg['target'][target_name]
             if not target['available']:
@@ -106,23 +129,9 @@ def translate_dump_manifest(manifest: str, f, nightly=False):
             assert url.startswith(start) and url.endswith(end), f'Unexpected url: {url}'
             url_version = url[len(start):-len(end)]
 
-        if url_version == 'nightly':
-            assert nightly
-            url_version_kind = 0
-        elif url_version == pkg['version'].split(' ')[0]:
-            assert not nightly
-            url_version_kind = 0
-        elif url_version == pkg['version']:
-            url_version_kind = 1
-        elif url_version == version:
-            url_version_kind = 2
-        else:
-            assert False, f'Unknow url version `{url_version}` for `{pkg_name}`'
-
         f.write(f'{pkg_name}={{')
-        f.write(f'v={escape_nix_string(pkg["version"])};')
-        if url_version_kind != 0:
-            f.write(f'k={url_version_kind};')
+        if not (url_version == rustc_version or (url_version == 'nightly' and nightly)):
+            f.write(f'u={escape_nix_string(url_version)};')
         for target_name in pkg_targets:
             target = pkg['target'][target_name]
             if not target['available']:

flake.nix

@@ -73,6 +73,12 @@
         url-kind-nightly = assertUrl nightly."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-nightly-${rustTarget}.tar.xz";
         url-fix = assertUrl nightly."2019-01-10".rustc "https://static.rust-lang.org/dist/2019-01-10/rustc-nightly-${rustTarget}.tar.xz";
 
+        rename-available = assertEq stable."1.48.0".rustfmt stable."1.48.0".rustfmt-preview;
+        rename-unavailable = {
+          assertion = !(stable."1.30.0" ? rustfmt);
+          message = "1.30.0 has rustfmt still in preview state";
+        };
+
         latest-stable = assertEq pkgs.latest.rustChannels.stable.rust stable.latest.rust;
         latest-nightly = assertEq pkgs.latest.rustChannels.nightly.rust nightly.latest.rust;
 
@@ -88,19 +94,18 @@
         rustup-toolchain-customization = assertEq
           (fromRustupToolchain {
             channel = "1.48.0";
-            # FIXME: Handle renames `rustfmt` -> `rustfmt-preview`.
-            components = [ "rustfmt-preview" "rustc-dev" ];
+            components = [ "rustfmt" "rustc-dev" ];
             targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ];
           })
           (stable."1.48.0".rust.override {
-            extensions = [ "rustfmt-preview" "rustc-dev" ];
+            extensions = [ "rustfmt" "rustc-dev" ];
             targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ];
           });
 
         rustup-toolchain-file-toml = assertEq
           (fromRustupToolchainFile ./tests/rust-toolchain-toml)
           (nightly."2020-07-10".rust.override {
-            extensions = [ "rustfmt-preview" "rustc-dev" ];
+            extensions = [ "rustfmt" "rustc-dev" ];
             targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ];
           });
         rustup-toolchain-file-legacy = assertEq

manifest.nix

@@ -3,6 +3,7 @@ with (prev.lib);
 with builtins;
 let
   targets = import ./manifests/targets.nix // { _ = "*"; };
+  renamesList = import ./manifests/renames.nix;
 
   inherit (final.rust-bin) distRoot;
 
@@ -39,32 +40,38 @@ let
     extensions = singleTargetTups ++ multiTargetTups;
   };
 
-  # version -> { pkgName = { _1 = "..."; } } -> { pkgName = { x86_64-unknown-linux-gnu = fetchurl { .. }; } }
-  uncompressManifest = nightly: version: { date, ... }@manifest: rec {
-    inherit date;
+  # Uncompress the compressed manifest to the original one
+  # (not complete but has enough information to make up the toolchain).
+  uncompressManifest = nightly: version: {
+    v, # rustc version
+    d, # date
+    r, # rename index
+    ...
+  }@manifest: rec {
+    date = d;
+    renames = mapAttrs (from: to: { inherit to; }) (elemAt renamesList r);
     pkg =
-      mapAttrs (pkgName: { v, k ? 0, ... }@hashes: {
-        version = v;
+      mapAttrs (pkgName: { u ? null /* url version */, ... }@hashes: {
+        # We use rustc version for all components to reduce manifest size.
+        # This version is just used for component derivation name.
+        version = "${v} (000000000 ${d})"; # "<version> (<commit-hash> yyyy-mm-dd)"
         target =
           mapAttrs' (targetIdx: hash: let
             target = targets.${targetIdx};
             pkgNameStripped = removeSuffix "-preview" pkgName;
             targetTail = if targetIdx == "_" then "" else "-" + target;
-            vHead = head (match "([^ ]*) .*" v);
             urlVersion =
-              if nightly then "nightly"     # 'nightly'
-              else if k == 0 then vHead     # '0.44.1 (aaaaaaaaa 2018-01-01)' -> '0.44.1' [package version]
-              else if k == 1 then v         # '0.44.1 (aaaaaaaaa 2018-01-01)' [package version]
-              else if k == 2 then version   # '1.49.0' [stable toolchain version]
-              else throw "Invalid k";
+              if u != null then u             # Use specified url version if exists.
+              else if nightly then "nightly"  # Otherwise, for nightly channel, default to be "nightly".
+              else v;                         # For stable channel, default to be rustc version.
           in {
             name = target;
             value = {
               xz_url = "${distRoot}/${date}/${pkgNameStripped}-${urlVersion}${targetTail}.tar.xz";
               xz_hash = hash;
             } // (if pkgName == "rust" then rustPkgExtra pkg target else {});
-          }) (removeAttrs hashes ["v" "k"]);
-      }) (removeAttrs manifest ["date"]);
+          }) (removeAttrs hashes ["u"]);
+      }) (removeAttrs manifest ["v" "d" "r"]);
   };
 
   uncompressManifestSet = nightly: set: let