diff --git a/fetch.py b/fetch.py index aa500813ff8..1e11e9e1d46 100755 --- a/fetch.py +++ b/fetch.py @@ -23,6 +23,7 @@ DIST_ROOT = 'https://static.rust-lang.org/dist' NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'} MANIFEST_TMP_PATH = Path('manifest.tmp') TARGETS_PATH = Path('manifests/targets.nix') +RENAMES_PATH = Path('manifests/renames.nix') RE_STABLE_VERSION = re.compile(r'^\d+\.\d+\.\d+$') @@ -66,6 +67,25 @@ def compress_target(target: str) -> str: f.write('}\n') return f'_{idx}' +renames_map = dict((line.strip(), i) for i, line in enumerate(RENAMES_PATH.read_text().strip().split('\n')[1:-1])) +def compress_renames(renames: dict) -> int: + serialized = '{ ' + ''.join( + f'{escape_nix_key(k)} = {escape_nix_string(v["to"])}; ' + for k, v in sorted(renames.items()) + ) + '}' + + if serialized in renames_map: + return renames_map[serialized] + idx = len(renames_map) + renames_map[serialized] = idx + + with open(str(RENAMES_PATH), 'w') as f: + f.write('[\n') + for _, ser in sorted((idx, ser) for ser, idx in renames_map.items()): + f.write(' ' + ser + '\n') + f.write(']\n') + return idx + def retry_with(f): i = 0 while True: @@ -82,10 +102,12 @@ def translate_dump_manifest(manifest: str, f, nightly=False): manifest = toml.loads(manifest) date = manifest['date'] version = manifest['pkg']['rustc']['version'].split()[0] + renames_idx = compress_renames(manifest['renames']) strip_tail = '-preview' f.write('{') f.write(f'date={escape_nix_string(date)};') + f.write(f'r={renames_idx};') for pkg_name in sorted(manifest['pkg'].keys()): pkg = manifest['pkg'][pkg_name] pkg_name_stripped = pkg_name[:-len(strip_tail)] if pkg_name.endswith(strip_tail) else pkg_name diff --git a/flake.nix b/flake.nix index 685b6e10abc..a99aac42d57 100644 --- a/flake.nix +++ b/flake.nix @@ -73,6 +73,12 @@ url-kind-nightly = assertUrl nightly."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-nightly-${rustTarget}.tar.xz"; url-fix = assertUrl nightly."2019-01-10".rustc "https://static.rust-lang.org/dist/2019-01-10/rustc-nightly-${rustTarget}.tar.xz"; + rename-available = assertEq stable."1.48.0".rustfmt stable."1.48.0".rustfmt-preview; + rename-unavailable = { + assertion = !(stable."1.30.0" ? rustfmt); + message = "1.30.0 has rustfmt still in preview state"; + }; + latest-stable = assertEq pkgs.latest.rustChannels.stable.rust stable.latest.rust; latest-nightly = assertEq pkgs.latest.rustChannels.nightly.rust nightly.latest.rust; @@ -88,19 +94,18 @@ rustup-toolchain-customization = assertEq (fromRustupToolchain { channel = "1.48.0"; - # FIXME: Handle renames `rustfmt` -> `rustfmt-preview`. - components = [ "rustfmt-preview" "rustc-dev" ]; + components = [ "rustfmt" "rustc-dev" ]; targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ]; }) (stable."1.48.0".rust.override { - extensions = [ "rustfmt-preview" "rustc-dev" ]; + extensions = [ "rustfmt" "rustc-dev" ]; targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ]; }); rustup-toolchain-file-toml = assertEq (fromRustupToolchainFile ./tests/rust-toolchain-toml) (nightly."2020-07-10".rust.override { - extensions = [ "rustfmt-preview" "rustc-dev" ]; + extensions = [ "rustfmt" "rustc-dev" ]; targets = [ "wasm32-unknown-unknown" "aarch64-unknown-linux-gnu" ]; }); rustup-toolchain-file-legacy = assertEq diff --git a/manifest.nix b/manifest.nix index 76ec893a562..f07ea03f07f 100644 --- a/manifest.nix +++ b/manifest.nix @@ -3,6 +3,7 @@ with (prev.lib); with builtins; let targets = import ./manifests/targets.nix // { _ = "*"; }; + renamesList = import ./manifests/renames.nix; inherit (final.rust-bin) distRoot; @@ -39,11 +40,13 @@ let extensions = singleTargetTups ++ multiTargetTups; }; - # version -> { pkgName = { _1 = "..."; } } -> { pkgName = { x86_64-unknown-linux-gnu = fetchurl { .. }; } } - uncompressManifest = nightly: version: { date, ... }@manifest: rec { + # Uncompress the compressed manifest to the original one + # (not complete but has enough information to make up the toolchain). + uncompressManifest = nightly: version: { date /* date */, r /* rename index */, ... }@manifest: rec { inherit date; + renames = mapAttrs (from: to: { inherit to; }) (elemAt renamesList r); pkg = - mapAttrs (pkgName: { v, k ? 0, ... }@hashes: { + mapAttrs (pkgName: { v /* pkg version */, k ? 0 /* url kind */, ... }@hashes: { version = v; target = mapAttrs' (targetIdx: hash: let @@ -64,7 +67,7 @@ let xz_hash = hash; } // (if pkgName == "rust" then rustPkgExtra pkg target else {}); }) (removeAttrs hashes ["v" "k"]); - }) (removeAttrs manifest ["date"]); + }) (removeAttrs manifest ["date" "r"]); }; uncompressManifestSet = nightly: set: let diff --git a/manifests/renames.nix b/manifests/renames.nix new file mode 100644 index 00000000000..0d4f101c7a3 --- /dev/null +++ b/manifests/renames.nix @@ -0,0 +1,2 @@ +[ +] diff --git a/repl.nix b/repl.nix new file mode 100644 index 00000000000..c2997257a15 --- /dev/null +++ b/repl.nix @@ -0,0 +1,4 @@ +# For test only. +import { + overlays = [ (import ./.) ]; +} diff --git a/rust-overlay.nix b/rust-overlay.nix index 2943532c786..5977da666be 100644 --- a/rust-overlay.nix +++ b/rust-overlay.nix @@ -252,18 +252,21 @@ let # All extensions in this list will be installed for the target architectures. # *Attention* If you want to install an extension like rust-src, that has no fixed architecture (arch *), # you will need to specify this extension in the extensions options or it will not be installed! - toolchainFromManifest = pkgs: - let - inherit (builtins) elemAt; - inherit (super) makeOverridable; - inherit (super.lib) flip mapAttrs; - in - flip mapAttrs pkgs.pkg (name: pkg: + toolchainFromManifest = pkgs: let + inherit (builtins) elemAt; + inherit (super) makeOverridable; + inherit (super.lib) flip mapAttrs; + + maybeRename = name: pkgs.renames.${name}.to or name; + + mkPackage = name: pkg: makeOverridable ({ extensions, targets, targetExtensions, stdenv, fetchurl, patchelf }: let version' = builtins.match "([^ ]*) [(]([^ ]*) ([^ ]*)[)]" pkg.version; version = if version' == null then pkg.version else "${elemAt version' 0}-${elemAt version' 2}-${elemAt version' 1}"; - namesAndSrcs = getComponents pkgs.pkg name targets extensions targetExtensions stdenv fetchurl; + extensions' = map maybeRename extensions; + targetExtensions' = map maybeRename targetExtensions; + namesAndSrcs = getComponents pkgs.pkg name targets extensions' targetExtensions' stdenv fetchurl; components = installComponents stdenv namesAndSrcs; componentsOuts = builtins.map (comp: (super.lib.strings.escapeNixString (super.lib.getOutput "out" comp))) components; in @@ -296,8 +299,13 @@ let targets = []; targetExtensions = []; inherit (self) stdenv fetchurl patchelf; - } - ); + }; + + toolchain = + mapAttrs mkPackage pkgs.pkg // + mapAttrs (from: { to }: toolchain.${to}) pkgs.renames; + + in toolchain; # Same as `toolchainFromManifest` but read from a manifest file. toolchainFromManifestFile = path: toolchainFromManifest (builtins.fromTOML (builtins.readFile path)); diff --git a/tests/rust-toolchain-toml b/tests/rust-toolchain-toml index 397b6be6f9b..08e7991f1bd 100644 --- a/tests/rust-toolchain-toml +++ b/tests/rust-toolchain-toml @@ -1,4 +1,4 @@ [toolchain] channel = "nightly-2020-07-10" -components = [ "rustfmt-preview", "rustc-dev" ] +components = [ "rustfmt", "rustc-dev" ] targets = [ "wasm32-unknown-unknown", "aarch64-unknown-linux-gnu" ]