Add stable toolchains down to 1.29.0

default.nix

@@ -1,7 +1,7 @@
 final: prev:
+with (prev.lib);
+with builtins;
 let
-  inherit (prev.lib) filter hasAttr attrNames mapAttrs concatMap mapAttrs' replaceStrings;
-
   targets = import ./manifests/targets.nix // { _ = "*"; };
 
   distServer = "https://static.rust-lang.org";
@@ -43,20 +43,25 @@ let
   uncompressManifest = version: { date, ... }@manifest: rec {
     inherit date;
     pkg =
-      mapAttrs (pkgName: { v, ... }@hashes: {
+      mapAttrs (pkgName: { v, k ? 0, ... }@hashes: {
         version = v;
         target =
           mapAttrs' (targetIdx: hash: let
             target = targets.${targetIdx};
-            pkgNameStripped = replaceStrings ["-preview"] [""] pkgName;
+            pkgNameStripped = removeSuffix "-preview" pkgName;
             targetTail = if targetIdx == "_" then "" else "-" + target;
+            urlVersion =
+              if k == 0 then head (match "([^ ]*) .*" v) # '0.44.1 (aaaaaaaaa 2018-01-01)' -> '0.44.1' [package version]
+              else if k == 1 then v                      # '0.44.1 (aaaaaaaaa 2018-01-01)' [package version]
+              else if k == 2 then version                # '1.49.0' [stable toolchain version]
+              else throw "Invalid k";
           in {
             name = target;
             value = {
-              xz_url = "${distServer}/dist/${date}/${pkgNameStripped}-${version}${targetTail}.tar.xz";
+              xz_url = "${distServer}/dist/${date}/${pkgNameStripped}-${urlVersion}${targetTail}.tar.xz";
               xz_hash = hash;
             } // (if pkgName == "rust" then rustPkgExtra pkg target else {});
-          }) (removeAttrs hashes ["v"]);
+          }) (removeAttrs hashes ["v" "k"]);
       }) (removeAttrs manifest ["date"]);
   };
 

fetch.py

@@ -15,7 +15,7 @@ MAX_TRIES = 3
 RETRY_DELAY = 3.0
 SYNC_MAX_FETCH = 5
 
-STABLE_VERSION_FILTER = lambda v: parse_version(v) >= (1, 29, 0)
+MIN_STABLE_VERSION = '1.29.0'
 
 DIST_SERVER = 'https://static.rust-lang.org'
 NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'}
@@ -89,7 +89,7 @@ def translate_dump_manifest(manifest: str, f):
         pkg_name_stripped = pkg_name[:-len(strip_tail)] if pkg_name.endswith(strip_tail) else pkg_name
         pkg_targets = sorted(pkg['target'].keys())
 
-        pkg_version = version
+        url_version = version
         for target_name in pkg_targets:
             target = pkg['target'][target_name]
             if not target['available']:
@@ -99,10 +99,21 @@ def translate_dump_manifest(manifest: str, f):
             start = f'https://static.rust-lang.org/dist/{date}/{pkg_name_stripped}-'
             end = f'{target_tail}.tar.xz'
             assert url.startswith(start) and url.endswith(end), f'Unexpected url: {url}'
-            pkg_version = url[len(start):-len(end)]
+            url_version = url[len(start):-len(end)]
+
+        if url_version == pkg['version'].split(' ')[0]:
+            url_version_kind = 0
+        elif url_version == pkg['version']:
+            url_version_kind = 1
+        elif url_version == version:
+            url_version_kind = 2
+        else:
+            assert False, f'Known url version `{url_version}` for `{pkg_name}`'
 
         f.write(f'{pkg_name}={{')
         f.write(f'v={escape_nix_string(pkg["version"])};')
+        if url_version_kind != 0:
+            f.write(f'k={url_version_kind};')
         for target_name in pkg_targets:
             target = pkg['target'][target_name]
             if not target['available']:
@@ -110,7 +121,7 @@ def translate_dump_manifest(manifest: str, f):
             url = target['xz_url']
             hash = to_base64(target['xz_hash']) # Hash must not contains quotes.
             target_tail = '' if target_name == '*' else '-' + target_name
-            expect_url = f'https://static.rust-lang.org/dist/{date}/{pkg_name_stripped}-{pkg_version}{target_tail}.tar.xz'
+            expect_url = f'https://static.rust-lang.org/dist/{date}/{pkg_name_stripped}-{url_version}{target_tail}.tar.xz'
             assert url == expect_url, f'Unexpected url: {url}, expecting: {expect_url}'
             f.write(f'{compress_target(target_name)}="{hash}";')
         f.write('};')
@@ -142,35 +153,43 @@ def update_stable_index():
 
 def sync_stable_channel(*, stop_if_exists, max_fetch=None):
     GITHUB_RELEASES_URL = 'https://api.github.com/repos/rust-lang/rust/releases'
+    PER_PAGE = 100
 
+    versions = []
     page = 0
-    count = 0
-    try:
-        while True:
-            page += 1
-            print(f'Fetching release page {page}')
-            release_page = retry_with(lambda: requests.get(GITHUB_RELEASES_URL, params={'page': page}))
-            release_page.raise_for_status()
-            release_page = release_page.json()
-            if not release_page:
-                return
-            for release in release_page:
-                version = release['tag_name']
-                if not RE_STABLE_VERSION.match(version) or not STABLE_VERSION_FILTER(version):
-                    continue
-                out_path = Path(f'manifests/stable/{version}.nix')
-                if out_path.exists():
-                    if stop_if_exists:
-                        print('Stopped on existing version')
-                        return
-                    continue
-                fetch_stable_manifest(version, out_path)
-                count += 1
-                if max_fetch is not None and count >= max_fetch:
-                    print('Reached max fetch count')
-                    exit(1)
-    finally:
-        update_stable_index()
+    while True:
+        page += 1
+        print(f'Fetching release page {page}')
+        release_page = retry_with(lambda: requests.get(
+            GITHUB_RELEASES_URL,
+            params={'per_page': PER_PAGE, 'page': page},
+        ))
+        release_page.raise_for_status()
+        release_page = release_page.json()
+        versions.extend(
+            tag['tag_name']
+            for tag in release_page
+            if RE_STABLE_VERSION.match(tag['tag_name'])
+            and not version_less(tag['tag_name'], MIN_STABLE_VERSION)
+        )
+        if len(release_page) < PER_PAGE:
+            break
+    versions.sort(key=parse_version, reverse=True)
+
+    print(f'Got {len(release_page)} releases to fetch')
+
+    processed = 0
+    for version in versions:
+        out_path = Path(f'manifests/stable/{version}.nix')
+        if out_path.exists():
+            if not stop_if_exists:
+                continue
+            print('Stopped on existing version')
+            break
+        fetch_stable_manifest(version, out_path)
+        processed += 1
+        assert max_fetch is None or processed <= max_fetch, 'Too many versions'
+    update_stable_index()
 
 def init_sync_all():
     sync_stable_channel(stop_if_exists=False)
@@ -181,16 +200,23 @@ def main():
         print('Synchronizing channels')
         sync_stable_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
     elif len(args) == 1:
-        version = args[0]
-        assert RE_STABLE_VERSION.match(version), 'Invalid version'
-        fetch_stable_manifest(version, Path(f'manifests/stable/{version}.nix'))
-        update_stable_index()
+        if args[0] == 'all':
+            sync_stable_channel(stop_if_exists=False)
+        else:
+            version = args[0]
+            assert RE_STABLE_VERSION.match(version), 'Invalid version'
+            fetch_stable_manifest(version, Path(f'manifests/stable/{version}.nix'))
+            update_stable_index()
     else:
-        print(f'''
-Usage: {sys.argv[0]} [version]
-    Run without arguments to auto-sync new versions from channels.
-    Run with version to fetch a specific version.
-''')
+        print('''
+Usage:
+    {0}
+        Auto-sync new versions from channels.
+    {0} <version>
+        Force to fetch a specific version.
+    {0} all
+        Force to fetch all versions.
+'''.format(sys.argv[0]))
         exit(1)
 
 if __name__ == '__main__':

flake.nix

@@ -36,5 +36,11 @@
     packages = {
       rust-stable = pkgs.latest.rustChannels.stable.rust;
     };
+
+    checks = {
+      kind2 = (pkgs.rustChannelOf { channel = "1.48.0"; }).rust;
+      kind0 = (pkgs.rustChannelOf { channel = "1.47.0"; }).rust;
+      kind1 = (pkgs.rustChannelOf { channel = "1.34.2"; }).llvm-tools-preview;
+    };
   });
 }

manifests/stable/default.nix

@@ -1,4 +1,34 @@
 {
+  "1.29.0" = import ./1.29.0.nix;
+  "1.29.1" = import ./1.29.1.nix;
+  "1.29.2" = import ./1.29.2.nix;
+  "1.30.0" = import ./1.30.0.nix;
+  "1.30.1" = import ./1.30.1.nix;
+  "1.31.0" = import ./1.31.0.nix;
+  "1.31.1" = import ./1.31.1.nix;
+  "1.32.0" = import ./1.32.0.nix;
+  "1.33.0" = import ./1.33.0.nix;
+  "1.34.0" = import ./1.34.0.nix;
+  "1.34.1" = import ./1.34.1.nix;
+  "1.34.2" = import ./1.34.2.nix;
+  "1.35.0" = import ./1.35.0.nix;
+  "1.36.0" = import ./1.36.0.nix;
+  "1.37.0" = import ./1.37.0.nix;
+  "1.38.0" = import ./1.38.0.nix;
+  "1.39.0" = import ./1.39.0.nix;
+  "1.40.0" = import ./1.40.0.nix;
+  "1.41.0" = import ./1.41.0.nix;
+  "1.41.1" = import ./1.41.1.nix;
+  "1.42.0" = import ./1.42.0.nix;
+  "1.43.0" = import ./1.43.0.nix;
+  "1.43.1" = import ./1.43.1.nix;
+  "1.44.0" = import ./1.44.0.nix;
+  "1.44.1" = import ./1.44.1.nix;
+  "1.45.0" = import ./1.45.0.nix;
+  "1.45.1" = import ./1.45.1.nix;
+  "1.45.2" = import ./1.45.2.nix;
+  "1.46.0" = import ./1.46.0.nix;
+  "1.47.0" = import ./1.47.0.nix;
   "1.48.0" = import ./1.48.0.nix;
   "1.49.0" = import ./1.49.0.nix;
   latest = "1.49.0";

rust-overlay.nix

@@ -64,10 +64,15 @@ let
 
   getFetchUrl = pkgs: pkgname: target: stdenv: fetchurl:
     let
+      inherit (builtins) match elemAt;
       pkg = pkgs.${pkgname};
       srcInfo = pkg.target.${target};
+      url = builtins.replaceStrings [" "] ["%20"] srcInfo.xz_url; # This is required or download will fail.
+      # Filter names like `llvm-tools-1.34.2 (6c2484dc3 2019-05-13)-aarch64-unknown-linux-gnu.tar.xz`
+      matchParenPart = match ".*/([^ /]*) [(][^)]*[)](.*)" srcInfo.xz_url;
+      name = if matchParenPart == null then "" else (elemAt matchParenPart 0) + (elemAt matchParenPart 1);
     in
-      (super.fetchurl { url = srcInfo.xz_url; sha256 = srcInfo.xz_hash; });
+      (super.fetchurl { inherit name url; sha256 = srcInfo.xz_hash; });
 
   checkMissingExtensions = pkgs: pkgname: stdenv: extensions:
     let