Merge pull request #4 from oxalica/add-beta

Add beta channel support

.github/workflows/ci.yaml

@@ -29,21 +29,25 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-    name: Check build latest Rust stable
+        channel: [stable, beta]
+    name: Check build Rust
     runs-on: ${{ matrix.os }}
+    env:
+      build_expr: (import <nixpkgs> { overlays = [ (import ./.) ]; }).rust-bin.${{ matrix.channel }}.latest.rust
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 0 # Nix Flakes doesn't work on shallow clones
     - name: Install Nix (flakes, nixos-unstable)
       uses: cachix/install-nix-action@v12
       with:
-        install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20201221_9fab14a/install
-        extra_nix_config: |
-          experimental-features = nix-command flakes
         nix_path: nixpkgs=channel:nixos-unstable
-    - name: Build
+    - name: Build Rust
       run: |
-        nix build ".#rust" -L
+        nix-build -E "$build_expr"
         ./result/bin/rustc --version
+    - name: Check compile
+      run: |
+        echo 'fn main() { println!("hello, world"); }' >./test.rs
+        # Use nix-shell to bring propagatedBuildInputs into scope.
+        nix-shell -p "$build_expr" --pure --command "rustc test.rs -o test"
+        [[ "$(./test)" == "hello, world" ]]

.github/workflows/sync-channels.yaml

@@ -26,10 +26,16 @@ jobs:
       run: pip3 install toml requests
     - name: Install Nix
       uses: cachix/install-nix-action@v12
-    - name: Sync channels
+    - name: Sync stable channel
       timeout-minutes: 5
-      run: python3 ./fetch.py
-    - name: Commit changes
+      run: python3 ./fetch.py stable
+    - name: Sync beta channel
+      timeout-minutes: 5
+      run: python3 ./fetch.py beta
+    - name: Sync nightly channel
+      timeout-minutes: 5
+      run: python3 ./fetch.py nightly
+    - name: Check and commit changes
       id: commit
       continue-on-error: true
       run: |

README.md

@@ -11,8 +11,8 @@ so the evaluation is *pure* and no need to have network (but [nixpkgs-mozilla][m
 It also works well with [Nix Flakes](https://nixos.wiki/wiki/Flakes).
 
 - The toolchain hashes are auto-updated daily using GitHub Actions.
-- Current oldest supported version is stable 1.29.0 and nightly 2018-09-13
-  (which is randomly chosen).
+- Current oldest supported version is stable 1.29.0 and beta/nightly 2018-09-13
+  (which are randomly chosen).
 
 ## Use as a classic Nix overlay
 
@@ -108,6 +108,14 @@ Here's an example of using it in nixos configuration.
       # ... other versions.
     };
 
+    beta = {
+      # The latest beta toolchain.
+      latest = { /* toolchain */ };
+      "2021-01-01" = { /* toolchain */ };
+      "2020-12-30" = { /* toolchain */ };
+      # ... other versions.
+    };
+
     nightly = {
       # The latest nightly toolchain.
       latest = { /* toolchain */ };
@@ -130,10 +138,12 @@ Here's an example of using it in nixos configuration.
 
 Some examples (assume `nixpkgs` had the overlay applied):
 
-- Latest stable rust with all default components:
-  `nixpkgs.rust-bin.stable.latest.rust`
-- Latest nightly rust with all default components:
-  `nixpkgs.rust-bin.nightly.latest.rust`
+- Latest stable/beta/nightly rust with all default components:
+  `nixpkgs.rust-bin.{stable,beta,nightly}.latest.rust`
+- A specific version of stable rust:
+  `nixpkgs.rust-bin.stable."1.48.0".rust`
+- A specific date of beta rust:
+  `nixpkgs.rust-bin.nightly."2021-01-01".rust`
 - A specific version of stable rust:
   `nixpkgs.rust-bin.stable."1.48.0".rust`
 - A specific date of nightly rust:

fetch.py

@@ -17,7 +17,7 @@ RETRY_DELAY = 3.0
 SYNC_MAX_FETCH = 5
 
 MIN_STABLE_VERSION = '1.29.0'
-MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
+MIN_BETA_DATE = MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
 
 DIST_ROOT = 'https://static.rust-lang.org/dist'
 NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'}
@@ -86,19 +86,24 @@ def compress_renames(renames: dict) -> int:
         f.write(']\n')
     return idx
 
-def retry_with(f):
+def fetch_url(url: str, params=None, allow_not_found=False):
     i = 0
     while True:
+        resp = None
         try:
-            return f()
+            resp = requests.get(url, params=params)
+            if resp.status_code == 404 and allow_not_found:
+                return None
+            resp.raise_for_status()
+            return resp
         except requests.exceptions.RequestException as e:
             i += 1
-            if i >= MAX_TRIES:
+            if (resp is not None and resp.status_code == 404) or i >= MAX_TRIES:
                 raise
             print(e)
             time.sleep(RETRY_DELAY)
 
-def translate_dump_manifest(manifest: str, f, nightly=False):
+def translate_dump_manifest(channel: str, manifest: str, f):
     manifest = toml.loads(manifest)
     date = manifest['date']
     rustc_version = manifest['pkg']['rustc']['version'].split()[0]
@@ -148,20 +153,25 @@ def translate_dump_manifest(manifest: str, f, nightly=False):
         f.write('};')
     f.write('}\n')
 
-def fetch_stable_manifest(version: str, out_path: Path):
+def fetch_manifest(channel: str, version: str, out_path: Path):
     out_path.parent.mkdir(parents=True, exist_ok=True)
     tmp_path = out_path.with_suffix('.tmp')
-    print(f'Fetching stable {version}')
-    manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-{version}.toml'))
-    manifest.raise_for_status()
+    print(f'Fetching {channel} {version}')
+    if channel == 'stable':
+        url = f'{DIST_ROOT}/channel-rust-{version}.toml'
+    else:
+        url = f'{DIST_ROOT}/{version}/channel-rust-{channel}.toml'
+    manifest = fetch_url(url, allow_not_found=channel != 'stable')
+    if manifest is None:
+        print('Not found, skipped')
+        return
     manifest = manifest.text
     MANIFEST_TMP_PATH.write_text(manifest)
     with open(tmp_path, 'w') as fout:
-        translate_dump_manifest(manifest, fout)
+        translate_dump_manifest(channel, manifest, fout)
     tmp_path.rename(out_path)
 
-def update_stable_index():
-    dir = Path('manifests/stable')
+def update_stable_index(dir=Path('manifests/stable')):
     versions = sorted(
         (file.stem for file in dir.iterdir() if file.stem != 'default' and file.suffix == '.nix'),
         key=parse_version,
@@ -173,6 +183,19 @@ def update_stable_index():
         f.write(f'  latest = {escape_nix_string(versions[-1])};\n')
         f.write('}\n')
 
+def update_beta_index():
+    update_nightly_index(dir=Path('manifests/beta'))
+
+def update_nightly_index(dir=Path('manifests/nightly')):
+    dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
+    with open(str(dir / 'default.nix'), 'w') as f:
+        f.write('{\n')
+        for date in dates:
+            year = date.split('-')[0]
+            f.write(f'  {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
+        f.write(f'  latest = {escape_nix_string(dates[-1])};\n')
+        f.write('}\n')
+
 def sync_stable_channel(*, stop_if_exists, max_fetch=None):
     GITHUB_RELEASES_URL = 'https://api.github.com/repos/rust-lang/rust/releases'
     PER_PAGE = 100
@@ -182,12 +205,10 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
     while True:
         page += 1
         print(f'Fetching release page {page}')
-        release_page = retry_with(lambda: requests.get(
+        release_page = fetch_url(
             GITHUB_RELEASES_URL,
             params={'per_page': PER_PAGE, 'page': page},
-        ))
-        release_page.raise_for_status()
-        release_page = release_page.json()
+        ).json()
         versions.extend(
             tag['tag_name']
             for tag in release_page
@@ -208,87 +229,94 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
                 continue
             print(f'{version} is already fetched. Stopped')
             break
-        fetch_stable_manifest(version, out_path)
+        fetch_manifest('stable', version, out_path)
         processed += 1
         assert max_fetch is None or processed <= max_fetch, 'Too many versions'
     update_stable_index()
 
-def fetch_nightly_manifest(date: str, out_path: Path):
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    tmp_path = out_path.with_suffix('.tmp')
-    print(f'Fetching nightly {date}')
-    manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/{date}/channel-rust-nightly.toml'))
-    if manifest.status_code == 404:
-        print(f'Not found, skipped')
-        return
-    manifest.raise_for_status()
-    manifest = manifest.text
-    MANIFEST_TMP_PATH.write_text(manifest)
-    with open(tmp_path, 'w') as fout:
-        translate_dump_manifest(manifest, fout, nightly=True)
-    tmp_path.rename(out_path)
+def sync_beta_channel(*, stop_if_exists, max_fetch=None):
+    # Fetch the global nightly manifest to retrive the latest nightly version.
+    print('Fetching latest beta version')
+    manifest = fetch_url(f'{DIST_ROOT}/channel-rust-beta.toml').text
+    date = datetime.date.fromisoformat(toml.loads(manifest)['date'])
+    print(f'The latest beta version is {date}')
+
+    processed = 0
+    date += datetime.timedelta(days=1)
+    while date > MIN_BETA_DATE:
+        date -= datetime.timedelta(days=1)
+        date_str = date.isoformat()
+        out_path = Path(f'manifests/beta/{date.year}/{date_str}.nix')
+        if out_path.exists():
+            if not stop_if_exists:
+                continue
+            print(f'{date_str} is already fetched. Stopped')
+            break
+        fetch_manifest('beta', date_str, out_path)
+        processed += 1
+        assert max_fetch is None or processed <= max_fetch, 'Too many versions'
+    update_beta_index()
 
 def sync_nightly_channel(*, stop_if_exists, max_fetch=None):
     # Fetch the global nightly manifest to retrive the latest nightly version.
     print('Fetching latest nightly version')
-    manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-nightly.toml'))
-    manifest.raise_for_status()
-    date = datetime.date.fromisoformat(toml.loads(manifest.text)['date'])
+    manifest = fetch_url(f'{DIST_ROOT}/channel-rust-nightly.toml').text
+    date = datetime.date.fromisoformat(toml.loads(manifest)['date'])
     print(f'The latest nightly version is {date}')
 
     processed = 0
     date += datetime.timedelta(days=1)
     while date > MIN_NIGHTLY_DATE:
         date -= datetime.timedelta(days=1)
-        out_path = Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix')
+        date_str = date.isoformat()
+        out_path = Path(f'manifests/nightly/{date.year}/{date_str}.nix')
         if out_path.exists():
             if not stop_if_exists:
                 continue
-            print(f'{date} is already fetched. Stopped')
+            print(f'{date_str} is already fetched. Stopped')
             break
-        fetch_nightly_manifest(date.isoformat(), out_path)
+        fetch_manifest('nightly', date_str, out_path)
         processed += 1
         assert max_fetch is None or processed <= max_fetch, 'Too many versions'
     update_nightly_index()
 
-def update_nightly_index():
-    dir = Path('manifests/nightly')
-    dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
-    with open(str(dir / 'default.nix'), 'w') as f:
-        f.write('{\n')
-        for date in dates:
-            year = date.split('-')[0]
-            f.write(f'  {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
-        f.write(f'  latest = {escape_nix_string(dates[-1])};\n')
-        f.write('}\n')
-
 def main():
     args = sys.argv[1:]
-    if len(args) == 0:
-        print('Synchronizing stable channels')
-        sync_stable_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
-        print('Synchronizing nightly channels')
-        sync_nightly_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
+    if len(args) == 1 and args[0] in ['stable', 'beta', 'nightly']:
+        {
+            'stable': sync_stable_channel,
+            'beta': sync_beta_channel,
+            'nightly': sync_nightly_channel,
+        }[args[0]](stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
     elif len(args) == 2 and args[0] == 'stable':
         if args[1] == 'all':
             sync_stable_channel(stop_if_exists=False)
         else:
             version = args[1]
             assert RE_STABLE_VERSION.match(version), 'Invalid version'
-            fetch_stable_manifest(version, Path(f'manifests/stable/{version}.nix'))
+            fetch_manifest('stable', version, Path(f'manifests/stable/{version}.nix'))
             update_stable_index()
+    elif len(args) == 2 and args[0] == 'beta':
+        if args[1] == 'all':
+            sync_beta_channel(stop_if_exists=False)
+        else:
+            date = datetime.date.fromisoformat(args[1])
+            date_str = date.isoformat()
+            fetch_manifest('beta', date_str, Path(f'manifests/beta/{date.year}/{date_str}.nix'))
+            update_beta_index()
     elif len(args) == 2 and args[0] == 'nightly':
         if args[1] == 'all':
             sync_nightly_channel(stop_if_exists=False)
         else:
             date = datetime.date.fromisoformat(args[1])
-            fetch_nightly_manifest(date, Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix'))
+            date_str = date.isoformat()
+            fetch_manifest('nightly', date_str, Path(f'manifests/nightly/{date.year}/{date_str}.nix'))
             update_nightly_index()
     else:
         print('''
 Usage:
-    {0}
-        Auto-sync new versions from channels.
+    {0} <channel>
+        Auto-sync new versions from a channel.
     {0} <channel> <version>
         Force to fetch a specific version from a channel.
     {0} <channel> all

flake.nix

@@ -53,7 +53,7 @@
 
     checks = let
       inherit (pkgs) rust-bin rustChannelOf;
-      inherit (pkgs.rust-bin) fromRustupToolchain fromRustupToolchainFile stable nightly;
+      inherit (pkgs.rust-bin) fromRustupToolchain fromRustupToolchainFile stable beta nightly;
 
       rustTarget = pkgs.rust.toRustTarget pkgs.hostPlatform;
 
@@ -71,6 +71,7 @@
         url-kind-0 = assertUrl stable."1.47.0".cargo "https://static.rust-lang.org/dist/2020-10-08/cargo-0.48.0-${rustTarget}.tar.xz";
         url-kind-1 = assertUrl stable."1.34.2".llvm-tools-preview "https://static.rust-lang.org/dist/2019-05-14/llvm-tools-1.34.2%20(6c2484dc3%202019-05-13)-${rustTarget}.tar.xz";
         url-kind-nightly = assertUrl nightly."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-nightly-${rustTarget}.tar.xz";
+        url-kind-beta = assertUrl beta."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-beta-${rustTarget}.tar.xz";
         url-fix = assertUrl nightly."2019-01-10".rustc "https://static.rust-lang.org/dist/2019-01-10/rustc-nightly-${rustTarget}.tar.xz";
 
         rename-available = assertEq stable."1.48.0".rustfmt stable."1.48.0".rustfmt-preview;
@@ -80,17 +81,22 @@
         };
 
         latest-stable = assertEq pkgs.latest.rustChannels.stable.rust stable.latest.rust;
+        latest-beta = assertEq pkgs.latest.rustChannels.beta.rust beta.latest.rust;
         latest-nightly = assertEq pkgs.latest.rustChannels.nightly.rust nightly.latest.rust;
 
         rust-channel-of-stable = assertEq (rustChannelOf { channel = "stable"; }).rust stable.latest.rust;
+        rust-channel-of-beta = assertEq (rustChannelOf { channel = "beta"; }).rust beta.latest.rust;
         rust-channel-of-nightly = assertEq (rustChannelOf { channel = "nightly"; }).rust nightly.latest.rust;
         rust-channel-of-version = assertEq (rustChannelOf { channel = "1.48.0"; }).rust stable."1.48.0".rust;
         rust-channel-of-nightly-date = assertEq (rustChannelOf { channel = "nightly"; date = "2021-01-01"; }).rust nightly."2021-01-01".rust;
+        rust-channel-of-beta-date = assertEq (rustChannelOf { channel = "beta"; date = "2021-01-01"; }).rust beta."2021-01-01".rust;
 
         rustup-toolchain-stable = assertEq (fromRustupToolchain { channel = "stable"; }) stable.latest.rust;
+        rustup-toolchain-beta = assertEq (fromRustupToolchain { channel = "beta"; }) beta.latest.rust;
         rustup-toolchain-nightly = assertEq (fromRustupToolchain { channel = "nightly"; }) nightly.latest.rust;
         rustup-toolchain-version = assertEq (fromRustupToolchain { channel = "1.48.0"; }) stable."1.48.0".rust;
         rustup-toolchain-nightly-date = assertEq (fromRustupToolchain { channel = "nightly-2021-01-01"; }) nightly."2021-01-01".rust;
+        rustup-toolchain-beta-date = assertEq (fromRustupToolchain { channel = "beta-2021-01-01"; }) beta."2021-01-01".rust;
         rustup-toolchain-customization = assertEq
           (fromRustupToolchain {
             channel = "1.48.0";

manifest.nix

@@ -42,7 +42,7 @@ let
 
   # Uncompress the compressed manifest to the original one
   # (not complete but has enough information to make up the toolchain).
-  uncompressManifest = nightly: version: {
+  uncompressManifest = channel: version: {
     v, # rustc version
     d, # date
     r, # rename index
@@ -61,9 +61,9 @@ let
             pkgNameStripped = removeSuffix "-preview" pkgName;
             targetTail = if targetIdx == "_" then "" else "-" + target;
             urlVersion =
-              if u != null then u             # Use specified url version if exists.
-              else if nightly then "nightly"  # Otherwise, for nightly channel, default to be "nightly".
-              else v;                         # For stable channel, default to be rustc version.
+              if u != null then u                     # Use specified url version if exists.
+              else if channel == "stable" then v      # For stable channel, default to be rustc version.
+              else channel;                           # Otherwise, for beta/nightly channel, default to be "beta"/"nightly".
           in {
             name = target;
             value = {
@@ -74,8 +74,8 @@ let
       }) (removeAttrs manifest ["v" "d" "r"]);
   };
 
-  uncompressManifestSet = nightly: set: let
-    ret = mapAttrs (uncompressManifest nightly) (removeAttrs set ["latest"]);
+  uncompressManifestSet = channel: set: let
+    ret = mapAttrs (uncompressManifest channel) (removeAttrs set ["latest"]);
   in ret // { latest = ret.${set.latest}; };
 
 in {
@@ -86,8 +86,9 @@ in {
 
     # For internal usage.
     manifests = {
-      stable = uncompressManifestSet false (import ./manifests/stable);
-      nightly = uncompressManifestSet true (import ./manifests/nightly);
+      stable  = uncompressManifestSet "stable"  (import ./manifests/stable);
+      beta    = uncompressManifestSet "beta"    (import ./manifests/beta);
+      nightly = uncompressManifestSet "nightly" (import ./manifests/nightly);
     };
   };
 }