nuget-to-nix: find sources deterministically

The source used to download a particular package still isn't
deterministic in nuget. Even worse, the hash of the package can vary
between sources. This makes nuget use the first enabled source
containing the package.

The order of the dependencies may be slightly different because it now
uses glob order of the lower-case package names and versions, instead of
sorting the output.

If the package actually downloaded was the first source that contains
the package, then it will be hashed from disk to avoid downloading it
again.

pkgs/build-support/dotnet/build-dotnet-module/default.nix

@@ -172,7 +172,7 @@ stdenvNoCC.mkDerivation (args // {
       writeShellScript "fetch-${pname}-deps" ''
         set -euo pipefail
 
-        export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}"
+        export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}"
 
         for arg in "$@"; do
             case "$arg" in
@@ -218,6 +218,8 @@ stdenvNoCC.mkDerivation (args // {
                 -p:Deterministic=true \
                 --packages "$tmp/nuget_pkgs" \
                 --runtime "$rid" \
+                --no-cache \
+                --force \
                 ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \
                 ${lib.optionalString (flags != []) (toString flags)}
         }

pkgs/build-support/dotnet/nuget-to-nix/default.nix

@@ -4,11 +4,12 @@
 , substituteAll
 , nix
 , coreutils
-, findutils
-, gnused
 , jq
+, yq
 , curl
 , gnugrep
+, gawk
+, dotnet-sdk
 }:
 
 runCommandLocal "nuget-to-nix" {
@@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" {
     binPath = lib.makeBinPath [
       nix
       coreutils
-      findutils
-      gnused
       jq
+      yq
       curl
       gnugrep
+      gawk
+      dotnet-sdk
     ];
   };
 

pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh

@@ -3,6 +3,8 @@
 set -euo pipefail
 
 export PATH="@binPath@"
+# used for glob ordering of package names
+export LC_ALL=C
 
 if [ $# -eq 0 ]; then
   >&2 echo "Usage: $0 <packages directory> [path to excluded package source] > deps.nix"
@@ -10,37 +12,73 @@ if [ $# -eq 0 ]; then
 fi
 
 pkgs=$1
-tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX)
-trap "rm -f ${tmpfile}" EXIT
+tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)")
+trap 'rm -r "$tmp"' EXIT
 excluded_source=$(realpath "${2:-$tmp/empty}")
 
-declare -A nuget_sources_cache
+export DOTNET_NOLOGO=1
+export DOTNET_CLI_TELEMETRY_OPTOUT=1
 
-echo "{ fetchNuGet }: ["
+mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }')
+
+declare -A base_addresses
 
-while read pkg_spec; do
-  { read pkg_name; read pkg_version; } < <(
-    # Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3`
-    sed -nE 's/.*<id>([^<]*).*/\1/p; s/.*<version>([^<+]*).*/\1/p' "$pkg_spec")
+for index in "${sources[@]}"; do
+  base_addresses[$index]=$(
+    curl --compressed --netrc -fsL "$index" | \
+      jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')
+done
 
-  if [[ -e "$excluded_source/${pkg_name}.$pkg_version".nupkg ]]; then
-    continue
-  fi
+echo "{ fetchNuGet }: ["
 
-  pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)"
+cd "$pkgs"
+for package in *; do
+  cd "$package"
+  for version in *; do
+    id=$(xq -r .package.metadata.id "$version/$package".nuspec)
 
-  pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")"
-  if [[ -d $pkg_src ]]; then
+    if [[ -e "$excluded_source/$id.$version".nupkg ]]; then
       continue
-  elif [[ $pkg_src != https://api.nuget.org/* ]]; then
-    pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}"
-    pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg"
-    echo "  (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile}
-  else
-    echo "  (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile}
-  fi
-done < <(find $1 -name '*.nuspec')
-
-LC_ALL=C sort --ignore-case ${tmpfile}
-
-echo "]"
+    fi
+
+    used_source="$(jq -r '.source' "$version"/.nupkg.metadata)"
+    for source in "${sources[@]}"; do
+      url="${base_addresses[$source]}$package/$version/$package.$version.nupkg"
+      if [[ "$source" == "$used_source" ]]; then
+        sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)"
+        found=true
+        break
+      else
+        if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then
+          # If multiple remote sources are enabled, nuget will try them all
+          # concurrently and use the one that responds first. We always use the
+          # first source that has the package.
+          echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2
+          found=true
+          break
+        else
+          if ! grep -q 'HTTP error 404' "$tmp/error"; then
+            cat "$tmp/error" 1>&2
+            exit 1
+          fi
+        fi
+      fi
+    done
+
+    if ! ${found-false}; then
+      echo "couldn't find $package $version" >&2
+      exit 1
+    fi
+
+    if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then
+      echo "  (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })"
+    else
+      echo "  (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })"
+    fi
+  done
+  cd ..
+done
+
+cat << EOL
+]
+EOL

pkgs/development/tools/continuous-integration/github-runner/default.nix

@@ -302,12 +302,12 @@ stdenv.mkDerivation rec {
   # Inspired by passthru.fetch-deps in pkgs/build-support/build-dotnet-module/default.nix
   passthru.createDepsFile = writeShellApplication {
     name = "create-deps-file";
-    runtimeInputs = [ dotnetSdk nuget-to-nix ];
+    runtimeInputs = [ dotnetSdk (nuget-to-nix.override { dotnet-sdk = dotnetSdk; }) ];
     text = ''
       # Disable telemetry data
       export DOTNET_CLI_TELEMETRY_OPTOUT=1
 
-      rundir=$(pwd)
+      deps_file="$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")"
 
       printf "\n* Setup workdir\n"
       workdir="$(mktemp -d /tmp/${pname}.XXX)"
@@ -324,8 +324,6 @@ stdenv.mkDerivation rec {
       dotnet restore src/ActionsRunner.sln --packages nuget_pkgs --no-cache --force --runtime "${rid}"
       '') (lib.attrValues runtimeIds)}
 
-      cd "$rundir"
-      deps_file=''${1-"/tmp/${pname}-deps.nix"}
       printf "\n* Make %s file\n" "$(basename "$deps_file")"
       nuget-to-nix "$workdir/nuget_pkgs" > "$deps_file"
       printf "\n* Dependency file writen to %s" "$deps_file"