buildBazelPackage: clear markers

Turns out markers are non-deterministic after all and even our patching still
doesn't solve this problem completely. For example (tensorflow deps, this is a
complete diff so actual dependencies don't differ):

30509c30509
< bc527ff00916b15caee38793bca8f294c748df4a256de55c5199281be0489e73  result/@bazel_skylib.marker
---
> 4e0303e815c78df1e43d4b88dfe65e73046e0c6157fb10aa9a4e8b910113cd9c  result/@bazel_skylib.marker
31045c31045
< fa13d04b2316214c3b4008b52546c2d5b633e006f6f019d597bb3f9745bacf7b  result/@bazel_toolchains.marker
---
> b36174bf5535e5157801b6de30c35ee03a03fe57766306393c3d65dd65cbebf4  result/@bazel_toolchains.marker
31144c31144
< b0ce4a3ac29ac22528336dd3a54b5b7af9ecc43bef2a2630713c1981a5cbbb51  result/@build_bazel_rules_swift.marker
---
> 7492528068ec4f8e7ace2ecf8f933ec4e1b2235bd7426ce6f70177919f1cd05e  result/@build_bazel_rules_swift.marker
36245c36245
< be2993536a8233d63251b664caf35b1e7cd57d194ab2a39a293876c232d6bbd0  result/@io_bazel_rules_closure.marker
---
> b6655cc3f2c78525e5a724d8a4e93b1e7f09f1e09fc817d231109e7f39103e88  result/@io_bazel_rules_closure.marker
36329c36329
< 087bc674c9509dfe157400d111db4a13eeb45fc76aeccd490cee9aad6771ecad  result/@io_bazel_rules_docker.marker
---
> f920ec07315ec71e800b05cd22b2a341c0a80807c6e335ee81739b13c532b422  result/@io_bazel_rules_docker.marker
79544d79543
< 85893a05a817036c61f6cd9f8247757baa1654f473c494ce4fc5253c2bbd2790  result/@platforms.marker

And here's an example of differences:

$ cat result-a/@bazel_skylib.marker
7dc7472d37424ba5ec6a5532765bc911
$MANAGED
cat result-b/@bazel_skylib.marker
a8f3f577798201157128e8e9934c4705
$MANAGED

Instead of trying to patch these markers further we now completely clear them.
Nix hacks for ignoring markers are restored and expanded so that we don't even
attempt to parse the marker.

pkgs/build-support/build-bazel-package/default.nix

@@ -60,13 +60,10 @@ in stdenv.mkDerivation (fBuildAttrs // {
       # Remove all built in external workspaces, Bazel will recreate them when building
       rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
       rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
-      rm -rf $bazelOut/external/{local_*,\@local_*}
+      rm -rf $bazelOut/external/{local_*,\@local_*.marker}
 
-      # Patching markers to make them deterministic
-      find $bazelOut/external -name '@*\.marker' -exec sed -i \
-        -e 's, -\?[0-9][0-9]*$, 1,' \
-        -e '/^ENV:TMP.*/d' \
-        '{}' \;
+      # Clear markers
+      find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
 
       # Remove all vcs files
       rm -rf $(find $bazelOut/external -type d -name .git)

pkgs/development/tools/build-managers/bazel/default.nix

@@ -14,8 +14,8 @@
 , runtimeShell
 # Downstream packages for tests
 , bazel-watcher
-# Always assume all markers valid (don't redownload dependencies).
-# Also, don't clean up environment variables.
+# Always assume all markers valid (this is needed because we remove markers; they are non-deterministic).
+# Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers).
 , enableNixHacks ? false
 , gcc-unwrapped
 , autoPatchelfHook

pkgs/development/tools/build-managers/bazel/nix-hacks.patch

@@ -1,7 +1,39 @@
-diff -Naur a/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java b/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java
---- a/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java	2019-06-12 20:39:37.538708196 -0700
-+++ b/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java	2019-06-12 20:44:18.863429602 -0700
-@@ -146,7 +146,6 @@
+diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
+index 8e772005cd..6ffa1c919c 100644
+--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
++++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
+@@ -432,25 +432,7 @@ public final class RepositoryDelegatorFunction implements SkyFunction {
+       String content;
+       try {
+         content = FileSystemUtils.readContent(markerPath, StandardCharsets.UTF_8);
+-        String markerRuleKey = readMarkerFile(content, markerData);
+-        boolean verified = false;
+-        if (Preconditions.checkNotNull(ruleKey).equals(markerRuleKey)
+-            && Objects.equals(
+-                markerData.get(MANAGED_DIRECTORIES_MARKER),
+-                this.markerData.get(MANAGED_DIRECTORIES_MARKER))) {
+-          verified = handler.verifyMarkerData(rule, markerData, env);
+-          if (env.valuesMissing()) {
+-            return null;
+-          }
+-        }
+-
+-        if (verified) {
+-          return new Fingerprint().addString(content).digestAndReset();
+-        } else {
+-          // So that we are in a consistent state if something happens while fetching the repository
+-          markerPath.delete();
+-          return null;
+-        }
++        return new Fingerprint().addString(content).digestAndReset();
+       } catch (IOException e) {
+         throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+       }
+diff --git a/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java b/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java
+index c282d57ab6..f9b0c08627 100644
+--- a/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java
++++ b/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java
+@@ -146,7 +146,6 @@ public class JavaSubprocessFactory implements SubprocessFactory {
      ProcessBuilder builder = new ProcessBuilder();
      builder.command(params.getArgv());
      if (params.getEnv() != null) {