From b739da06931e6df03ac18f6d85be40d301042969 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec Date: Mon, 3 Oct 2022 12:43:33 +0200 Subject: [PATCH 1/2] buildBazelPackage: optionally run bazel tests in checkPhase Tests from the bazelTestTargets argument will be run before the build. The new bazelTestFlags argument can be used to pass additional flags to this phase. --- .../build-bazel-package/default.nix | 120 +++++++++++------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index d69fddaf03f..14f7ac38d3f 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -13,35 +13,37 @@ args@{ , bazel ? bazelPkg , bazelFlags ? [] , bazelBuildFlags ? [] +, bazelTestFlags ? [] , bazelFetchFlags ? [] , bazelTarget +, bazelTestTargets ? [] , buildAttrs , fetchAttrs -# Newer versions of Bazel are moving away from built-in rules_cc and instead -# allow fetching it as an external dependency in a WORKSPACE file[1]. If -# removed in the fixed-output fetch phase, building will fail to download it. -# This can be seen e.g. in #73097 -# -# This option allows configuring the removal of rules_cc in cases where a -# project depends on it via an external dependency. -# -# [1]: https://github.com/bazelbuild/rules_cc + # Newer versions of Bazel are moving away from built-in rules_cc and instead + # allow fetching it as an external dependency in a WORKSPACE file[1]. If + # removed in the fixed-output fetch phase, building will fail to download it. + # This can be seen e.g. in #73097 + # + # This option allows configuring the removal of rules_cc in cases where a + # project depends on it via an external dependency. + # + # [1]: https://github.com/bazelbuild/rules_cc , removeRulesCC ? true , removeLocalConfigCc ? true , removeLocal ? true -# Use build --nobuild instead of fetch. This allows fetching the dependencies -# required for the build as configured, rather than fetching all the dependencies -# which may not work in some situations (e.g. Java code which ends up relying on -# Debian-specific /usr/share/java paths, but doesn't in the configured build). + # Use build --nobuild instead of fetch. This allows fetching the dependencies + # required for the build as configured, rather than fetching all the dependencies + # which may not work in some situations (e.g. Java code which ends up relying on + # Debian-specific /usr/share/java paths, but doesn't in the configured build). , fetchConfigured ? true -# Don’t add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE / -# NIX_LDFLAGS. This is necessary when using a custom toolchain which -# Bazel wants all headers / libraries to come from, like when using -# CROSSTOOL. Weirdly, we can still get the flags through the wrapped -# compiler. + # Don’t add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE / + # NIX_LDFLAGS. This is necessary when using a custom toolchain which + # Bazel wants all headers / libraries to come from, like when using + # CROSSTOOL. Weirdly, we can still get the flags through the wrapped + # compiler. , dontAddBazelOpts ? false , ... }: @@ -50,13 +52,33 @@ let fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" "removeRulesCC" ]; fBuildAttrs = fArgs // buildAttrs; fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ]; - -in stdenv.mkDerivation (fBuildAttrs // { - inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget; + bazelCmd = { cmd, additionalFlags, targets }: + lib.optionalString (targets != [ ]) '' + # See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables] + BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \ + USER=homeless-shelter \ + bazel \ + --batch \ + --output_base="$bazelOut" \ + --output_user_root="$bazelUserRoot" \ + ${cmd} \ + --curses=no \ + -j $NIX_BUILD_CORES \ + "''${copts[@]}" \ + "''${host_copts[@]}" \ + "''${linkopts[@]}" \ + "''${host_linkopts[@]}" \ + $bazelFlags \ + ${lib.strings.concatStringsSep " " additionalFlags} \ + ${lib.strings.concatStringsSep " " targets} + ''; +in +stdenv.mkDerivation (fBuildAttrs // { + inherit name bazelFlags bazelBuildFlags bazelTestFlags bazelFetchFlags bazelTarget bazelTestTargets; deps = stdenv.mkDerivation (fFetchAttrs // { name = "${name}-deps.tar.gz"; - inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget; + inherit bazelFlags bazelBuildFlags bazelTestFlags bazelFetchFlags bazelTarget bazelTestTargets; impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ fFetchAttrs.impureEnvVars or []; @@ -77,14 +99,7 @@ in stdenv.mkDerivation (fBuildAttrs // { buildPhase = fFetchAttrs.buildPhase or '' runHook preBuild - # Bazel computes the default value of output_user_root before parsing the - # flag. The computation of the default value involves getting the $USER - # from the environment. I don't have that variable when building with - # sandbox enabled. Code here - # https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124 - # - # On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN - + # See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]. # We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads: # https://github.com/bazelbuild/bazel/issues/6502 BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \ @@ -97,7 +112,8 @@ in stdenv.mkDerivation (fBuildAttrs // { --loading_phase_threads=1 \ $bazelFlags \ $bazelFetchFlags \ - $bazelTarget + ${bazelTarget} \ + ${lib.strings.concatStringsSep " " bazelTestTargets} runHook postBuild ''; @@ -189,7 +205,7 @@ in stdenv.mkDerivation (fBuildAttrs // { # the wrappers are expecting will not be set. So instead of relying on the # wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt` # and related flags. - # + copts=() host_copts=() linkopts=() @@ -209,23 +225,29 @@ in stdenv.mkDerivation (fBuildAttrs // { done fi - BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \ - USER=homeless-shelter \ - bazel \ - --batch \ - --output_base="$bazelOut" \ - --output_user_root="$bazelUserRoot" \ - build \ - --curses=no \ - -j $NIX_BUILD_CORES \ - "''${copts[@]}" \ - "''${host_copts[@]}" \ - "''${linkopts[@]}" \ - "''${host_linkopts[@]}" \ - $bazelFlags \ - $bazelBuildFlags \ - $bazelTarget - + ${ + bazelCmd { + cmd = "test"; + additionalFlags = + ["--test_output=errors"] ++ bazelTestFlags; + targets = bazelTestTargets; + } + } + ${ + bazelCmd { + cmd = "build"; + additionalFlags = bazelBuildFlags; + targets = [bazelTarget]; + } + } runHook postBuild ''; }) + +# [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]: +# Bazel computes the default value of output_user_root before parsing the +# flag. The computation of the default value involves getting the $USER +# from the environment. Code here : +# https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124 +# +# On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN. From 6b1a33bede0e96f19b73e64de6dc4b5cbf400771 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec Date: Mon, 3 Oct 2022 12:44:12 +0200 Subject: [PATCH 2/2] vertible: run tests before building in buildBazelPackage --- pkgs/development/tools/verible/default.nix | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pkgs/development/tools/verible/default.nix b/pkgs/development/tools/verible/default.nix index 6763638cab7..dc2d015a46f 100644 --- a/pkgs/development/tools/verible/default.nix +++ b/pkgs/development/tools/verible/default.nix @@ -31,11 +31,15 @@ buildBazelPackage rec { ./remove-unused-deps.patch ]; - bazelFlags = [ "--//bazel:use_local_flex_bison" ]; + bazelFlags = [ + "--//bazel:use_local_flex_bison" + "--javabase=@bazel_tools//tools/jdk:remote_jdk11" + "--host_javabase=@bazel_tools//tools/jdk:remote_jdk11" + ]; fetchAttrs = { # Fixed output derivation hash after bazel fetch - sha256 = "sha256-XoLdlEeoDJlyWlnXZADHOKu06zKHgHJfgey8UhOt+LM="; + sha256 = "sha256-45PINJ7VtL5Jl/nAQNkiSCt8wUwtytNfgeNMZaz3Y9U="; }; nativeBuildInputs = [ @@ -45,14 +49,23 @@ buildBazelPackage rec { ]; postPatch = '' - patchShebangs bazel/build-version.py \ - common/util/create_version_header.sh \ + patchShebangs\ + bazel/build-version.py \ + bazel/sh_test_with_runfiles_lib.sh \ + common/lsp/dummy-ls_test.sh \ common/parser/move_yacc_stack_symbols.sh \ - common/parser/record_syntax_error.sh + common/parser/record_syntax_error.sh \ + common/tools/patch_tool_test.sh \ + common/tools/verible-transform-interactive.sh \ + common/tools/verible-transform-interactive-test.sh \ + common/util/create_version_header.sh \ + kythe-browse.sh \ + verilog/tools ''; removeRulesCC = false; bazelTarget = ":install-binaries"; + bazelTestTargets = [ "//..." ]; bazelBuildFlags = [ "-c opt" ];