nix: factor out common.nix

main
Bernardo Meurer 2 years ago
parent ad79594d1e
commit 1ba2428545
No known key found for this signature in database
GPG Key ID: F4C0D53B8D14C246
  1. 213
      pkgs/tools/package-management/nix/common.nix
  2. 258
      pkgs/tools/package-management/nix/default.nix
  3. 44
      pkgs/tools/package-management/nix/nix-perl.nix
  4. 0
      pkgs/tools/package-management/nix/patches/aws-sdk-cpp-TransferManager-ContentEncoding.patch
  5. 42
      pkgs/tools/package-management/nix/patches/boehmgc-coroutine-sp-fallback.patch
  6. 36
      pkgs/tools/package-management/nix/patches/install-nlohmann_json-headers.patch

@ -0,0 +1,213 @@
{ stdenv
, autoconf-archive
, autoreconfHook
, bash
, bison
, boehmgc
, boost
, brotli
, busybox-sandbox-shell
, bzip2
, callPackage
, coreutils
, curl
, editline
, flex
, gnutar
, gtest
, gzip
, jq
, lib
, libarchive
, libcpuid
, libsodium
, lowdown
, mdbook
, nlohmann_json
, openssl
, perl
, pkg-config
, Security
, sqlite
, util-linuxMinimal
, xz
, enableDocumentation ? lib.versionOlder version "2.4pre" || stdenv.hostPlatform == stdenv.buildPlatform
, enableStatic ? stdenv.hostPlatform.isStatic
, withAWS ? !enableStatic && (stdenv.isLinux || stdenv.isDarwin), aws-sdk-cpp
, withLibseccomp ? lib.meta.availableOn stdenv.hostPlatform libseccomp, libseccomp
, confDir
, stateDir
, storeDir
, version
, src
, suffix ? ""
, patches ? [ ]
}:
let
sh = busybox-sandbox-shell;
atLeast24 = lib.versionAtLeast version "2.4pre";
atLeast25 = lib.versionAtLeast version "2.5pre";
nix-aws-sdk = (aws-sdk-cpp.override {
apis = [ "s3" "transfer" ];
customMemoryManagement = false;
}).overrideDerivation (args: {
patches = (args.patches or [ ]) ++ [ ./patches/aws-sdk-cpp-TransferManager-ContentEncoding.patch ];
});
in
stdenv.mkDerivation {
pname = "nix";
inherit src patches;
version = "${version}${suffix}";
VERSION_SUFFIX = suffix;
outputs =
[ "out" "dev" ]
++ lib.optionals enableDocumentation [ "man" "doc" ];
hardeningEnable = lib.optionals (!stdenv.isDarwin) [ "pie" ];
nativeBuildInputs = [
pkg-config
] ++ lib.optionals atLeast24 [
autoconf-archive
autoreconfHook
bison
flex
jq
] ++ lib.optionals (atLeast24 && enableDocumentation) [
(lib.getBin lowdown)
mdbook
] ++ lib.optionals stdenv.isLinux [
util-linuxMinimal
];
buildInputs = [
boost
brotli
bzip2
curl
editline
libsodium
openssl
sqlite
xz
] ++ lib.optionals stdenv.isDarwin [
Security
] ++ lib.optionals atLeast24 [
gtest
libarchive
lowdown
] ++ lib.optionals (atLeast24 && stdenv.isx86_64) [
libcpuid
] ++ lib.optionals withLibseccomp [
libseccomp
] ++ lib.optionals withAWS [
nix-aws-sdk
];
propagatedBuildInputs = [ boehmgc ];
NIX_LDFLAGS = lib.optionals (!atLeast24) [
# https://github.com/NixOS/nix/commit/3e85c57a6cbf46d5f0fe8a89b368a43abd26daba
(lib.optionalString enableStatic "-lssl -lbrotlicommon -lssh2 -lz -lnghttp2 -lcrypto")
# https://github.com/NixOS/nix/commits/74b4737d8f0e1922ef5314a158271acf81cd79f8
(lib.optionalString (stdenv.hostPlatform.system == "armv5tel-linux" || stdenv.hostPlatform.system == "armv6l-linux") "-latomic")
];
preConfigure =
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
lib.optionalString (!enableStatic) ''
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
${lib.optionalString stdenv.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
'' +
# On all versions before c9f51e87057652db0013289a95deffba495b35e7, which
# removes config.nix entirely and is not present in 2.3.x, we need to
# patch around an issue where the Nix configure step pulls in the build
# system's bash and other utilities when cross-compiling.
lib.optionalString (stdenv.buildPlatform != stdenv.hostPlatform && !atLeast24) ''
mkdir tmp/
substitute corepkgs/config.nix.in tmp/config.nix.in \
--subst-var-by bash ${bash}/bin/bash \
--subst-var-by coreutils ${coreutils}/bin \
--subst-var-by bzip2 ${bzip2}/bin/bzip2 \
--subst-var-by gzip ${gzip}/bin/gzip \
--subst-var-by xz ${xz}/bin/xz \
--subst-var-by tar ${gnutar}/bin/tar \
--subst-var-by tr ${coreutils}/bin/tr
mv tmp/config.nix.in corepkgs/config.nix.in
'';
configureFlags = [
"--with-store-dir=${storeDir}"
"--localstatedir=${stateDir}"
"--sysconfdir=${confDir}"
"--enable-gc"
] ++ lib.optionals (!enableDocumentation) [
"--disable-doc-gen"
] ++ lib.optionals (!atLeast24) [
# option was removed in 2.4
"--disable-init-state"
] ++ lib.optionals stdenv.isLinux [
"--with-sandbox-shell=${sh}/bin/busybox"
] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform && stdenv.hostPlatform ? nix && stdenv.hostPlatform.nix ? system) [
"--with-system=${stdenv.hostPlatform.nix.system}"
] ++ lib.optionals (!withLibseccomp) [
# RISC-V support in progress https://github.com/seccomp/libseccomp/pull/50
"--disable-seccomp-sandboxing"
];
makeFlags = [
"profiledir=$(out)/etc/profile.d"
] ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "PRECOMPILE_HEADERS=0";
installFlags = [ "sysconfdir=$(out)/etc" ];
doInstallCheck = true;
# socket path becomes too long otherwise
preInstallCheck = lib.optionalString stdenv.isDarwin ''
export TMPDIR=$NIX_BUILD_TOP
''
# See https://github.com/NixOS/nix/issues/5687
+ lib.optionalString (atLeast25 && stdenv.isDarwin) ''
echo "exit 99" > tests/gc-non-blocking.sh
'';
separateDebugInfo = stdenv.isLinux && (atLeast24 -> !enableStatic);
enableParallelBuilding = true;
meta = with lib; {
description = "Powerful package manager that makes package management reliable and reproducible";
longDescription = ''
Nix is a powerful package manager for Linux and other Unix systems that
makes package management reliable and reproducible. It provides atomic
upgrades and rollbacks, side-by-side installation of multiple versions of
a package, multi-user package management and easy setup of build
environments.
'';
homepage = "https://nixos.org/";
license = licenses.lgpl2Plus;
maintainers = with maintainers; [ eelco lovesegfault ];
platforms = platforms.unix;
outputsToInstall = [ "out" ] ++ optional enableDocumentation "man";
};
passthru = {
inherit boehmgc;
perl-bindings = perl.pkgs.toPerlModule (callPackage ./nix-perl.nix { inherit version src; });
};
}

@ -1,241 +1,34 @@
{ lib, fetchurl, fetchFromGitHub, fetchpatch, callPackage
{ lib
, boehmgc
, callPackage
, fetchFromGitHub
, fetchurl
, Security
, storeDir ? "/nix/store"
, stateDir ? "/nix/var"
, confDir ? "/etc"
, boehmgc
, Security
}:
let
common =
{ lib, stdenv, perl, curl, bzip2, sqlite, openssl ? null, xz
, bash, coreutils, util-linuxMinimal, gzip, gnutar
, pkg-config, boehmgc, libsodium, brotli, boost, editline, nlohmann_json
, autoreconfHook, autoconf-archive, bison, flex
, jq, libarchive, libcpuid
, lowdown, mdbook
# Used by tests
, gtest
, busybox-sandbox-shell
, storeDir
, stateDir
, confDir
, withLibseccomp ? lib.meta.availableOn stdenv.hostPlatform libseccomp, libseccomp
, withAWS ? !enableStatic && (stdenv.isLinux || stdenv.isDarwin), aws-sdk-cpp
, enableStatic ? stdenv.hostPlatform.isStatic
, enableDocumentation ? lib.versionOlder version "2.4pre" ||
stdenv.hostPlatform == stdenv.buildPlatform
, pname, version, suffix ? "", src
, patches ? [ ]
}:
let
sh = busybox-sandbox-shell;
is24 = lib.versionAtLeast version "2.4pre";
is25 = lib.versionAtLeast version "2.5pre";
nix = stdenv.mkDerivation {
inherit pname version src patches;
VERSION_SUFFIX = suffix;
outputs =
[ "out" "dev" ]
++ lib.optionals enableDocumentation [ "man" "doc" ];
hardeningEnable = lib.optionals (!stdenv.isDarwin) [ "pie" ];
nativeBuildInputs =
[ pkg-config ]
++ lib.optionals stdenv.isLinux [ util-linuxMinimal ]
++ lib.optionals (is24 && enableDocumentation) [
(lib.getBin lowdown) mdbook
]
++ lib.optionals is24
[ autoreconfHook
autoconf-archive
bison flex
jq
];
buildInputs =
[ curl libsodium openssl sqlite xz bzip2
brotli boost editline
]
++ lib.optionals stdenv.isDarwin [ Security ]
++ lib.optionals is24 [ libarchive gtest lowdown ]
++ lib.optional (is24 && stdenv.isx86_64) libcpuid
++ lib.optional withLibseccomp libseccomp
++ lib.optional withAWS
((aws-sdk-cpp.override {
apis = ["s3" "transfer"];
customMemoryManagement = false;
}).overrideDerivation (args: {
patches = args.patches or [] ++ [
./aws-sdk-cpp-TransferManager-ContentEncoding.patch
];
}));
propagatedBuildInputs = [ boehmgc ];
NIX_LDFLAGS = lib.optionals (!is24) [
# https://github.com/NixOS/nix/commit/3e85c57a6cbf46d5f0fe8a89b368a43abd26daba
(lib.optionalString enableStatic "-lssl -lbrotlicommon -lssh2 -lz -lnghttp2 -lcrypto")
# https://github.com/NixOS/nix/commits/74b4737d8f0e1922ef5314a158271acf81cd79f8
(lib.optionalString (stdenv.hostPlatform.system == "armv5tel-linux" || stdenv.hostPlatform.system == "armv6l-linux") "-latomic")
];
preConfigure =
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
lib.optionalString (!enableStatic) ''
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
${lib.optionalString stdenv.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
'' +
# On all versions before c9f51e87057652db0013289a95deffba495b35e7, which
# removes config.nix entirely and is not present in 2.3.x, we need to
# patch around an issue where the Nix configure step pulls in the build
# system's bash and other utilities when cross-compiling.
lib.optionalString (
stdenv.buildPlatform != stdenv.hostPlatform && !is24
) ''
mkdir tmp/
substitute corepkgs/config.nix.in tmp/config.nix.in \
--subst-var-by bash ${bash}/bin/bash \
--subst-var-by coreutils ${coreutils}/bin \
--subst-var-by bzip2 ${bzip2}/bin/bzip2 \
--subst-var-by gzip ${gzip}/bin/gzip \
--subst-var-by xz ${xz}/bin/xz \
--subst-var-by tar ${gnutar}/bin/tar \
--subst-var-by tr ${coreutils}/bin/tr
mv tmp/config.nix.in corepkgs/config.nix.in
'';
configureFlags =
[ "--with-store-dir=${storeDir}"
"--localstatedir=${stateDir}"
"--sysconfdir=${confDir}"
"--enable-gc"
]
++ lib.optional (!enableDocumentation) "--disable-doc-gen"
++ lib.optionals (!is24) [
# option was removed in 2.4
"--disable-init-state"
]
++ lib.optionals stdenv.isLinux [
"--with-sandbox-shell=${sh}/bin/busybox"
]
++ lib.optional (
stdenv.hostPlatform != stdenv.buildPlatform && stdenv.hostPlatform ? nix && stdenv.hostPlatform.nix ? system
) "--with-system=${stdenv.hostPlatform.nix.system}"
# RISC-V support in progress https://github.com/seccomp/libseccomp/pull/50
++ lib.optional (!withLibseccomp) "--disable-seccomp-sandboxing";
makeFlags = [ "profiledir=$(out)/etc/profile.d" ]
++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "PRECOMPILE_HEADERS=0";
installFlags = [ "sysconfdir=$(out)/etc" ];
doInstallCheck = true; # not cross
# socket path becomes too long otherwise
preInstallCheck = lib.optionalString stdenv.isDarwin ''
export TMPDIR=$NIX_BUILD_TOP
''
# See https://github.com/NixOS/nix/issues/5687
+ lib.optionalString (is25 && stdenv.isDarwin) ''
echo "exit 99" > tests/gc-non-blocking.sh
'';
separateDebugInfo = stdenv.isLinux && (is24 -> !enableStatic);
enableParallelBuilding = true;
meta = with lib; {
description = "Powerful package manager that makes package management reliable and reproducible";
longDescription = ''
Nix is a powerful package manager for Linux and other Unix systems that
makes package management reliable and reproducible. It provides atomic
upgrades and rollbacks, side-by-side installation of multiple versions of
a package, multi-user package management and easy setup of build
environments.
'';
homepage = "https://nixos.org/";
license = licenses.lgpl2Plus;
maintainers = with maintainers; [ eelco lovesegfault ];
platforms = platforms.unix;
outputsToInstall = [ "out" ] ++ optional enableDocumentation "man";
};
passthru = {
is24 = lib.warn ''nix package: attribute .is24 is deprecated. Please use lib.versionAtLeast X.version "2.4pre".'' is24;
is25 = lib.warn ''nix package: attribute .is25 is deprecated. Please use lib.versionAtLeast X.version "2.5pre".'' is25;
perl-bindings = perl.pkgs.toPerlModule (stdenv.mkDerivation {
pname = "nix-perl";
inherit version;
inherit src;
postUnpack = "sourceRoot=$sourceRoot/perl";
# This is not cross-compile safe, don't have time to fix right now
# but noting for future travellers.
nativeBuildInputs =
[ perl pkg-config curl nix libsodium boost autoreconfHook autoconf-archive nlohmann_json ];
configureFlags =
[ "--with-dbi=${perl.pkgs.DBI}/${perl.libPrefix}"
"--with-dbd-sqlite=${perl.pkgs.DBDSQLite}/${perl.libPrefix}"
];
preConfigure = "export NIX_STATE_DIR=$TMPDIR";
preBuild = "unset NIX_INDENT_MAKE";
});
inherit boehmgc;
};
};
in nix;
boehmgc_nix_2_3 = boehmgc.override {
enableLargeConfig = true;
};
boehmgc_nix_2_3 = boehmgc.override { enableLargeConfig = true; };
boehmgc_nix = boehmgc_nix_2_3.overrideAttrs (drv: {
patches = (drv.patches or []) ++ [
# Part of the GC solution in https://github.com/NixOS/nix/pull/4944
(fetchpatch {
url = "https://github.com/hercules-ci/nix/raw/5c58d84a76d96f269e3ff1e72c9c9ba5f68576af/boehmgc-coroutine-sp-fallback.diff";
sha256 = "sha256-JvnWVTlkltmQUs/0qApv/LPZ690UX1/2hEP+LYRwKbI=";
})
];
# Part of the GC solution in https://github.com/NixOS/nix/pull/4944
patches = (drv.patches or [ ]) ++ [ ./patches/boehmgc-coroutine-sp-fallback.patch ];
});
# master: https://github.com/NixOS/nix/pull/5536
# 2.4: https://github.com/NixOS/nix/pull/5537
installNlohmannJsonPatch = fetchpatch {
url = "https://github.com/NixOS/nix/pull/5536.diff";
sha256 = "sha256-SPnam4xNIjbMgnq6IP1AaM1V62X0yZNo4DEVmI8sHOo=";
};
buildNix =
{ version, suffix ? ""
, src ? null, sha256 ? null
, boehmgc ? boehmgc_nix, patches ? [ ]
}:
assert (src == null) -> (sha256 != null);
assert (sha256 == null) -> (src != null);
callPackage common {
pname = "nix";
version = "${version}${suffix}";
inherit suffix;
{ version
, suffix ? ""
, src ? null
, sha256 ? null
, boehmgc ? boehmgc_nix
, patches ? [ ]
}:
assert (src == null) -> (sha256 != null);
assert (sha256 == null) -> (src != null);
callPackage ./common.nix {
inherit version suffix;
src =
if src != null
@ -247,10 +40,9 @@ common =
inherit sha256;
};
inherit boehmgc patches;
inherit boehmgc patches Security;
inherit storeDir stateDir confDir;
};
in rec {
nix = nixStable;
@ -268,13 +60,15 @@ in rec {
nix_2_4 = buildNix {
version = "2.4";
sha256 = "sha256-op48CCDgLHK0qV1Batz4Ln5FqBiRjlE6qHTiZgt3b6k=";
patches = [ installNlohmannJsonPatch ];
# https://github.com/NixOS/nix/pull/5537
patches = [ ./patches/install-nlohmann_json-headers.patch ];
};
nix_2_5 = buildNix {
version = "2.5.1";
sha256 = "sha256-GOsiqy9EaTwDn2PLZ4eFj1VkXcBUbqrqHehRE9GuGdU=";
patches = [ installNlohmannJsonPatch ];
# https://github.com/NixOS/nix/pull/5536
patches = [ ./patches/install-nlohmann_json-headers.patch ];
};
nix_2_6 = buildNix {

@ -0,0 +1,44 @@
{ stdenv
, perl
, pkg-config
, curl
, nix
, libsodium
, boost
, autoreconfHook
, autoconf-archive
, nlohmann_json
, version
, src
}:
stdenv.mkDerivation {
pname = "nix-perl";
inherit version src;
postUnpack = "sourceRoot=$sourceRoot/perl";
# This is not cross-compile safe, don't have time to fix right now
# but noting for future travellers.
nativeBuildInputs = [
autoconf-archive
autoreconfHook
boost
curl
libsodium
nix
nlohmann_json
perl
pkg-config
];
configureFlags = [
"--with-dbi=${perl.pkgs.DBI}/${perl.libPrefix}"
"--with-dbd-sqlite=${perl.pkgs.DBDSQLite}/${perl.libPrefix}"
];
preConfigure = "export NIX_STATE_DIR=$TMPDIR";
preBuild = "unset NIX_INDENT_MAKE";
}

@ -0,0 +1,42 @@
diff --git a/pthread_stop_world.c b/pthread_stop_world.c
index 1cee6a0b..46c3acd9 100644
--- a/pthread_stop_world.c
+++ b/pthread_stop_world.c
@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
struct GC_traced_stack_sect_s *traced_stack_sect;
pthread_t self = pthread_self();
word total_size = 0;
+ size_t stack_limit;
+ pthread_attr_t pattr;
if (!EXPECT(GC_thr_initialized, TRUE))
GC_thr_init();
@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
hi = p->altstack + p->altstack_size;
/* FIXME: Need to scan the normal stack too, but how ? */
/* FIXME: Assume stack grows down */
+ } else {
+ if (pthread_getattr_np(p->id, &pattr)) {
+ ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
+ }
+ if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
+ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
+ }
+ // When a thread goes into a coroutine, we lose its original sp until
+ // control flow returns to the thread.
+ // While in the coroutine, the sp points outside the thread stack,
+ // so we can detect this and push the entire thread stack instead,
+ // as an approximation.
+ // We assume that the coroutine has similarly added its entire stack.
+ // This could be made accurate by cooperating with the application
+ // via new functions and/or callbacks.
+ #ifndef STACK_GROWS_UP
+ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
+ lo = hi - stack_limit;
+ }
+ #else
+ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
+ #endif
}
GC_push_all_stack_sections(lo, hi, traced_stack_sect);
# ifdef STACK_GROWS_UP

@ -0,0 +1,36 @@
From 3884f7a69a57d8ecfcbcaae476ec2ff53ffbd549 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 11 Nov 2021 11:03:21 +0100
Subject: [PATCH] Install nlohmann_json headers
These headers are included by the libexpr, libfetchers, libstore
and libutil headers.
Considering that these are vendored sources, Nix should expose them,
as it is not a good idea for reverse dependencies to rely on a
potentially different source that can go out of sync.
---
Makefile | 1 +
src/nlohmann/local.mk | 2 ++
2 files changed, 3 insertions(+)
create mode 100644 src/nlohmann/local.mk
diff --git a/Makefile b/Makefile
index 5040d288485..e6ce50cbdb7 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,7 @@ makefiles = \
src/libexpr/local.mk \
src/libcmd/local.mk \
src/nix/local.mk \
+ src/nlohmann/local.mk \
src/resolve-system-dependencies/local.mk \
scripts/local.mk \
misc/bash/local.mk \
diff --git a/src/nlohmann/local.mk b/src/nlohmann/local.mk
new file mode 100644
index 00000000000..63c427e000e
--- /dev/null
+++ b/src/nlohmann/local.mk
@@ -0,0 +1,2 @@
+$(foreach i, $(wildcard src/nlohmann/*.hpp), \
+ $(eval $(call install-file-in, $(i), $(includedir)/nlohmann, 0644)))
Loading…
Cancel
Save