Upgrades Hydra to the latest master/flake branch. To perform this upgrade, it's needed to do a non-trivial db-migration which provides a massive performance-improvement[1]. The basic ideas behind multi-step upgrades of services between NixOS versions have been gathered already[2]. For further context it's recommended to read this first. Basically, the following steps are needed: * Upgrade to a non-breaking version of Hydra with the db-changes (columns are still nullable here). If `system.stateVersion` is set to something older than 20.03, the package will be selected automatically, otherwise `pkgs.hydra-migration` needs to be used. * Run `hydra-backfill-ids` on the server. * Deploy either `pkgs.hydra-unstable` (for Hydra master) or `pkgs.hydra-flakes` (for flakes-support) to activate the optimization. The steps are also documented in the release-notes and in the module using `warnings`. `pkgs.hydra` has been removed as latest Hydra doesn't compile with `pkgs.nixStable` and to ensure a graceful migration using the newly introduced packages. To verify the approach, a simple vm-test has been added which verifies the migration steps. [1] https://github.com/NixOS/hydra/pull/711 [2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471wip/yesman
Maximilian Bosch
4 years ago
parent
ea5c800175
commit
bd5324c4fc
@ -0,0 +1,47 @@ |
||||
{ system, ... }: |
||||
{ |
||||
baseConfig = { pkgs, ... }: let |
||||
trivialJob = pkgs.writeTextDir "trivial.nix" '' |
||||
{ trivial = builtins.derivation { |
||||
name = "trivial"; |
||||
system = "${system}"; |
||||
builder = "/bin/sh"; |
||||
allowSubstitutes = false; |
||||
preferLocalBuild = true; |
||||
args = ["-c" "echo success > $out; exit 0"]; |
||||
}; |
||||
} |
||||
''; |
||||
|
||||
createTrivialProject = pkgs.stdenv.mkDerivation { |
||||
name = "create-trivial-project"; |
||||
dontUnpack = true; |
||||
buildInputs = [ pkgs.makeWrapper ]; |
||||
installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh"; |
||||
postFixup = '' |
||||
wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob} |
||||
''; |
||||
}; |
||||
in { |
||||
virtualisation.memorySize = 2048; |
||||
time.timeZone = "UTC"; |
||||
environment.systemPackages = [ createTrivialProject pkgs.jq ]; |
||||
services.hydra = { |
||||
enable = true; |
||||
# Hydra needs those settings to start up, so we add something not harmfull. |
||||
hydraURL = "example.com"; |
||||
notificationSender = "example@example.com"; |
||||
extraConfig = '' |
||||
email_notification = 1 |
||||
''; |
||||
}; |
||||
services.postfix.enable = true; |
||||
nix = { |
||||
buildMachines = [{ |
||||
hostName = "localhost"; |
||||
systems = [ system ]; |
||||
}]; |
||||
binaryCaches = []; |
||||
}; |
||||
}; |
||||
} |
||||
@ -0,0 +1,86 @@ |
||||
{ system ? builtins.currentSystem, ... }: |
||||
|
||||
let inherit (import ./common.nix { inherit system; }) baseConfig; in |
||||
|
||||
{ mig = import ../make-test-python.nix ({ pkgs, lib, ... }: { |
||||
name = "hydra-db-migration"; |
||||
meta = with pkgs.stdenv.lib.maintainers; { |
||||
maintainers = [ ma27 ]; |
||||
}; |
||||
|
||||
nodes = { |
||||
original = { pkgs, lib, ... }: { |
||||
imports = [ baseConfig ]; |
||||
|
||||
# An older version of Hydra before the db change |
||||
# for testing purposes. |
||||
services.hydra.package = pkgs.hydra-migration.overrideAttrs (old: { |
||||
inherit (old) pname; |
||||
version = "2020-02-06"; |
||||
src = pkgs.fetchFromGitHub { |
||||
owner = "NixOS"; |
||||
repo = "hydra"; |
||||
rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc"; |
||||
sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick"; |
||||
}; |
||||
}); |
||||
}; |
||||
|
||||
migration_phase1 = { pkgs, lib, ... }: { |
||||
imports = [ baseConfig ]; |
||||
services.hydra.package = pkgs.hydra-migration; |
||||
}; |
||||
|
||||
finished = { pkgs, lib, ... }: { |
||||
imports = [ baseConfig ]; |
||||
services.hydra.package = pkgs.hydra-unstable; |
||||
}; |
||||
}; |
||||
|
||||
testScript = { nodes, ... }: let |
||||
next = nodes.migration_phase1.config.system.build.toplevel; |
||||
finished = nodes.finished.config.system.build.toplevel; |
||||
in '' |
||||
original.start() |
||||
original.wait_for_unit("multi-user.target") |
||||
original.wait_for_unit("postgresql.service") |
||||
original.wait_for_unit("hydra-init.service") |
||||
original.require_unit_state("hydra-queue-runner.service") |
||||
original.require_unit_state("hydra-evaluator.service") |
||||
original.require_unit_state("hydra-notify.service") |
||||
original.succeed("hydra-create-user admin --role admin --password admin") |
||||
original.wait_for_open_port(3000) |
||||
original.succeed("create-trivial-project.sh") |
||||
original.wait_until_succeeds( |
||||
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq' |
||||
) |
||||
|
||||
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'") |
||||
assert "jobset_id" not in out |
||||
|
||||
original.succeed( |
||||
"${next}/bin/switch-to-configuration test >&2" |
||||
) |
||||
original.wait_for_unit("hydra-init.service") |
||||
|
||||
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'") |
||||
assert "jobset_id|integer|||" in out |
||||
|
||||
original.succeed("hydra-backfill-ids") |
||||
|
||||
original.succeed( |
||||
"${finished}/bin/switch-to-configuration test >&2" |
||||
) |
||||
original.wait_for_unit("hydra-init.service") |
||||
|
||||
out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'") |
||||
assert "jobset_id|integer||not null|" in out |
||||
|
||||
original.wait_until_succeeds( |
||||
'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" | jq .buildstatus | xargs test 0 -eq' |
||||
) |
||||
|
||||
original.shutdown() |
||||
''; |
||||
}); |
||||
} |
||||
@ -0,0 +1,135 @@ |
||||
{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub |
||||
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx |
||||
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt |
||||
, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json |
||||
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar |
||||
, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook, src ? null, version ? null |
||||
, migration ? false |
||||
}: |
||||
|
||||
with stdenv; |
||||
|
||||
if lib.versions.major nix.version == "1" |
||||
then throw "This Hydra version doesn't support Nix 1.x" |
||||
else |
||||
|
||||
let |
||||
perlDeps = buildEnv { |
||||
name = "hydra-perl-deps"; |
||||
paths = with perlPackages; lib.closePropagation |
||||
[ ModulePluggable |
||||
CatalystActionREST |
||||
CatalystAuthenticationStoreDBIxClass |
||||
CatalystDevel |
||||
CatalystDispatchTypeRegex |
||||
CatalystPluginAccessLog |
||||
CatalystPluginAuthorizationRoles |
||||
CatalystPluginCaptcha |
||||
CatalystPluginSessionStateCookie |
||||
CatalystPluginSessionStoreFastMmap |
||||
CatalystPluginStackTrace |
||||
CatalystPluginUnicodeEncoding |
||||
CatalystTraitForRequestProxyBase |
||||
CatalystViewDownload |
||||
CatalystViewJSON |
||||
CatalystViewTT |
||||
CatalystXScriptServerStarman |
||||
CatalystXRoleApplicator |
||||
CryptRandPasswd |
||||
DBDPg |
||||
DBDSQLite |
||||
DataDump |
||||
DateTime |
||||
DigestSHA1 |
||||
EmailMIME |
||||
EmailSender |
||||
FileSlurp |
||||
IOCompress |
||||
IPCRun |
||||
JSON |
||||
JSONAny |
||||
JSONXS |
||||
LWP |
||||
LWPProtocolHttps |
||||
NetAmazonS3 |
||||
NetPrometheus |
||||
NetStatsd |
||||
PadWalker |
||||
Readonly |
||||
SQLSplitStatement |
||||
SetScalar |
||||
Starman |
||||
SysHostnameLong |
||||
TermSizeAny |
||||
TestMore |
||||
TextDiff |
||||
TextTable |
||||
XMLSimple |
||||
nix |
||||
nix.perl-bindings |
||||
git |
||||
boehmgc |
||||
]; |
||||
}; |
||||
in stdenv.mkDerivation rec { |
||||
pname = "hydra"; |
||||
|
||||
inherit stdenv src version; |
||||
|
||||
buildInputs = |
||||
[ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx |
||||
gitAndTools.top-git mercurial /*darcs*/ subversion bazaar openssl bzip2 libxslt |
||||
perlDeps perl nix |
||||
postgresql # for running the tests |
||||
nlohmann_json |
||||
boost |
||||
]; |
||||
|
||||
hydraPath = lib.makeBinPath ( |
||||
[ sqlite subversion openssh nix coreutils findutils pixz |
||||
gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial /*darcs*/ gnused bazaar |
||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); |
||||
|
||||
nativeBuildInputs = [ autoreconfHook pkgconfig ]; |
||||
|
||||
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; |
||||
|
||||
NIX_CFLAGS_COMPILE = "-pthread"; |
||||
|
||||
shellHook = '' |
||||
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH |
||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB; |
||||
''; |
||||
|
||||
enableParallelBuilding = true; |
||||
|
||||
preCheck = '' |
||||
patchShebangs . |
||||
export LOGNAME=''${LOGNAME:-foo} |
||||
''; |
||||
|
||||
postInstall = '' |
||||
mkdir -p $out/nix-support |
||||
for i in $out/bin/*; do |
||||
read -n 4 chars < $i |
||||
if [[ $chars =~ ELF ]]; then continue; fi |
||||
wrapProgram $i \ |
||||
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ |
||||
--prefix PATH ':' $out/bin:$hydraPath \ |
||||
--set HYDRA_RELEASE ${version} \ |
||||
--set HYDRA_HOME $out/libexec/hydra \ |
||||
--set NIX_RELEASE ${nix.name or "unknown"} |
||||
done |
||||
''; # */ |
||||
|
||||
dontStrip = true; |
||||
|
||||
passthru = { inherit perlDeps migration; }; |
||||
|
||||
meta = with stdenv.lib; { |
||||
description = "Nix-based continuous build system"; |
||||
license = licenses.gpl3; |
||||
platforms = platforms.linux; |
||||
maintainers = with maintainers; [ ma27 ]; |
||||
}; |
||||
} |
||||
@ -1,143 +1,42 @@ |
||||
{ stdenv, nix, perlPackages, buildEnv, fetchFromGitHub |
||||
, makeWrapper, autoconf, automake, libtool, unzip, pkgconfig, sqlite, libpqxx |
||||
, gitAndTools, mercurial, darcs, subversion, bazaar, openssl, bzip2, libxslt |
||||
, guile, perl, postgresql, nukeReferences, git, boehmgc, nlohmann_json |
||||
, docbook_xsl, openssh, gnused, coreutils, findutils, gzip, lzma, gnutar |
||||
, rpm, dpkg, cdrkit, pixz, lib, boost, autoreconfHook |
||||
}: |
||||
|
||||
with stdenv; |
||||
|
||||
if lib.versions.major nix.version == "1" |
||||
then throw "This Hydra version doesn't support Nix 1.x" |
||||
else |
||||
|
||||
let |
||||
perlDeps = buildEnv { |
||||
name = "hydra-perl-deps"; |
||||
paths = with perlPackages; lib.closePropagation |
||||
[ ModulePluggable |
||||
CatalystActionREST |
||||
CatalystAuthenticationStoreDBIxClass |
||||
CatalystDevel |
||||
CatalystDispatchTypeRegex |
||||
CatalystPluginAccessLog |
||||
CatalystPluginAuthorizationRoles |
||||
CatalystPluginCaptcha |
||||
CatalystPluginSessionStateCookie |
||||
CatalystPluginSessionStoreFastMmap |
||||
CatalystPluginStackTrace |
||||
CatalystPluginUnicodeEncoding |
||||
CatalystTraitForRequestProxyBase |
||||
CatalystViewDownload |
||||
CatalystViewJSON |
||||
CatalystViewTT |
||||
CatalystXScriptServerStarman |
||||
CatalystXRoleApplicator |
||||
CryptRandPasswd |
||||
DBDPg |
||||
DBDSQLite |
||||
DataDump |
||||
DateTime |
||||
DigestSHA1 |
||||
EmailMIME |
||||
EmailSender |
||||
FileSlurp |
||||
IOCompress |
||||
IPCRun |
||||
JSON |
||||
JSONAny |
||||
JSONXS |
||||
LWP |
||||
LWPProtocolHttps |
||||
NetAmazonS3 |
||||
NetPrometheus |
||||
NetStatsd |
||||
PadWalker |
||||
Readonly |
||||
SQLSplitStatement |
||||
SetScalar |
||||
Starman |
||||
SysHostnameLong |
||||
TermSizeAny |
||||
TestMore |
||||
TextDiff |
||||
TextTable |
||||
XMLSimple |
||||
nix |
||||
nix.perl-bindings |
||||
git |
||||
boehmgc |
||||
]; |
||||
{ fetchFromGitHub, nixStable, nixUnstable, callPackage, nixFlakes }: |
||||
|
||||
{ |
||||
# Package for phase-1 of the db migration for Hydra. |
||||
# https://github.com/NixOS/hydra/pull/711 |
||||
hydra-migration = callPackage ./common.nix { |
||||
version = "2020-02-10"; |
||||
src = fetchFromGitHub { |
||||
owner = "NixOS"; |
||||
repo = "hydra"; |
||||
rev = "add4f610ce6f206fb44702b5a894d877b3a30e3a"; |
||||
sha256 = "1d8hdgjx2ys0zmixi2ydmimdq7ml20h1ji4amwawcyw59kssh6l3"; |
||||
}; |
||||
nix = nixStable; |
||||
migration = true; |
||||
}; |
||||
in stdenv.mkDerivation rec { |
||||
pname = "hydra"; |
||||
version = "2020-02-06"; |
||||
|
||||
inherit stdenv; |
||||
|
||||
src = fetchFromGitHub { |
||||
owner = "NixOS"; |
||||
repo = pname; |
||||
rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc"; |
||||
sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick"; |
||||
# Hydra from latest master (or flakes) branch. Contains breaking changes, |
||||
# so when having an older version, `pkgs.hydra-migration` should be deployed first. |
||||
|
||||
hydra-unstable = callPackage ./common.nix { |
||||
version = "2020-03-24"; |
||||
src = fetchFromGitHub { |
||||
owner = "NixOS"; |
||||
repo = "hydra"; |
||||
rev = "12cc46cdb36321acd4c982429a86eb0f8f3cc969"; |
||||
sha256 = "10ipxzdxr47c8w5jg69mbax2ykc7lb5fs9bbdd3iai9wzyfz17ln"; |
||||
}; |
||||
nix = nixUnstable; |
||||
}; |
||||
|
||||
buildInputs = |
||||
[ makeWrapper autoconf automake libtool unzip nukeReferences sqlite libpqxx |
||||
gitAndTools.top-git mercurial darcs subversion bazaar openssl bzip2 libxslt |
||||
guile # optional, for Guile + Guix support |
||||
perlDeps perl nix |
||||
postgresql # for running the tests |
||||
nlohmann_json |
||||
boost |
||||
]; |
||||
|
||||
hydraPath = lib.makeBinPath ( |
||||
[ sqlite subversion openssh nix coreutils findutils pixz |
||||
gzip bzip2 lzma gnutar unzip git gitAndTools.top-git mercurial darcs gnused bazaar |
||||
] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); |
||||
|
||||
nativeBuildInputs = [ autoreconfHook pkgconfig ]; |
||||
|
||||
configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; |
||||
|
||||
NIX_CFLAGS_COMPILE = "-pthread"; |
||||
|
||||
shellHook = '' |
||||
PATH=$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$(pwd)/src/hydra-evaluator:$PATH |
||||
PERL5LIB=$(pwd)/src/lib:$PERL5LIB; |
||||
''; |
||||
|
||||
enableParallelBuilding = true; |
||||
|
||||
preCheck = '' |
||||
patchShebangs . |
||||
export LOGNAME=''${LOGNAME:-foo} |
||||
''; |
||||
|
||||
postInstall = '' |
||||
mkdir -p $out/nix-support |
||||
for i in $out/bin/*; do |
||||
read -n 4 chars < $i |
||||
if [[ $chars =~ ELF ]]; then continue; fi |
||||
wrapProgram $i \ |
||||
--prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ |
||||
--prefix PATH ':' $out/bin:$hydraPath \ |
||||
--set HYDRA_RELEASE ${version} \ |
||||
--set HYDRA_HOME $out/libexec/hydra \ |
||||
--set NIX_RELEASE ${nix.name or "unknown"} |
||||
done |
||||
''; # */ |
||||
|
||||
dontStrip = true; |
||||
|
||||
passthru.perlDeps = perlDeps; |
||||
|
||||
meta = with stdenv.lib; { |
||||
description = "Nix-based continuous build system"; |
||||
license = licenses.gpl3; |
||||
platforms = platforms.linux; |
||||
maintainers = with maintainers; [ ma27 ]; |
||||
hydra-flakes = callPackage ./common.nix { |
||||
version = "2020-03-27"; |
||||
src = fetchFromGitHub { |
||||
owner = "NixOS"; |
||||
repo = "hydra"; |
||||
rev = "a7540b141d085a7e78c21fda8e8c05907c659b34"; |
||||
sha256 = "08fs7593w5zs8vh4c66gvrxk6s840pp6hj8nwf51wsa27kg5a943"; |
||||
}; |
||||
nix = nixFlakes; |
||||
}; |
||||
} |
||||
|
||||
Loading…
Reference in new issue