Merge pull request #168932 from hercules-ci/init-postgresqlTestHook

`postgresqlTestHook`: init
main
Robert Hensing 2 years ago committed by GitHub
commit 33cf95ef36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 10
      doc/hooks/index.xml
  2. 59
      doc/hooks/postgresql-test-hook.section.md
  3. 1
      doc/manual.xml
  4. 7
      nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
  5. 2
      nixos/doc/manual/release-notes/rl-2205.section.md
  6. 9
      pkgs/build-support/setup-hooks/postgresql-test-hook/default.nix
  7. 79
      pkgs/build-support/setup-hooks/postgresql-test-hook/postgresql-test-hook.sh
  8. 27
      pkgs/build-support/setup-hooks/postgresql-test-hook/test.nix
  9. 54
      pkgs/development/haskell-modules/configuration-common.nix
  10. 2
      pkgs/top-level/all-packages.nix

@ -0,0 +1,10 @@
<chapter xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
xml:id="chap-hooks">
<title>Hooks reference</title>
<para>
Nixpkgs has several hook packages that augment the stdenv phases.
</para>
<xi:include href="./postgresql-test-hook.section.xml" />
</chapter>

@ -0,0 +1,59 @@
# `postgresqlTestHook` {#sec-postgresqlTestHook}
This hook starts a PostgreSQL server during the `checkPhase`. Example:
```nix
{ stdenv, postgresql, postgresqlTestHook }:
stdenv.mkDerivation {
# ...
checkInputs = [
postgresql
postgresqlTestHook
];
}
```
If you use a custom `checkPhase`, remember to add the `runHook` calls:
```nix
checkPhase ''
runHook preCheck
# ... your tests
runHook postCheck
''
```
## Variables {#sec-postgresqlTestHook-variables}
The hook logic will read a number of variables and set them to a default value if unset or empty.
Exported variables:
- `PGDATA`: location of server files.
- `PGHOST`: location of UNIX domain socket directory; the default `host` in a connection string.
- `PGUSER`: user to create / log in with, default: `test_user`.
- `PGDATABASE`: database name, default: `test_db`.
Bash-only variables:
- `postgresqlTestUserOptions`: SQL options to use when creating the `$PGUSER` role, default: `LOGIN`.
- `postgresqlTestSetupSQL`: SQL commands to run as database administrator after startup, default: statements that create `$PGUSER` and `$PGDATABASE`.
- `postgresqlTestSetupCommands`: bash commands to run after database start, defaults to running `$postgresqlTestSetupSQL` as database administrator.
- `postgresqlEnableTCP`: set to `1` to enable TCP listening. Flaky; not recommended.
- `postgresqlStartCommands`: defaults to `pg_ctl start`.
## TCP and the Nix sandbox {#sec-postgresqlTestHook-tcp}
`postgresqlEnableTCP` relies on network sandboxing, which is not available on macOS and some custom Nix installations, resulting in flaky tests.
For this reason, it is disabled by default.
The preferred solution is to make the test suite use a UNIX domain socket connection. This is the default behavior when no `host` connection parameter is provided.
Some test suites hardcode a value for `host` though, so a patch may be required. If you can upstream the patch, you can make `host` default to the `PGHOST` environment variable when set. Otherwise, you can patch it locally to omit the `host` connection string parameter altogether.
::: {.note}
The error `libpq: failed (could not receive data from server: Connection refused` is generally an indication that the test suite is trying to connect through TCP.
:::

@ -27,6 +27,7 @@
<xi:include href="builders/trivial-builders.chapter.xml" />
<xi:include href="builders/special.xml" />
<xi:include href="builders/images.xml" />
<xi:include href="hooks/index.xml" />
<xi:include href="languages-frameworks/index.xml" />
<xi:include href="builders/packages/index.xml" />
</part>

@ -73,6 +73,13 @@
Systemd has been upgraded to the version 250.
</para>
</listitem>
<listitem>
<para>
The new
<link xlink:href="https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook"><literal>postgresqlTestHook</literal></link>
runs a PostgreSQL server for the duration of package checks.
</para>
</listitem>
<listitem>
<para>
<link xlink:href="https://kops.sigs.k8s.io"><literal>kops</literal></link>

@ -27,6 +27,8 @@ In addition to numerous new and upgraded packages, this release has the followin
- Systemd has been upgraded to the version 250.
- The new [`postgresqlTestHook`](https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook) runs a PostgreSQL server for the duration of package checks.
- [`kops`](https://kops.sigs.k8s.io) defaults to 1.22.4, which will enable [Instance Metadata Service Version 2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) and require tokens on new clusters with Kubernetes 1.22. This will increase security by default, but may break some types of workloads. See the [release notes](https://kops.sigs.k8s.io/releases/1.22-notes/) for details.
- Module authors can use `mkRenamedOptionModuleWith` to automate the deprecation cycle without annoying out-of-tree module authors and their users.

@ -0,0 +1,9 @@
{ callPackage, makeSetupHook }:
(makeSetupHook {
name = "postgresql-test-hook";
} ./postgresql-test-hook.sh).overrideAttrs (o: {
passthru.tests = {
simple = callPackage ./test.nix { };
};
})

@ -0,0 +1,79 @@
preCheckHooks+=('postgresqlStart')
postCheckHooks+=('postgresqlStop')
postgresqlStart() {
# Add default environment variable values
#
# Client variables:
# - https://www.postgresql.org/docs/current/libpq-envars.html
#
# Server variables:
# - only PGDATA: https://www.postgresql.org/docs/current/creating-cluster.html
if [[ "${PGDATA:-}" == "" ]]; then
PGDATA="$NIX_BUILD_TOP/postgresql"
fi
export PGDATA
if [[ "${PGHOST:-}" == "" ]]; then
mkdir -p "$NIX_BUILD_TOP/run/postgresql"
PGHOST="$NIX_BUILD_TOP/run/postgresql"
fi
export PGHOST
if [[ "${PGUSER:-}" == "" ]]; then
PGUSER="test_user"
fi
export PGUSER
if [[ "${PGDATABASE:-}" == "" ]]; then
PGDATABASE="test_db"
fi
export PGDATABASE
if [[ "${postgresqlTestUserOptions:-}" == "" ]]; then
postgresqlTestUserOptions="LOGIN"
fi
if [[ "${postgresqlTestSetupSQL:-}" == "" ]]; then
postgresqlTestSetupSQL="$(cat <<EOF
CREATE ROLE "$PGUSER" $postgresqlTestUserOptions;
CREATE DATABASE "$PGDATABASE" OWNER '$PGUSER';
EOF
)"
fi
if [[ "${postgresqlTestSetupCommands:-}" == "" ]]; then
postgresqlTestSetupCommands='echo "$postgresqlTestSetupSQL" | PGUSER=postgres psql postgres'
fi
if ! type initdb >/dev/null; then
echo >&2 'initdb not found. Did you add postgresql to the checkInputs?'
false
fi
header 'initializing postgresql'
initdb -U postgres
# Move the socket
echo "unix_socket_directories = '$NIX_BUILD_TOP/run/postgresql'" >>"$PGDATA/postgresql.conf"
# TCP ports can be a problem in some sandboxes,
# so we disable tcp listening by default
if ! [[ "${postgresqlEnableTCP:-}" = 1 ]]; then
echo "listen_addresses = ''" >>"$PGDATA/postgresql.conf"
fi
header 'starting postgresql'
eval "${postgresqlStartCommands:-pg_ctl start}"
header 'setting up postgresql'
eval "$postgresqlTestSetupCommands"
}
postgresqlStop() {
header 'stopping postgresql'
pg_ctl stop
}

@ -0,0 +1,27 @@
{ postgresql, postgresqlTestHook, stdenv }:
stdenv.mkDerivation {
name = "postgresql-test-hook-test";
buildInputs = [ postgresqlTestHook ];
checkInputs = [ postgresql ];
dontUnpack = true;
doCheck = true;
passAsFile = ["sql"];
sql = ''
CREATE TABLE hello (
message text
);
INSERT INTO hello VALUES ('it '||'worked');
SELECT * FROM hello;
'';
checkPhase = ''
runHook preCheck
psql <$sqlPath | grep 'it worked'
TEST_RAN=1
runHook postCheck
'';
installPhase = ''
[[ $TEST_RAN == 1 ]]
touch $out
'';
}

@ -1190,8 +1190,29 @@ self: super: {
# Fix build with attr-2.4.48 (see #53716)
xattr = appendPatch ./patches/xattr-fix-build.patch super.xattr;
# Some tests depend on a postgresql instance
esqueleto = dontCheck super.esqueleto;
esqueleto =
overrideCabal
(drv: {
postPatch = drv.postPatch or "" + ''
# patch out TCP usage: https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook-tcp
sed -i test/PostgreSQL/Test.hs \
-e s^host=localhost^^
'';
# Match the test suite defaults (or hardcoded values?)
preCheck = drv.preCheck or "" + ''
PGUSER=esqutest
PGDATABASE=esqutest
'';
testFlags = drv.testFlags or [] ++ [
# We don't have a MySQL test hook yet
"--skip=/Esqueleto/MySQL"
];
testToolDepends = drv.testToolDepends or [] ++ [
pkgs.postgresql
pkgs.postgresqlTestHook
];
})
super.esqueleto;
# Requires API keys to run tests
algolia = dontCheck super.algolia;
@ -1310,7 +1331,25 @@ self: super: {
# Test suite requires database
persistent-mysql = dontCheck super.persistent-mysql;
persistent-postgresql = dontCheck super.persistent-postgresql;
persistent-postgresql =
overrideCabal
(drv: {
postPatch = drv.postPath or "" + ''
# patch out TCP usage: https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook-tcp
# NOTE: upstream host variable takes only two values...
sed -i test/PgInit.hs \
-e s^'host=" <> host <> "'^^
'';
preCheck = drv.preCheck or "" + ''
PGDATABASE=test
PGUSER=test
'';
testToolDepends = drv.testToolDepends or [] ++ [
pkgs.postgresql
pkgs.postgresqlTestHook
];
})
super.persistent-postgresql;
# Fix EdisonAPI and EdisonCore for GHC 8.8:
# https://github.com/robdockins/edison/pull/16
@ -1515,8 +1554,13 @@ self: super: {
};
pg-client = overrideCabal (drv: {
librarySystemDepends = with pkgs; [ postgresql krb5.dev openssl.dev ];
# wants a running DB to check against
doCheck = false;
testToolDepends = drv.testToolDepends or [] ++ [
pkgs.postgresql pkgs.postgresqlTestHook
];
preCheck = drv.preCheck or "" + ''
# empty string means use default connection
export DATABASE_URL=""
'';
}) (super.pg-client.override {
resource-pool = self.hasura-resource-pool;
ekg-core = self.hasura-ekg-core;

@ -22181,6 +22181,8 @@ with pkgs;
postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { };
postgresqlTestHook = callPackage ../build-support/setup-hooks/postgresql-test-hook { };
prom2json = callPackage ../servers/monitoring/prometheus/prom2json.nix { };
prometheus = callPackage ../servers/monitoring/prometheus { };
prometheus-alertmanager = callPackage ../servers/monitoring/prometheus/alertmanager.nix { };

Loading…
Cancel
Save