nixos/postgresql-wal-receiver: add module (#63799)
parent
12a7cc0d1f
commit
4ff9a48398
@ -0,0 +1,203 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
receiverSubmodule = { |
||||
options = { |
||||
postgresqlPackage = mkOption { |
||||
type = types.package; |
||||
example = literalExample "pkgs.postgresql_11"; |
||||
description = '' |
||||
PostgreSQL package to use. |
||||
''; |
||||
}; |
||||
|
||||
directory = mkOption { |
||||
type = types.path; |
||||
example = literalExample "/mnt/pg_wal/main/"; |
||||
description = '' |
||||
Directory to write the output to. |
||||
''; |
||||
}; |
||||
|
||||
statusInterval = mkOption { |
||||
type = types.int; |
||||
default = 10; |
||||
description = '' |
||||
Specifies the number of seconds between status packets sent back to the server. |
||||
This allows for easier monitoring of the progress from server. |
||||
A value of zero disables the periodic status updates completely, |
||||
although an update will still be sent when requested by the server, to avoid timeout disconnect. |
||||
''; |
||||
}; |
||||
|
||||
slot = mkOption { |
||||
type = types.str; |
||||
default = ""; |
||||
example = "some_slot_name"; |
||||
description = '' |
||||
Require <command>pg_receivewal</command> to use an existing replication slot (see |
||||
<link xlink:href="https://www.postgresql.org/docs/current/warm-standby.html#STREAMING-REPLICATION-SLOTS">Section 26.2.6 of the PostgreSQL manual</link>). |
||||
When this option is used, <command>pg_receivewal</command> will report a flush position to the server, |
||||
indicating when each segment has been synchronized to disk so that the server can remove that segment if it is not otherwise needed. |
||||
|
||||
When the replication client of <command>pg_receivewal</command> is configured on the server as a synchronous standby, |
||||
then using a replication slot will report the flush position to the server, but only when a WAL file is closed. |
||||
Therefore, that configuration will cause transactions on the primary to wait for a long time and effectively not work satisfactorily. |
||||
The option <option>synchronous</option> must be specified in addition to make this work correctly. |
||||
''; |
||||
}; |
||||
|
||||
synchronous = mkOption { |
||||
type = types.bool; |
||||
default = false; |
||||
description = '' |
||||
Flush the WAL data to disk immediately after it has been received. |
||||
Also send a status packet back to the server immediately after flushing, regardless of <option>statusInterval</option>. |
||||
|
||||
This option should be specified if the replication client of <command>pg_receivewal</command> is configured on the server as a synchronous standby, |
||||
to ensure that timely feedback is sent to the server. |
||||
''; |
||||
}; |
||||
|
||||
compress = mkOption { |
||||
type = types.ints.between 0 9; |
||||
default = 0; |
||||
description = '' |
||||
Enables gzip compression of write-ahead logs, and specifies the compression level |
||||
(<literal>0</literal> through <literal>9</literal>, <literal>0</literal> being no compression and <literal>9</literal> being best compression). |
||||
The suffix <literal>.gz</literal> will automatically be added to all filenames. |
||||
|
||||
This option requires PostgreSQL >= 10. |
||||
''; |
||||
}; |
||||
|
||||
connection = mkOption { |
||||
type = types.str; |
||||
example = "postgresql://user@somehost"; |
||||
description = '' |
||||
Specifies parameters used to connect to the server, as a connection string. |
||||
See <link xlink:href="https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING">Section 34.1.1 of the PostgreSQL manual</link> for more information. |
||||
|
||||
Because <command>pg_receivewal</command> doesn't connect to any particular database in the cluster, |
||||
database name in the connection string will be ignored. |
||||
''; |
||||
}; |
||||
|
||||
extraArgs = mkOption { |
||||
type = with types; listOf str; |
||||
default = [ ]; |
||||
example = literalExample '' |
||||
[ |
||||
"--no-sync" |
||||
] |
||||
''; |
||||
description = '' |
||||
A list of extra arguments to pass to the <command>pg_receivewal</command> command. |
||||
''; |
||||
}; |
||||
|
||||
environment = mkOption { |
||||
type = with types; attrsOf str; |
||||
default = { }; |
||||
example = literalExample '' |
||||
{ |
||||
PGPASSFILE = "/private/passfile"; |
||||
PGSSLMODE = "require"; |
||||
} |
||||
''; |
||||
description = '' |
||||
Environment variables passed to the service. |
||||
Usable parameters are listed in <link xlink:href="https://www.postgresql.org/docs/current/libpq-envars.html">Section 34.14 of the PostgreSQL manual</link>. |
||||
''; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
in { |
||||
options = { |
||||
services.postgresqlWalReceiver = { |
||||
receivers = mkOption { |
||||
type = with types; attrsOf (submodule receiverSubmodule); |
||||
default = { }; |
||||
example = literalExample '' |
||||
{ |
||||
main = { |
||||
postgresqlPackage = pkgs.postgresql_11; |
||||
directory = /mnt/pg_wal/main/; |
||||
slot = "main_wal_receiver"; |
||||
connection = "postgresql://user@somehost"; |
||||
}; |
||||
} |
||||
''; |
||||
description = '' |
||||
PostgreSQL WAL receivers. |
||||
Stream write-ahead logs from a PostgreSQL server using <command>pg_receivewal</command> (formerly <command>pg_receivexlog</command>). |
||||
See <link xlink:href="https://www.postgresql.org/docs/current/app-pgreceivewal.html">the man page</link> for more information. |
||||
''; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
config = let |
||||
receivers = config.services.postgresqlWalReceiver.receivers; |
||||
in mkIf (receivers != { }) { |
||||
users = { |
||||
users.postgres = { |
||||
uid = config.ids.uids.postgres; |
||||
group = "postgres"; |
||||
description = "PostgreSQL server user"; |
||||
}; |
||||
|
||||
groups.postgres = { |
||||
gid = config.ids.gids.postgres; |
||||
}; |
||||
}; |
||||
|
||||
assertions = concatLists (attrsets.mapAttrsToList (name: config: [ |
||||
{ |
||||
assertion = config.compress > 0 -> versionAtLeast config.postgresqlPackage.version "10"; |
||||
message = "Invalid configuration for WAL receiver \"${name}\": compress requires PostgreSQL version >= 10."; |
||||
} |
||||
]) receivers); |
||||
|
||||
systemd.tmpfiles.rules = mapAttrsToList (name: config: '' |
||||
d ${escapeShellArg config.directory} 0750 postgres postgres - - |
||||
'') receivers; |
||||
|
||||
systemd.services = with attrsets; mapAttrs' (name: config: nameValuePair "postgresql-wal-receiver-${name}" { |
||||
description = "PostgreSQL WAL receiver (${name})"; |
||||
wantedBy = [ "multi-user.target" ]; |
||||
|
||||
serviceConfig = { |
||||
User = "postgres"; |
||||
Group = "postgres"; |
||||
KillSignal = "SIGINT"; |
||||
Restart = "always"; |
||||
RestartSec = 30; |
||||
}; |
||||
|
||||
inherit (config) environment; |
||||
|
||||
script = let |
||||
receiverCommand = postgresqlPackage: |
||||
if (versionAtLeast postgresqlPackage.version "10") |
||||
then "${postgresqlPackage}/bin/pg_receivewal" |
||||
else "${postgresqlPackage}/bin/pg_receivexlog"; |
||||
in '' |
||||
${receiverCommand config.postgresqlPackage} \ |
||||
--no-password \ |
||||
--directory=${escapeShellArg config.directory} \ |
||||
--status-interval=${toString config.statusInterval} \ |
||||
--dbname=${escapeShellArg config.connection} \ |
||||
${optionalString (config.compress > 0) "--compress=${toString config.compress}"} \ |
||||
${optionalString (config.slot != "") "--slot=${escapeShellArg config.slot}"} \ |
||||
${optionalString config.synchronous "--synchronous"} \ |
||||
${concatStringsSep " " config.extraArgs} |
||||
''; |
||||
}) receivers; |
||||
}; |
||||
|
||||
meta.maintainers = with maintainers; [ pacien ]; |
||||
} |
@ -0,0 +1,86 @@ |
||||
{ system ? builtins.currentSystem |
||||
, config ? { } |
||||
, pkgs ? import ../.. { inherit system config; } }: |
||||
|
||||
with import ../lib/testing.nix { inherit system pkgs; }; |
||||
with pkgs.lib; |
||||
|
||||
let |
||||
postgresqlDataDir = "/var/db/postgresql/test"; |
||||
replicationUser = "wal_receiver_user"; |
||||
replicationSlot = "wal_receiver_slot"; |
||||
replicationConn = "postgresql://${replicationUser}@localhost"; |
||||
baseBackupDir = "/tmp/pg_basebackup"; |
||||
walBackupDir = "/tmp/pg_wal"; |
||||
recoveryConf = pkgs.writeText "recovery.conf" '' |
||||
restore_command = 'cp ${walBackupDir}/%f %p' |
||||
''; |
||||
|
||||
makePostgresqlWalReceiverTest = subTestName: postgresqlPackage: makeTest { |
||||
name = "postgresql-wal-receiver-${subTestName}"; |
||||
meta.maintainers = with maintainers; [ pacien ]; |
||||
|
||||
machine = { ... }: { |
||||
services.postgresql = { |
||||
package = postgresqlPackage; |
||||
enable = true; |
||||
dataDir = postgresqlDataDir; |
||||
extraConfig = '' |
||||
wal_level = archive # alias for replica on pg >= 9.6 |
||||
max_wal_senders = 10 |
||||
max_replication_slots = 10 |
||||
''; |
||||
authentication = '' |
||||
host replication ${replicationUser} all trust |
||||
''; |
||||
initialScript = pkgs.writeText "init.sql" '' |
||||
create user ${replicationUser} replication; |
||||
select * from pg_create_physical_replication_slot('${replicationSlot}'); |
||||
''; |
||||
}; |
||||
|
||||
services.postgresqlWalReceiver.receivers.main = { |
||||
inherit postgresqlPackage; |
||||
connection = replicationConn; |
||||
slot = replicationSlot; |
||||
directory = walBackupDir; |
||||
}; |
||||
}; |
||||
|
||||
testScript = '' |
||||
# make an initial base backup |
||||
$machine->waitForUnit('postgresql'); |
||||
$machine->waitForUnit('postgresql-wal-receiver-main'); |
||||
# WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other |
||||
# required only for 9.4 |
||||
$machine->sleep(5); |
||||
$machine->succeed('${postgresqlPackage}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}'); |
||||
|
||||
# create a dummy table with 100 records |
||||
$machine->succeed('sudo -u postgres psql --command="create table dummy as select * from generate_series(1, 100) as val;"'); |
||||
|
||||
# stop postgres and destroy data |
||||
$machine->systemctl('stop postgresql'); |
||||
$machine->systemctl('stop postgresql-wal-receiver-main'); |
||||
$machine->succeed('rm -r ${postgresqlDataDir}/{base,global,pg_*}'); |
||||
|
||||
# restore the base backup |
||||
$machine->succeed('cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}'); |
||||
|
||||
# prepare WAL and recovery |
||||
$machine->succeed('chmod a+rX -R ${walBackupDir}'); |
||||
$machine->execute('for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done'); # make use of partial segments too |
||||
$machine->succeed('cp ${recoveryConf} ${postgresqlDataDir}/recovery.conf && chmod 666 ${postgresqlDataDir}/recovery.conf'); |
||||
|
||||
# replay WAL |
||||
$machine->systemctl('start postgresql'); |
||||
$machine->waitForFile('${postgresqlDataDir}/recovery.done'); |
||||
$machine->systemctl('restart postgresql'); |
||||
$machine->waitForUnit('postgresql'); |
||||
|
||||
# check that our records have been restored |
||||
$machine->succeed('test $(sudo -u postgres psql --pset="pager=off" --tuples-only --command="select count(distinct val) from dummy;") -eq 100'); |
||||
''; |
||||
}; |
||||
|
||||
in mapAttrs makePostgresqlWalReceiverTest (import ../../pkgs/servers/sql/postgresql pkgs) |
Loading…
Reference in new issue