nixos/postgresql-wal-receiver: add module (#63799)

wip/yesman
Notkea 5 years ago committed by Danylo Hlynskyi
parent 12a7cc0d1f
commit 4ff9a48398
  1. 1
      nixos/modules/module-list.nix
  2. 203
      nixos/modules/services/backup/postgresql-wal-receiver.nix
  3. 1
      nixos/tests/all-tests.nix
  4. 86
      nixos/tests/postgresql-wal-receiver.nix

@ -214,6 +214,7 @@
./services/backup/duplicity.nix
./services/backup/mysql-backup.nix
./services/backup/postgresql-backup.nix
./services/backup/postgresql-wal-receiver.nix
./services/backup/restic.nix
./services/backup/restic-rest-server.nix
./services/backup/rsnapshot.nix

@ -0,0 +1,203 @@
{ config, lib, pkgs, ... }:
with lib;
let
receiverSubmodule = {
options = {
postgresqlPackage = mkOption {
type = types.package;
example = literalExample "pkgs.postgresql_11";
description = ''
PostgreSQL package to use.
'';
};
directory = mkOption {
type = types.path;
example = literalExample "/mnt/pg_wal/main/";
description = ''
Directory to write the output to.
'';
};
statusInterval = mkOption {
type = types.int;
default = 10;
description = ''
Specifies the number of seconds between status packets sent back to the server.
This allows for easier monitoring of the progress from server.
A value of zero disables the periodic status updates completely,
although an update will still be sent when requested by the server, to avoid timeout disconnect.
'';
};
slot = mkOption {
type = types.str;
default = "";
example = "some_slot_name";
description = ''
Require <command>pg_receivewal</command> to use an existing replication slot (see
<link xlink:href="https://www.postgresql.org/docs/current/warm-standby.html#STREAMING-REPLICATION-SLOTS">Section 26.2.6 of the PostgreSQL manual</link>).
When this option is used, <command>pg_receivewal</command> will report a flush position to the server,
indicating when each segment has been synchronized to disk so that the server can remove that segment if it is not otherwise needed.
When the replication client of <command>pg_receivewal</command> is configured on the server as a synchronous standby,
then using a replication slot will report the flush position to the server, but only when a WAL file is closed.
Therefore, that configuration will cause transactions on the primary to wait for a long time and effectively not work satisfactorily.
The option <option>synchronous</option> must be specified in addition to make this work correctly.
'';
};
synchronous = mkOption {
type = types.bool;
default = false;
description = ''
Flush the WAL data to disk immediately after it has been received.
Also send a status packet back to the server immediately after flushing, regardless of <option>statusInterval</option>.
This option should be specified if the replication client of <command>pg_receivewal</command> is configured on the server as a synchronous standby,
to ensure that timely feedback is sent to the server.
'';
};
compress = mkOption {
type = types.ints.between 0 9;
default = 0;
description = ''
Enables gzip compression of write-ahead logs, and specifies the compression level
(<literal>0</literal> through <literal>9</literal>, <literal>0</literal> being no compression and <literal>9</literal> being best compression).
The suffix <literal>.gz</literal> will automatically be added to all filenames.
This option requires PostgreSQL >= 10.
'';
};
connection = mkOption {
type = types.str;
example = "postgresql://user@somehost";
description = ''
Specifies parameters used to connect to the server, as a connection string.
See <link xlink:href="https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING">Section 34.1.1 of the PostgreSQL manual</link> for more information.
Because <command>pg_receivewal</command> doesn't connect to any particular database in the cluster,
database name in the connection string will be ignored.
'';
};
extraArgs = mkOption {
type = with types; listOf str;
default = [ ];
example = literalExample ''
[
"--no-sync"
]
'';
description = ''
A list of extra arguments to pass to the <command>pg_receivewal</command> command.
'';
};
environment = mkOption {
type = with types; attrsOf str;
default = { };
example = literalExample ''
{
PGPASSFILE = "/private/passfile";
PGSSLMODE = "require";
}
'';
description = ''
Environment variables passed to the service.
Usable parameters are listed in <link xlink:href="https://www.postgresql.org/docs/current/libpq-envars.html">Section 34.14 of the PostgreSQL manual</link>.
'';
};
};
};
in {
options = {
services.postgresqlWalReceiver = {
receivers = mkOption {
type = with types; attrsOf (submodule receiverSubmodule);
default = { };
example = literalExample ''
{
main = {
postgresqlPackage = pkgs.postgresql_11;
directory = /mnt/pg_wal/main/;
slot = "main_wal_receiver";
connection = "postgresql://user@somehost";
};
}
'';
description = ''
PostgreSQL WAL receivers.
Stream write-ahead logs from a PostgreSQL server using <command>pg_receivewal</command> (formerly <command>pg_receivexlog</command>).
See <link xlink:href="https://www.postgresql.org/docs/current/app-pgreceivewal.html">the man page</link> for more information.
'';
};
};
};
config = let
receivers = config.services.postgresqlWalReceiver.receivers;
in mkIf (receivers != { }) {
users = {
users.postgres = {
uid = config.ids.uids.postgres;
group = "postgres";
description = "PostgreSQL server user";
};
groups.postgres = {
gid = config.ids.gids.postgres;
};
};
assertions = concatLists (attrsets.mapAttrsToList (name: config: [
{
assertion = config.compress > 0 -> versionAtLeast config.postgresqlPackage.version "10";
message = "Invalid configuration for WAL receiver \"${name}\": compress requires PostgreSQL version >= 10.";
}
]) receivers);
systemd.tmpfiles.rules = mapAttrsToList (name: config: ''
d ${escapeShellArg config.directory} 0750 postgres postgres - -
'') receivers;
systemd.services = with attrsets; mapAttrs' (name: config: nameValuePair "postgresql-wal-receiver-${name}" {
description = "PostgreSQL WAL receiver (${name})";
wantedBy = [ "multi-user.target" ];
serviceConfig = {
User = "postgres";
Group = "postgres";
KillSignal = "SIGINT";
Restart = "always";
RestartSec = 30;
};
inherit (config) environment;
script = let
receiverCommand = postgresqlPackage:
if (versionAtLeast postgresqlPackage.version "10")
then "${postgresqlPackage}/bin/pg_receivewal"
else "${postgresqlPackage}/bin/pg_receivexlog";
in ''
${receiverCommand config.postgresqlPackage} \
--no-password \
--directory=${escapeShellArg config.directory} \
--status-interval=${toString config.statusInterval} \
--dbname=${escapeShellArg config.connection} \
${optionalString (config.compress > 0) "--compress=${toString config.compress}"} \
${optionalString (config.slot != "") "--slot=${escapeShellArg config.slot}"} \
${optionalString config.synchronous "--synchronous"} \
${concatStringsSep " " config.extraArgs}
'';
}) receivers;
};
meta.maintainers = with maintainers; [ pacien ];
}

@ -210,6 +210,7 @@ in
plotinus = handleTest ./plotinus.nix {};
postgis = handleTest ./postgis.nix {};
postgresql = handleTest ./postgresql.nix {};
postgresql-wal-receiver = handleTest ./postgresql-wal-receiver.nix {};
powerdns = handleTest ./powerdns.nix {};
predictable-interface-names = handleTest ./predictable-interface-names.nix {};
printing = handleTest ./printing.nix {};

@ -0,0 +1,86 @@
{ system ? builtins.currentSystem
, config ? { }
, pkgs ? import ../.. { inherit system config; } }:
with import ../lib/testing.nix { inherit system pkgs; };
with pkgs.lib;
let
postgresqlDataDir = "/var/db/postgresql/test";
replicationUser = "wal_receiver_user";
replicationSlot = "wal_receiver_slot";
replicationConn = "postgresql://${replicationUser}@localhost";
baseBackupDir = "/tmp/pg_basebackup";
walBackupDir = "/tmp/pg_wal";
recoveryConf = pkgs.writeText "recovery.conf" ''
restore_command = 'cp ${walBackupDir}/%f %p'
'';
makePostgresqlWalReceiverTest = subTestName: postgresqlPackage: makeTest {
name = "postgresql-wal-receiver-${subTestName}";
meta.maintainers = with maintainers; [ pacien ];
machine = { ... }: {
services.postgresql = {
package = postgresqlPackage;
enable = true;
dataDir = postgresqlDataDir;
extraConfig = ''
wal_level = archive # alias for replica on pg >= 9.6
max_wal_senders = 10
max_replication_slots = 10
'';
authentication = ''
host replication ${replicationUser} all trust
'';
initialScript = pkgs.writeText "init.sql" ''
create user ${replicationUser} replication;
select * from pg_create_physical_replication_slot('${replicationSlot}');
'';
};
services.postgresqlWalReceiver.receivers.main = {
inherit postgresqlPackage;
connection = replicationConn;
slot = replicationSlot;
directory = walBackupDir;
};
};
testScript = ''
# make an initial base backup
$machine->waitForUnit('postgresql');
$machine->waitForUnit('postgresql-wal-receiver-main');
# WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other
# required only for 9.4
$machine->sleep(5);
$machine->succeed('${postgresqlPackage}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}');
# create a dummy table with 100 records
$machine->succeed('sudo -u postgres psql --command="create table dummy as select * from generate_series(1, 100) as val;"');
# stop postgres and destroy data
$machine->systemctl('stop postgresql');
$machine->systemctl('stop postgresql-wal-receiver-main');
$machine->succeed('rm -r ${postgresqlDataDir}/{base,global,pg_*}');
# restore the base backup
$machine->succeed('cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}');
# prepare WAL and recovery
$machine->succeed('chmod a+rX -R ${walBackupDir}');
$machine->execute('for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done'); # make use of partial segments too
$machine->succeed('cp ${recoveryConf} ${postgresqlDataDir}/recovery.conf && chmod 666 ${postgresqlDataDir}/recovery.conf');
# replay WAL
$machine->systemctl('start postgresql');
$machine->waitForFile('${postgresqlDataDir}/recovery.done');
$machine->systemctl('restart postgresql');
$machine->waitForUnit('postgresql');
# check that our records have been restored
$machine->succeed('test $(sudo -u postgres psql --pset="pager=off" --tuples-only --command="select count(distinct val) from dummy;") -eq 100');
'';
};
in mapAttrs makePostgresqlWalReceiverTest (import ../../pkgs/servers/sql/postgresql pkgs)
Loading…
Cancel
Save