Initial Exhibitor nix package and nixos module for Netflix's Exhibitor, which is a manager for Apache Zookeeper.wip/yesman
parent
91dc811566
commit
4b42fc4b8a
@ -0,0 +1,361 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
cfg = config.services.exhibitor; |
||||
exhibitor = cfg.package; |
||||
exhibitorConfig = '' |
||||
zookeeper-install-directory=${cfg.baseDir}/zookeeper |
||||
zookeeper-data-directory=${cfg.zkDataDir} |
||||
zookeeper-log-directory=${cfg.zkLogDir} |
||||
zoo-cfg-extra=${cfg.zkExtraCfg} |
||||
client-port=${toString cfg.zkClientPort} |
||||
connect-port=${toString cfg.zkConnectPort} |
||||
election-port=${toString cfg.zkElectionPort} |
||||
cleanup-period-ms=${toString cfg.zkCleanupPeriod} |
||||
servers-spec=${concatStringsSep "," cfg.zkServersSpec} |
||||
auto-manage-instances=${toString cfg.autoManageInstances} |
||||
${cfg.extraConf} |
||||
''; |
||||
configDir = pkgs.writeTextDir "exhibitor.properties" exhibitorConfig; |
||||
cliOptionsCommon = { |
||||
configtype = cfg.configType; |
||||
defaultconfig = "${configDir}/exhibitor.properties"; |
||||
port = toString cfg.port; |
||||
hostname = cfg.hostname; |
||||
}; |
||||
s3CommonOptions = { s3region = cfg.s3Region; s3credentials = cfg.s3Credentials; }; |
||||
cliOptionsPerConfig = { |
||||
s3 = { |
||||
s3config = "${cfg.s3Config.bucketName}:${cfg.s3Config.objectKey}"; |
||||
s3configprefix = cfg.s3Config.configPrefix; |
||||
}; |
||||
zookeeper = { |
||||
zkconfigconnect = concatStringsSep "," cfg.zkConfigConnect; |
||||
zkconfigexhibitorpath = cfg.zkConfigExhibitorPath; |
||||
zkconfigpollms = toString cfg.zkConfigPollMs; |
||||
zkconfigretry = "${toString cfg.zkConfigRetry.sleepMs}:${toString cfg.zkConfigRetry.retryQuantity}"; |
||||
zkconfigzpath = cfg.zkConfigZPath; |
||||
zkconfigexhibitorport = toString cfg.zkConfigExhibitorPort; # NB: This might be null |
||||
}; |
||||
file = { |
||||
fsconfigdir = cfg.fsConfigDir; |
||||
fsconfiglockprefix = cfg.fsConfigLockPrefix; |
||||
fsConfigName = fsConfigName; |
||||
}; |
||||
none = { |
||||
noneconfigdir = configDir; |
||||
}; |
||||
}; |
||||
cliOptions = concatStringsSep " " (mapAttrsToList (k: v: "--${k} ${v}") (filterAttrs (k: v: v != null && v != "") (cliOptionsCommon // |
||||
cliOptionsPerConfig."${cfg.configType}" // |
||||
s3CommonOptions // |
||||
optionalAttrs cfg.s3Backup { s3backup = "true"; } // |
||||
optionalAttrs cfg.fileSystemBackup { filesystembackup = "true"; } |
||||
))); |
||||
in |
||||
{ |
||||
options = { |
||||
services.exhibitor = { |
||||
enable = mkOption { |
||||
type = types.bool; |
||||
default = false; |
||||
description = " |
||||
Whether to enable the exhibitor server. |
||||
"; |
||||
}; |
||||
# See https://github.com/soabase/exhibitor/wiki/Running-Exhibitor for what these mean |
||||
# General options for any type of config |
||||
port = mkOption { |
||||
type = types.int; |
||||
default = 8080; |
||||
description = '' |
||||
The port for exhibitor to listen on and communicate with other exhibitors. |
||||
''; |
||||
}; |
||||
baseDir = mkOption { |
||||
type = types.str; |
||||
default = "/var/exhibitor"; |
||||
description = '' |
||||
Baseline directory for exhibitor runtime config. |
||||
''; |
||||
}; |
||||
configType = mkOption { |
||||
type = types.enum [ "file" "s3" "zookeeper" "none" ]; |
||||
description = '' |
||||
Which configuration type you want to use. Additional config will be |
||||
required depending on which type you are using. |
||||
''; |
||||
}; |
||||
hostname = mkOption { |
||||
type = types.nullOr types.str; |
||||
description = '' |
||||
Hostname to use and advertise |
||||
''; |
||||
default = null; |
||||
}; |
||||
autoManageInstances = mkOption { |
||||
type = types.bool; |
||||
description = '' |
||||
Automatically manage ZooKeeper instances in the ensemble |
||||
''; |
||||
default = false; |
||||
}; |
||||
zkDataDir = mkOption { |
||||
type = types.str; |
||||
default = "${cfg.baseDir}/zkData"; |
||||
description = '' |
||||
The Zookeeper data directory |
||||
''; |
||||
}; |
||||
zkLogDir = mkOption { |
||||
type = types.path; |
||||
default = "${cfg.baseDir}/zkLogs"; |
||||
description = '' |
||||
The Zookeeper logs directory |
||||
''; |
||||
}; |
||||
extraConf = mkOption { |
||||
type = types.str; |
||||
default = ""; |
||||
description = '' |
||||
Extra Exhibitor configuration to put in the ZooKeeper config file. |
||||
''; |
||||
}; |
||||
zkExtraCfg = mkOption { |
||||
type = types.str; |
||||
default = ''initLimit=5&syncLimit=2&tickTime=2000''; |
||||
description = '' |
||||
Extra options to pass into Zookeeper |
||||
''; |
||||
}; |
||||
zkClientPort = mkOption { |
||||
type = types.int; |
||||
default = 2181; |
||||
description = '' |
||||
Zookeeper client port |
||||
''; |
||||
}; |
||||
zkConnectPort = mkOption { |
||||
type = types.int; |
||||
default = 2888; |
||||
description = '' |
||||
The port to use for followers to talk to each other. |
||||
''; |
||||
}; |
||||
zkElectionPort = mkOption { |
||||
type = types.int; |
||||
default = 3888; |
||||
description = '' |
||||
The port for Zookeepers to use for leader election. |
||||
''; |
||||
}; |
||||
zkCleanupPeriod = mkOption { |
||||
type = types.int; |
||||
default = 0; |
||||
description = '' |
||||
How often (in milliseconds) to run the Zookeeper log cleanup task. |
||||
''; |
||||
}; |
||||
zkServersSpec = mkOption { |
||||
type = types.listOf types.str; |
||||
default = []; |
||||
description = '' |
||||
Zookeeper server spec for all servers in the ensemble. |
||||
''; |
||||
example = [ "S:1:zk1.example.com" "S:2:zk2.example.com" "S:3:zk3.example.com" "O:4:zk-observer.example.com" ]; |
||||
}; |
||||
|
||||
# Backup options |
||||
s3Backup = mkOption { |
||||
type = types.bool; |
||||
default = false; |
||||
description = '' |
||||
Whether to enable backups to S3 |
||||
''; |
||||
}; |
||||
fileSystemBackup = mkOption { |
||||
type = types.bool; |
||||
default = false; |
||||
description = '' |
||||
Enables file system backup of ZooKeeper log files |
||||
''; |
||||
}; |
||||
|
||||
# Options for using zookeeper configType |
||||
zkConfigConnect = mkOption { |
||||
type = types.listOf types.str; |
||||
description = '' |
||||
The initial connection string for ZooKeeper shared config storage |
||||
''; |
||||
example = ["host1:2181" "host2:2181"]; |
||||
}; |
||||
zkConfigExhibitorPath = mkOption { |
||||
type = types.string; |
||||
description = '' |
||||
If the ZooKeeper shared config is also running Exhibitor, the URI path for the REST call |
||||
''; |
||||
default = "/"; |
||||
}; |
||||
zkConfigExhibitorPort = mkOption { |
||||
type = types.nullOr types.int; |
||||
description = '' |
||||
If the ZooKeeper shared config is also running Exhibitor, the port that |
||||
Exhibitor is listening on. IMPORTANT: if this value is not set it implies |
||||
that Exhibitor is not being used on the ZooKeeper shared config. |
||||
''; |
||||
}; |
||||
zkConfigPollMs = mkOption { |
||||
type = types.int; |
||||
description = '' |
||||
The period in ms to check for changes in the config ensemble |
||||
''; |
||||
default = 10000; |
||||
}; |
||||
zkConfigRetry = mkOption { |
||||
type = types.submodule { |
||||
options = { |
||||
sleepMs = mkOption { |
||||
type = types.int; |
||||
}; |
||||
retryQuantity = mkOption { |
||||
type = types.int; |
||||
}; |
||||
}; |
||||
}; |
||||
description = '' |
||||
The retry values to use |
||||
''; |
||||
default = { sleepMs = 1000; retryQuantity = 3; }; |
||||
}; |
||||
zkConfigZPath = mkOption { |
||||
type = types.str; |
||||
description = '' |
||||
The base ZPath that Exhibitor should use |
||||
''; |
||||
example = "/exhibitor/config"; |
||||
}; |
||||
|
||||
# Config options for s3 configType |
||||
s3Config = mkOption { |
||||
type = types.submodule { |
||||
options = { |
||||
bucketName = mkOption { |
||||
type = types.str; |
||||
description = '' |
||||
Bucket name to store config |
||||
''; |
||||
}; |
||||
objectKey = mkOption { |
||||
type = types.str; |
||||
description = '' |
||||
S3 key name to store the config |
||||
''; |
||||
}; |
||||
configPrefix = mkOption { |
||||
type = types.str; |
||||
description = '' |
||||
When using AWS S3 shared config files, the prefix to use for values such as locks |
||||
''; |
||||
default = "exhibitor-"; |
||||
}; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
# The next two are used for either s3backup or s3 configType |
||||
s3Credentials = mkOption { |
||||
type = types.nullOr types.path; |
||||
description = '' |
||||
Optional credentials to use for s3backup or s3config. Argument is the path |
||||
to an AWS credential properties file with two properties: |
||||
com.netflix.exhibitor.s3.access-key-id and com.netflix.exhibitor.s3.access-secret-key |
||||
''; |
||||
default = null; |
||||
}; |
||||
s3Region = mkOption { |
||||
type = types.nullOr types.str; |
||||
description = '' |
||||
Optional region for S3 calls |
||||
''; |
||||
default = null; |
||||
}; |
||||
|
||||
# Config options for file config type |
||||
fsConfigDir = mkOption { |
||||
type = types.path; |
||||
description = '' |
||||
Directory to store Exhibitor properties (cannot be used with s3config). |
||||
Exhibitor uses file system locks so you can specify a shared location |
||||
so as to enable complete ensemble management. |
||||
''; |
||||
}; |
||||
fsConfigLockPrefix = mkOption { |
||||
type = types.str; |
||||
description = '' |
||||
A prefix for a locking mechanism used in conjunction with fsconfigdir |
||||
''; |
||||
default = "exhibitor-lock-"; |
||||
}; |
||||
fsConfigName = mkOption { |
||||
type = types.str; |
||||
description = '' |
||||
The name of the file to store config in |
||||
''; |
||||
default = "exhibitor.properties"; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
config = mkIf cfg.enable { |
||||
systemd.services.exhibitor = { |
||||
description = "Exhibitor Daemon"; |
||||
wantedBy = [ "multi-user.target" ]; |
||||
after = [ "network.target" ]; |
||||
environment = { |
||||
ZOO_LOG_DIR = cfg.baseDir; |
||||
}; |
||||
serviceConfig = { |
||||
/*** |
||||
Exhibitor is a bit un-nixy. It wants to present to you a user interface in order to |
||||
mutate the configuration of both itself and ZooKeeper, and to coordinate changes |
||||
among the members of the Zookeeper ensemble. I'm going for a different approach here, |
||||
which is to manage all the configuration via nix and have it write out the configuration |
||||
files that exhibitor will use, and to reduce the amount of inter-exhibitor orchestration. |
||||
***/ |
||||
ExecStart = '' |
||||
${pkgs.exhibitor}/bin/startExhibitor.sh ${cliOptions} |
||||
''; |
||||
User = "zookeeper"; |
||||
PermissionsStartOnly = true; |
||||
}; |
||||
# This is a bit wonky, but the reason for this is that Exhibitor tries to write to |
||||
# ${cfg.baseDir}/zookeeper/bin/../conf/zoo.cfg |
||||
# I want everything but the conf directory to be in the immutable nix store, and I want defaults |
||||
# from the nix store |
||||
# If I symlink the bin directory in, then bin/../ will resolve to the parent of the symlink in the |
||||
# immutable nix store. Bind mounting a writable conf over the existing conf might work, but it gets very |
||||
# messy with trying to copy the existing out into a mutable store. |
||||
# Another option is to try to patch upstream exhibitor, but the current package just pulls down the |
||||
# prebuild JARs off of Maven, rather than building them ourselves, as Maven support in Nix isn't |
||||
# very mature. So, it seems like a reasonable compromise is to just copy out of the immutable store |
||||
# just before starting the service, so we're running binaries from the immutable store, but we work around |
||||
# Exhibitor's desire to mutate its current installation. |
||||
preStart = '' |
||||
mkdir -m 0700 -p ${cfg.baseDir}/zookeeper |
||||
# Not doing a chown -R to keep the base ZK files owned by root |
||||
chown zookeeper ${cfg.baseDir} ${cfg.baseDir}/zookeeper |
||||
cp -Rf ${pkgs.zookeeper}/* ${cfg.baseDir}/zookeeper |
||||
chown -R zookeeper ${cfg.baseDir}/zookeeper/conf |
||||
chmod -R u+w ${cfg.baseDir}/zookeeper/conf |
||||
''; |
||||
}; |
||||
users.extraUsers = singleton { |
||||
name = "zookeeper"; |
||||
uid = config.ids.uids.zookeeper; |
||||
description = "Zookeeper daemon user"; |
||||
home = cfg.baseDir; |
||||
}; |
||||
}; |
||||
} |
@ -0,0 +1,54 @@ |
||||
{ fetchFromGitHub, buildMaven, maven, jdk, makeWrapper, stdenv, ... }: |
||||
stdenv.mkDerivation rec { |
||||
name = "exhibitor-${version}"; |
||||
version = "1.5.6"; |
||||
|
||||
src = fetchFromGitHub { |
||||
owner = "soabase"; |
||||
repo = "exhibitor"; |
||||
sha256 = "07vikhkldxy51jbpy3jgva6wz75jksch6bjd6dqkagfgqd6baw45"; |
||||
rev = "5fcdb411d06e8638c2380f7acb72a8a6909739cd"; |
||||
}; |
||||
mavenDependenciesSha256 = "00r69n9hwvrn5cbhxklx7w00sjmqvcxs7gvhbm150ggy7bc865qv"; |
||||
# This is adapted from https://github.com/volth/nixpkgs/blob/6aa470dfd57cae46758b62010a93c5ff115215d7/pkgs/applications/networking/cluster/hadoop/default.nix#L20-L32 |
||||
fetchedMavenDeps = stdenv.mkDerivation { |
||||
name = "exhibitor-${version}-maven-deps"; |
||||
inherit src nativeBuildInputs; |
||||
buildPhase = '' |
||||
cd $pomFileDir; |
||||
while timeout --kill-after=21m 20m mvn package -Dmaven.repo.local=$out/.m2; [ $? = 124 ]; do |
||||
echo "maven hangs while downloading :(" |
||||
done |
||||
''; |
||||
installPhase = ''find $out/.m2 -type f \! -regex '.+\(pom\|jar\|xml\|sha1\)' -delete''; # delete files with lastModified timestamps inside |
||||
outputHashAlgo = "sha256"; |
||||
outputHashMode = "recursive"; |
||||
outputHash = mavenDependenciesSha256; |
||||
}; |
||||
|
||||
# The purpose of this is to fetch the jar file out of public Maven and use Maven |
||||
# to build a monolithic, standalone jar, rather than build everything from source |
||||
# (given the state of Maven support in Nix). We're not actually building any java |
||||
# source here. |
||||
pomFileDir = "exhibitor-standalone/src/main/resources/buildscripts/standalone/maven"; |
||||
nativeBuildInputs = [ maven ]; |
||||
buildInputs = [ makeWrapper ]; |
||||
buildPhase = '' |
||||
cd $pomFileDir |
||||
mvn package --offline -Dmaven.repo.local=$(cp -dpR ${fetchedMavenDeps}/.m2 ./ && chmod +w -R .m2 && pwd)/.m2 |
||||
''; |
||||
meta = with stdenv.lib; { |
||||
homepage = "https://github.com/soabase/exhibitor"; |
||||
description = "ZooKeeper co-process for instance monitoring, backup/recovery, cleanup and visualization"; |
||||
license = licenses.asl20; |
||||
platforms = platforms.unix; |
||||
}; |
||||
|
||||
installPhase = '' |
||||
mkdir -p $out/bin |
||||
mkdir -p $out/share/java |
||||
mv target/$name.jar $out/share/java/ |
||||
makeWrapper ${jdk}/bin/java $out/bin/startExhibitor.sh --add-flags "-jar $out/share/java/$name.jar" --suffix PATH : ${stdenv.lib.makeBinPath [ jdk ]} |
||||
''; |
||||
|
||||
} |
Loading…
Reference in new issue