- All kubernetes components have been seperated into different files - All TLS-enabled ports have been deprecated and disabled by default - EasyCert option added to support automatic cluster PKI-bootstrap - RBAC has been enforced for all cluster components by default - NixOS kubernetes test cases make use of easyCerts to setup PKIwip/yesman
Johan Thomsen
6 years ago
committed by
Franz Pletz
parent
5a4c8092c0
commit
e2380e79e1
@ -0,0 +1,167 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.addonManager; |
||||
|
||||
isRBACEnabled = elem "RBAC" top.apiserver.authorizationMode; |
||||
|
||||
addons = pkgs.runCommand "kubernetes-addons" { } '' |
||||
mkdir -p $out |
||||
# since we are mounting the addons to the addon manager, they need to be copied |
||||
${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon: |
||||
pkgs.writeTextDir "${name}.json" (builtins.toJSON addon) |
||||
) (cfg.addons))} |
||||
''; |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.addonManager = with lib.types; { |
||||
|
||||
bootstrapAddons = mkOption { |
||||
description = '' |
||||
Bootstrap addons are like regular addons, but they are applied with cluster-admin rigths. |
||||
They are applied at addon-manager startup only. |
||||
''; |
||||
default = { }; |
||||
type = attrsOf attrs; |
||||
example = literalExample '' |
||||
{ |
||||
"my-service" = { |
||||
"apiVersion" = "v1"; |
||||
"kind" = "Service"; |
||||
"metadata" = { |
||||
"name" = "my-service"; |
||||
"namespace" = "default"; |
||||
}; |
||||
"spec" = { ... }; |
||||
}; |
||||
} |
||||
''; |
||||
}; |
||||
|
||||
addons = mkOption { |
||||
description = "Kubernetes addons (any kind of Kubernetes resource can be an addon)."; |
||||
default = { }; |
||||
type = attrsOf (either attrs (listOf attrs)); |
||||
example = literalExample '' |
||||
{ |
||||
"my-service" = { |
||||
"apiVersion" = "v1"; |
||||
"kind" = "Service"; |
||||
"metadata" = { |
||||
"name" = "my-service"; |
||||
"namespace" = "default"; |
||||
}; |
||||
"spec" = { ... }; |
||||
}; |
||||
} |
||||
// import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; }; |
||||
''; |
||||
}; |
||||
|
||||
enable = mkEnableOption "Whether to enable Kubernetes addon manager."; |
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkIf cfg.enable { |
||||
environment.etc."kubernetes/addons".source = "${addons}/"; |
||||
|
||||
systemd.services.kube-addon-manager = { |
||||
description = "Kubernetes addon manager"; |
||||
wantedBy = [ "kubernetes.target" ]; |
||||
after = [ "kube-apiserver.service" ]; |
||||
environment.ADDON_PATH = "/etc/kubernetes/addons/"; |
||||
path = [ pkgs.gawk ]; |
||||
serviceConfig = { |
||||
Slice = "kubernetes.slice"; |
||||
ExecStart = "${top.package}/bin/kube-addons"; |
||||
WorkingDirectory = top.dataDir; |
||||
User = "kubernetes"; |
||||
Group = "kubernetes"; |
||||
Restart = "on-failure"; |
||||
RestartSec = 10; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled |
||||
(let |
||||
name = system:kube-addon-manager; |
||||
namespace = "kube-system"; |
||||
in |
||||
{ |
||||
|
||||
kube-addon-manager-r = { |
||||
apiVersion = "rbac.authorization.k8s.io/v1"; |
||||
kind = "Role"; |
||||
metadata = { |
||||
inherit name namespace; |
||||
}; |
||||
rules = [{ |
||||
apiGroups = ["*"]; |
||||
resources = ["*"]; |
||||
verbs = ["*"]; |
||||
}]; |
||||
}; |
||||
|
||||
kube-addon-manager-rb = { |
||||
apiVersion = "rbac.authorization.k8s.io/v1"; |
||||
kind = "RoleBinding"; |
||||
metadata = { |
||||
inherit name namespace; |
||||
}; |
||||
roleRef = { |
||||
apiGroup = "rbac.authorization.k8s.io"; |
||||
kind = "Role"; |
||||
inherit name; |
||||
}; |
||||
subjects = [{ |
||||
apiGroup = "rbac.authorization.k8s.io"; |
||||
kind = "User"; |
||||
inherit name; |
||||
}]; |
||||
}; |
||||
|
||||
kube-addon-manager-cluster-lister-cr = { |
||||
apiVersion = "rbac.authorization.k8s.io/v1"; |
||||
kind = "ClusterRole"; |
||||
metadata = { |
||||
name = "${name}:cluster-lister"; |
||||
}; |
||||
rules = [{ |
||||
apiGroups = ["*"]; |
||||
resources = ["*"]; |
||||
verbs = ["list"]; |
||||
}]; |
||||
}; |
||||
|
||||
kube-addon-manager-cluster-lister-crb = { |
||||
apiVersion = "rbac.authorization.k8s.io/v1"; |
||||
kind = "ClusterRoleBinding"; |
||||
metadata = { |
||||
name = "${name}:cluster-lister"; |
||||
}; |
||||
roleRef = { |
||||
apiGroup = "rbac.authorization.k8s.io"; |
||||
kind = "ClusterRole"; |
||||
name = "${name}:cluster-lister"; |
||||
}; |
||||
subjects = [{ |
||||
kind = "User"; |
||||
inherit name; |
||||
}]; |
||||
}; |
||||
}); |
||||
|
||||
services.kubernetes.pki.certs = { |
||||
addonManager = top.lib.mkCert { |
||||
name = "kube-addon-manager"; |
||||
CN = "system:kube-addon-manager"; |
||||
action = "systemctl restart kube-addon-manager.service"; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
} |
||||
@ -0,0 +1,427 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.apiserver; |
||||
|
||||
isRBACEnabled = elem "RBAC" cfg.authorizationMode; |
||||
|
||||
apiserverServiceIP = (concatStringsSep "." ( |
||||
take 3 (splitString "." cfg.serviceClusterIpRange |
||||
)) + ".1"); |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.apiserver = with lib.types; { |
||||
|
||||
advertiseAddress = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver IP address on which to advertise the apiserver |
||||
to members of the cluster. This address must be reachable by the rest |
||||
of the cluster. |
||||
''; |
||||
default = null; |
||||
type = nullOr str; |
||||
}; |
||||
|
||||
allowPrivileged = mkOption { |
||||
description = "Whether to allow privileged containers on Kubernetes."; |
||||
default = false; |
||||
type = bool; |
||||
}; |
||||
|
||||
authorizationMode = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See |
||||
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> |
||||
''; |
||||
default = ["RBAC" "Node"]; # Enabling RBAC by default, although kubernetes default is AllowAllow |
||||
type = listOf (enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]); |
||||
}; |
||||
|
||||
authorizationPolicy = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver authorization policy file. See |
||||
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/> |
||||
''; |
||||
default = []; |
||||
type = listOf attrs; |
||||
}; |
||||
|
||||
basicAuthFile = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver basic authentication file. See |
||||
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
bindAddress = mkOption { |
||||
description = '' |
||||
The IP address on which to listen for the --secure-port port. |
||||
The associated interface(s) must be reachable by the rest |
||||
of the cluster, and by CLI/web clients. |
||||
''; |
||||
default = "0.0.0.0"; |
||||
type = str; |
||||
}; |
||||
|
||||
clientCaFile = mkOption { |
||||
description = "Kubernetes apiserver CA file for client auth."; |
||||
default = top.caFile; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
disableAdmissionPlugins = mkOption { |
||||
description = '' |
||||
Kubernetes admission control plugins to disable. See |
||||
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> |
||||
''; |
||||
default = []; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
enable = mkEnableOption "Kubernetes apiserver"; |
||||
|
||||
enableAdmissionPlugins = mkOption { |
||||
description = '' |
||||
Kubernetes admission control plugins to enable. See |
||||
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/> |
||||
''; |
||||
default = [ |
||||
"NamespaceLifecycle" "LimitRanger" "ServiceAccount" |
||||
"ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" |
||||
"NodeRestriction" |
||||
]; |
||||
example = [ |
||||
"NamespaceLifecycle" "NamespaceExists" "LimitRanger" |
||||
"SecurityContextDeny" "ServiceAccount" "ResourceQuota" |
||||
"PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass" |
||||
]; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
etcd = { |
||||
servers = mkOption { |
||||
description = "List of etcd servers."; |
||||
default = ["http://127.0.0.1:2379"]; |
||||
type = types.listOf types.str; |
||||
}; |
||||
|
||||
keyFile = mkOption { |
||||
description = "Etcd key file."; |
||||
default = null; |
||||
type = types.nullOr types.path; |
||||
}; |
||||
|
||||
certFile = mkOption { |
||||
description = "Etcd cert file."; |
||||
default = null; |
||||
type = types.nullOr types.path; |
||||
}; |
||||
|
||||
caFile = mkOption { |
||||
description = "Etcd ca file."; |
||||
default = top.caFile; |
||||
type = types.nullOr types.path; |
||||
}; |
||||
}; |
||||
|
||||
extraOpts = mkOption { |
||||
description = "Kubernetes apiserver extra command line options."; |
||||
default = ""; |
||||
type = str; |
||||
}; |
||||
|
||||
extraSANs = mkOption { |
||||
description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert."; |
||||
default = []; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
featureGates = mkOption { |
||||
description = "List set of feature gates"; |
||||
default = top.featureGates; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
insecureBindAddress = mkOption { |
||||
description = "The IP address on which to serve the --insecure-port."; |
||||
default = "127.0.0.1"; |
||||
type = str; |
||||
}; |
||||
|
||||
insecurePort = mkOption { |
||||
description = "Kubernetes apiserver insecure listening port. (0 = disabled)"; |
||||
default = 0; |
||||
type = int; |
||||
}; |
||||
|
||||
kubeletClientCaFile = mkOption { |
||||
description = "Path to a cert file for connecting to kubelet."; |
||||
default = top.caFile; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
kubeletClientCertFile = mkOption { |
||||
description = "Client certificate to use for connections to kubelet."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
kubeletClientKeyFile = mkOption { |
||||
description = "Key to use for connections to kubelet."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
kubeletHttps = mkOption { |
||||
description = "Whether to use https for connections to kubelet."; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
runtimeConfig = mkOption { |
||||
description = '' |
||||
Api runtime configuration. See |
||||
<link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/> |
||||
''; |
||||
default = "authentication.k8s.io/v1beta1=true"; |
||||
example = "api/all=false,api/v1=true"; |
||||
type = str; |
||||
}; |
||||
|
||||
storageBackend = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver storage backend. |
||||
''; |
||||
default = "etcd3"; |
||||
type = enum ["etcd2" "etcd3"]; |
||||
}; |
||||
|
||||
securePort = mkOption { |
||||
description = "Kubernetes apiserver secure port."; |
||||
default = 6443; |
||||
type = int; |
||||
}; |
||||
|
||||
serviceAccountKeyFile = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver PEM-encoded x509 RSA private or public key file, |
||||
used to verify ServiceAccount tokens. By default tls private key file |
||||
is used. |
||||
''; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
serviceClusterIpRange = mkOption { |
||||
description = '' |
||||
A CIDR notation IP range from which to assign service cluster IPs. |
||||
This must not overlap with any IP ranges assigned to nodes for pods. |
||||
''; |
||||
default = "10.0.0.0/24"; |
||||
type = str; |
||||
}; |
||||
|
||||
tlsCertFile = mkOption { |
||||
description = "Kubernetes apiserver certificate file."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
tlsKeyFile = mkOption { |
||||
description = "Kubernetes apiserver private key file."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
tokenAuthFile = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver token authentication file. See |
||||
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
verbosity = mkOption { |
||||
description = '' |
||||
Optional glog verbosity level for logging statements. See |
||||
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr int; |
||||
}; |
||||
|
||||
webhookConfig = mkOption { |
||||
description = '' |
||||
Kubernetes apiserver Webhook config file. It uses the kubeconfig file format. |
||||
See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
|
||||
###### implementation |
||||
config = mkMerge [ |
||||
|
||||
(mkIf cfg.enable { |
||||
systemd.services.kube-apiserver = { |
||||
description = "Kubernetes APIServer Service"; |
||||
wantedBy = [ "kubernetes.target" ]; |
||||
after = [ "network.target" ]; |
||||
serviceConfig = { |
||||
Slice = "kubernetes.slice"; |
||||
ExecStart = ''${top.package}/bin/kube-apiserver \ |
||||
--allow-privileged=${boolToString cfg.allowPrivileged} \ |
||||
--authorization-mode=${concatStringsSep "," cfg.authorizationMode} \ |
||||
${optionalString (elem "ABAC" cfg.authorizationMode) |
||||
"--authorization-policy-file=${ |
||||
pkgs.writeText "kube-auth-policy.jsonl" |
||||
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy) |
||||
}" |
||||
} \ |
||||
${optionalString (elem "Webhook" cfg.authorizationMode) |
||||
"--authorization-webhook-config-file=${cfg.webhookConfig}" |
||||
} \ |
||||
--bind-address=${cfg.bindAddress} \ |
||||
${optionalString (cfg.advertiseAddress != null) |
||||
"--advertise-address=${cfg.advertiseAddress}"} \ |
||||
${optionalString (cfg.clientCaFile != null) |
||||
"--client-ca-file=${cfg.clientCaFile}"} \ |
||||
--disable-admission-plugins=${concatStringsSep "," cfg.disableAdmissionPlugins} \ |
||||
--enable-admission-plugins=${concatStringsSep "," cfg.enableAdmissionPlugins} \ |
||||
--etcd-servers=${concatStringsSep "," cfg.etcd.servers} \ |
||||
${optionalString (cfg.etcd.caFile != null) |
||||
"--etcd-cafile=${cfg.etcd.caFile}"} \ |
||||
${optionalString (cfg.etcd.certFile != null) |
||||
"--etcd-certfile=${cfg.etcd.certFile}"} \ |
||||
${optionalString (cfg.etcd.keyFile != null) |
||||
"--etcd-keyfile=${cfg.etcd.keyFile}"} \ |
||||
${optionalString (cfg.featureGates != []) |
||||
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
||||
${optionalString (cfg.basicAuthFile != null) |
||||
"--basic-auth-file=${cfg.basicAuthFile}"} \ |
||||
--kubelet-https=${boolToString cfg.kubeletHttps} \ |
||||
${optionalString (cfg.kubeletClientCaFile != null) |
||||
"--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"} \ |
||||
${optionalString (cfg.kubeletClientCertFile != null) |
||||
"--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \ |
||||
${optionalString (cfg.kubeletClientKeyFile != null) |
||||
"--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \ |
||||
--insecure-bind-address=${cfg.insecureBindAddress} \ |
||||
--insecure-port=${toString cfg.insecurePort} \ |
||||
${optionalString (cfg.runtimeConfig != "") |
||||
"--runtime-config=${cfg.runtimeConfig}"} \ |
||||
--secure-port=${toString cfg.securePort} \ |
||||
${optionalString (cfg.serviceAccountKeyFile!=null) |
||||
"--service-account-key-file=${cfg.serviceAccountKeyFile}"} \ |
||||
--service-cluster-ip-range=${cfg.serviceClusterIpRange} \ |
||||
--storage-backend=${cfg.storageBackend} \ |
||||
${optionalString (cfg.tlsCertFile != null) |
||||
"--tls-cert-file=${cfg.tlsCertFile}"} \ |
||||
${optionalString (cfg.tlsKeyFile != null) |
||||
"--tls-private-key-file=${cfg.tlsKeyFile}"} \ |
||||
${optionalString (cfg.tokenAuthFile != null) |
||||
"--token-auth-file=${cfg.tokenAuthFile}"} \ |
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
||||
${cfg.extraOpts} |
||||
''; |
||||
WorkingDirectory = top.dataDir; |
||||
User = "kubernetes"; |
||||
Group = "kubernetes"; |
||||
AmbientCapabilities = "cap_net_bind_service"; |
||||
Restart = "on-failure"; |
||||
RestartSec = 5; |
||||
}; |
||||
}; |
||||
|
||||
services.etcd = { |
||||
clientCertAuth = mkDefault true; |
||||
peerClientCertAuth = mkDefault true; |
||||
listenClientUrls = mkDefault ["https://0.0.0.0:2379"]; |
||||
listenPeerUrls = mkDefault ["https://0.0.0.0:2380"]; |
||||
advertiseClientUrls = mkDefault ["https://${top.masterAddress}:2379"]; |
||||
initialCluster = mkDefault ["${top.masterAddress}=https://${top.masterAddress}:2380"]; |
||||
name = top.masterAddress; |
||||
initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"]; |
||||
}; |
||||
|
||||
services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled { |
||||
|
||||
apiserver-kubelet-api-admin-crb = { |
||||
apiVersion = "rbac.authorization.k8s.io/v1"; |
||||
kind = "ClusterRoleBinding"; |
||||
metadata = { |
||||
name = "system:kube-apiserver:kubelet-api-admin"; |
||||
}; |
||||
roleRef = { |
||||
apiGroup = "rbac.authorization.k8s.io"; |
||||
kind = "ClusterRole"; |
||||
name = "system:kubelet-api-admin"; |
||||
}; |
||||
subjects = [{ |
||||
kind = "User"; |
||||
name = "system:kube-apiserver"; |
||||
}]; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
services.kubernetes.pki.certs = with top.lib; { |
||||
apiServer = mkCert { |
||||
name = "kube-apiserver"; |
||||
CN = "kubernetes"; |
||||
hosts = [ |
||||
"kubernetes.default.svc" |
||||
"kubernetes.default.svc.${top.addons.dns.clusterDomain}" |
||||
cfg.advertiseAddress |
||||
top.masterAddress |
||||
apiserverServiceIP |
||||
"127.0.0.1" |
||||
] ++ cfg.extraSANs; |
||||
action = "systemctl restart kube-apiserver.service"; |
||||
}; |
||||
apiserverKubeletClient = mkCert { |
||||
name = "kube-apiserver-kubelet-client"; |
||||
CN = "system:kube-apiserver"; |
||||
action = "systemctl restart kube-apiserver.service"; |
||||
}; |
||||
apiserverEtcdClient = mkCert { |
||||
name = "kube-apiserver-etcd-client"; |
||||
CN = "etcd-client"; |
||||
action = "systemctl restart kube-apiserver.service"; |
||||
}; |
||||
clusterAdmin = mkCert { |
||||
name = "cluster-admin"; |
||||
CN = "cluster-admin"; |
||||
fields = { |
||||
O = "system:masters"; |
||||
}; |
||||
privateKeyOwner = "root"; |
||||
}; |
||||
etcd = mkCert { |
||||
name = "etcd"; |
||||
CN = top.masterAddress; |
||||
hosts = [ |
||||
"etcd.${top.addons.dns.clusterDomain}" |
||||
top.masterAddress |
||||
cfg.advertiseAddress |
||||
]; |
||||
privateKeyOwner = "etcd"; |
||||
action = "systemctl restart etcd.service"; |
||||
}; |
||||
}; |
||||
|
||||
}) |
||||
|
||||
]; |
||||
|
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.controllerManager; |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.controllerManager = with lib.types; { |
||||
|
||||
allocateNodeCIDRs = mkOption { |
||||
description = "Whether to automatically allocate CIDR ranges for cluster nodes."; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
bindAddress = mkOption { |
||||
description = "Kubernetes controller manager listening address."; |
||||
default = "127.0.0.1"; |
||||
type = str; |
||||
}; |
||||
|
||||
clusterCidr = mkOption { |
||||
description = "Kubernetes CIDR Range for Pods in cluster."; |
||||
default = top.clusterCidr; |
||||
type = str; |
||||
}; |
||||
|
||||
enable = mkEnableOption "Kubernetes controller manager."; |
||||
|
||||
extraOpts = mkOption { |
||||
description = "Kubernetes controller manager extra command line options."; |
||||
default = ""; |
||||
type = str; |
||||
}; |
||||
|
||||
featureGates = mkOption { |
||||
description = "List set of feature gates"; |
||||
default = top.featureGates; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
insecurePort = mkOption { |
||||
description = "Kubernetes controller manager insecure listening port."; |
||||
default = 0; |
||||
type = int; |
||||
}; |
||||
|
||||
kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager"; |
||||
|
||||
leaderElect = mkOption { |
||||
description = "Whether to start leader election before executing main loop."; |
||||
type = bool; |
||||
default = true; |
||||
}; |
||||
|
||||
rootCaFile = mkOption { |
||||
description = '' |
||||
Kubernetes controller manager certificate authority file included in |
||||
service account's token secret. |
||||
''; |
||||
default = top.caFile; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
securePort = mkOption { |
||||
description = "Kubernetes controller manager secure listening port."; |
||||
default = 10252; |
||||
type = int; |
||||
}; |
||||
|
||||
serviceAccountKeyFile = mkOption { |
||||
description = '' |
||||
Kubernetes controller manager PEM-encoded private RSA key file used to |
||||
sign service account tokens |
||||
''; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
tlsCertFile = mkOption { |
||||
description = "Kubernetes controller-manager certificate file."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
tlsKeyFile = mkOption { |
||||
description = "Kubernetes controller-manager private key file."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
verbosity = mkOption { |
||||
description = '' |
||||
Optional glog verbosity level for logging statements. See |
||||
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr int; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkIf cfg.enable { |
||||
systemd.services.kube-controller-manager = { |
||||
description = "Kubernetes Controller Manager Service"; |
||||
wantedBy = [ "kubernetes.target" ]; |
||||
after = [ "kube-apiserver.service" ]; |
||||
serviceConfig = { |
||||
RestartSec = "30s"; |
||||
Restart = "on-failure"; |
||||
Slice = "kubernetes.slice"; |
||||
ExecStart = ''${top.package}/bin/kube-controller-manager \ |
||||
--allocate-node-cidrs=${boolToString cfg.allocateNodeCIDRs} \ |
||||
--bind-address=${cfg.bindAddress} \ |
||||
${optionalString (cfg.clusterCidr!=null) |
||||
"--cluster-cidr=${cfg.clusterCidr}"} \ |
||||
${optionalString (cfg.featureGates != []) |
||||
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
||||
--kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \ |
||||
--leader-elect=${boolToString cfg.leaderElect} \ |
||||
${optionalString (cfg.rootCaFile!=null) |
||||
"--root-ca-file=${cfg.rootCaFile}"} \ |
||||
--port=${toString cfg.insecurePort} \ |
||||
--secure-port=${toString cfg.securePort} \ |
||||
${optionalString (cfg.serviceAccountKeyFile!=null) |
||||
"--service-account-private-key-file=${cfg.serviceAccountKeyFile}"} \ |
||||
${optionalString (cfg.tlsCertFile!=null) |
||||
"--tls-cert-file=${cfg.tlsCertFile}"} \ |
||||
${optionalString (cfg.tlsKeyFile!=null) |
||||
"--tls-key-file=${cfg.tlsKeyFile}"} \ |
||||
${optionalString (elem "RBAC" top.apiserver.authorizationMode) |
||||
"--use-service-account-credentials"} \ |
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
||||
${cfg.extraOpts} |
||||
''; |
||||
WorkingDirectory = top.dataDir; |
||||
User = "kubernetes"; |
||||
Group = "kubernetes"; |
||||
}; |
||||
path = top.path; |
||||
}; |
||||
|
||||
services.kubernetes.pki.certs = with top.lib; { |
||||
controllerManager = mkCert { |
||||
name = "kube-controller-manager"; |
||||
CN = "kube-controller-manager"; |
||||
action = "systemctl restart kube-controller-manager.service"; |
||||
}; |
||||
controllerManagerClient = mkCert { |
||||
name = "kube-controller-manager-client"; |
||||
CN = "system:kube-controller-manager"; |
||||
action = "systemctl restart kube-controller-manager.service"; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress; |
||||
}; |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,79 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.flannel; |
||||
|
||||
# needed for flannel to pass options to docker |
||||
mkDockerOpts = pkgs.runCommand "mk-docker-opts" { |
||||
buildInputs = [ pkgs.makeWrapper ]; |
||||
} '' |
||||
mkdir -p $out |
||||
cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh |
||||
|
||||
# bashInteractive needed for `compgen` |
||||
makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh" |
||||
''; |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.flannel = { |
||||
enable = mkEnableOption "enable flannel networking"; |
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkIf cfg.enable { |
||||
services.flannel = { |
||||
|
||||
enable = mkDefault true; |
||||
network = mkDefault top.clusterCidr; |
||||
}; |
||||
|
||||
services.kubernetes.kubelet = { |
||||
networkPlugin = mkDefault "cni"; |
||||
cni.config = mkDefault [{ |
||||
name = "mynet"; |
||||
type = "flannel"; |
||||
delegate = { |
||||
isDefaultGateway = true; |
||||
bridge = "docker0"; |
||||
}; |
||||
}]; |
||||
}; |
||||
|
||||
systemd.services."mk-docker-opts" = { |
||||
description = "Pre-Docker Actions"; |
||||
wantedBy = [ "flannel.service" ]; |
||||
before = [ "docker.service" ]; |
||||
after = [ "flannel.service" ]; |
||||
path = with pkgs; [ gawk gnugrep ]; |
||||
script = '' |
||||
mkdir -p /run/flannel |
||||
${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker |
||||
''; |
||||
serviceConfig.Type = "oneshot"; |
||||
}; |
||||
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker"; |
||||
|
||||
# read environment variables generated by mk-docker-opts |
||||
virtualisation.docker.extraOptions = "$DOCKER_OPTS"; |
||||
|
||||
networking = { |
||||
firewall.allowedUDPPorts = [ |
||||
8285 # flannel udp |
||||
8472 # flannel vxlan |
||||
]; |
||||
dhcpcd.denyInterfaces = [ "docker*" "flannel*" ]; |
||||
}; |
||||
|
||||
services.kubernetes.pki.certs = { |
||||
flannelEtcdClient = top.lib.mkCert { |
||||
name = "flannel-etcd-client"; |
||||
CN = "flannel-etcd-client"; |
||||
action = "systemctl restart flannel.service"; |
||||
}; |
||||
}; |
||||
}; |
||||
} |
||||
@ -0,0 +1,367 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.kubelet; |
||||
|
||||
cniConfig = |
||||
if cfg.cni.config != [] && !(isNull cfg.cni.configDir) then |
||||
throw "Verbatim CNI-config and CNI configDir cannot both be set." |
||||
else if !(isNull cfg.cni.configDir) then |
||||
cfg.cni.configDir |
||||
else |
||||
(pkgs.buildEnv { |
||||
name = "kubernetes-cni-config"; |
||||
paths = imap (i: entry: |
||||
pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) |
||||
) cfg.cni.config; |
||||
}); |
||||
|
||||
infraContainer = pkgs.dockerTools.buildImage { |
||||
name = "pause"; |
||||
tag = "latest"; |
||||
contents = top.package.pause; |
||||
config.Cmd = "/bin/pause"; |
||||
}; |
||||
|
||||
kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig; |
||||
|
||||
manifests = pkgs.buildEnv { |
||||
name = "kubernetes-manifests"; |
||||
paths = mapAttrsToList (name: manifest: |
||||
pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest) |
||||
) cfg.manifests; |
||||
}; |
||||
|
||||
manifestPath = "kubernetes/manifests"; |
||||
|
||||
taintOptions = with lib.types; { name, ... }: { |
||||
options = { |
||||
key = mkOption { |
||||
description = "Key of taint."; |
||||
default = name; |
||||
type = str; |
||||
}; |
||||
value = mkOption { |
||||
description = "Value of taint."; |
||||
type = str; |
||||
}; |
||||
effect = mkOption { |
||||
description = "Effect of taint."; |
||||
example = "NoSchedule"; |
||||
type = enum ["NoSchedule" "PreferNoSchedule" "NoExecute"]; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.taints); |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.kubelet = with lib.types; { |
||||
|
||||
address = mkOption { |
||||
description = "Kubernetes kubelet info server listening address."; |
||||
default = "0.0.0.0"; |
||||
type = str; |
||||
}; |
||||
|
||||
allowPrivileged = mkOption { |
||||
description = "Whether to allow Kubernetes containers to request privileged mode."; |
||||
default = false; |
||||
type = bool; |
||||
}; |
||||
|
||||
clusterDns = mkOption { |
||||
description = "Use alternative DNS."; |
||||
default = "10.1.0.1"; |
||||
type = str; |
||||
}; |
||||
|
||||
clusterDomain = mkOption { |
||||
description = "Use alternative domain."; |
||||
default = config.services.kubernetes.addons.dns.clusterDomain; |
||||
type = str; |
||||
}; |
||||
|
||||
clientCaFile = mkOption { |
||||
description = "Kubernetes apiserver CA file for client authentication."; |
||||
default = top.caFile; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
cni = { |
||||
packages = mkOption { |
||||
description = "List of network plugin packages to install."; |
||||
type = listOf package; |
||||
default = []; |
||||
}; |
||||
|
||||
config = mkOption { |
||||
description = "Kubernetes CNI configuration."; |
||||
type = listOf attrs; |
||||
default = []; |
||||
example = literalExample '' |
||||
[{ |
||||
"cniVersion": "0.2.0", |
||||
"name": "mynet", |
||||
"type": "bridge", |
||||
"bridge": "cni0", |
||||
"isGateway": true, |
||||
"ipMasq": true, |
||||
"ipam": { |
||||
"type": "host-local", |
||||
"subnet": "10.22.0.0/16", |
||||
"routes": [ |
||||
{ "dst": "0.0.0.0/0" } |
||||
] |
||||
} |
||||
} { |
||||
"cniVersion": "0.2.0", |
||||
"type": "loopback" |
||||
}] |
||||
''; |
||||
}; |
||||
|
||||
configDir = mkOption { |
||||
description = "Path to Kubernetes CNI configuration directory."; |
||||
type = nullOr path; |
||||
default = null; |
||||
}; |
||||
}; |
||||
|
||||
enable = mkEnableOption "Kubernetes kubelet."; |
||||
|
||||
extraOpts = mkOption { |
||||
description = "Kubernetes kubelet extra command line options."; |
||||
default = ""; |
||||
type = str; |
||||
}; |
||||
|
||||
featureGates = mkOption { |
||||
description = "List set of feature gates"; |
||||
default = top.featureGates; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
healthz = { |
||||
bind = mkOption { |
||||
description = "Kubernetes kubelet healthz listening address."; |
||||
default = "127.0.0.1"; |
||||
type = str; |
||||
}; |
||||
|
||||
port = mkOption { |
||||
description = "Kubernetes kubelet healthz port."; |
||||
default = 10248; |
||||
type = int; |
||||
}; |
||||
}; |
||||
|
||||
hostname = mkOption { |
||||
description = "Kubernetes kubelet hostname override."; |
||||
default = config.networking.hostName; |
||||
type = str; |
||||
}; |
||||
|
||||
kubeconfig = top.lib.mkKubeConfigOptions "Kubelet"; |
||||
|
||||
manifests = mkOption { |
||||
description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)"; |
||||
type = attrsOf attrs; |
||||
default = {}; |
||||
}; |
||||
|
||||
networkPlugin = mkOption { |
||||
description = "Network plugin to use by Kubernetes."; |
||||
type = nullOr (enum ["cni" "kubenet"]); |
||||
default = "kubenet"; |
||||
}; |
||||
|
||||
nodeIp = mkOption { |
||||
description = "IP address of the node. If set, kubelet will use this IP address for the node."; |
||||
default = null; |
||||
type = nullOr str; |
||||
}; |
||||
|
||||
registerNode = mkOption { |
||||
description = "Whether to auto register kubelet with API server."; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
port = mkOption { |
||||
description = "Kubernetes kubelet info server listening port."; |
||||
default = 10250; |
||||
type = int; |
||||
}; |
||||
|
||||
seedDockerImages = mkOption { |
||||
description = "List of docker images to preload on system"; |
||||
default = []; |
||||
type = listOf package; |
||||
}; |
||||
|
||||
taints = mkOption { |
||||
description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)."; |
||||
default = {}; |
||||
type = attrsOf (submodule [ taintOptions ]); |
||||
}; |
||||
|
||||
tlsCertFile = mkOption { |
||||
description = "File containing x509 Certificate for HTTPS."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
tlsKeyFile = mkOption { |
||||
description = "File containing x509 private key matching tlsCertFile."; |
||||
default = null; |
||||
type = nullOr path; |
||||
}; |
||||
|
||||
unschedulable = mkOption { |
||||
description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role."; |
||||
default = false; |
||||
type = bool; |
||||
}; |
||||
|
||||
verbosity = mkOption { |
||||
description = '' |
||||
Optional glog verbosity level for logging statements. See |
||||
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr int; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkMerge [ |
||||
(mkIf cfg.enable { |
||||
services.kubernetes.kubelet.seedDockerImages = [infraContainer]; |
||||
|
||||
systemd.services.kubelet-bootstrap = { |
||||
description = "Boostrap Kubelet"; |
||||
wantedBy = ["kubernetes.target"]; |
||||
after = ["docker.service" "network.target"]; |
||||
path = with pkgs; [ docker ]; |
||||
script = '' |
||||
${concatMapStrings (img: '' |
||||
echo "Seeding docker image: ${img}" |
||||
docker load <${img} |
||||
'') cfg.seedDockerImages} |
||||
|
||||
rm /opt/cni/bin/* || true |
||||
${concatMapStrings (package: '' |
||||
echo "Linking cni package: ${package}" |
||||
ln -fs ${package}/bin/* /opt/cni/bin |
||||
'') cfg.cni.packages} |
||||
''; |
||||
serviceConfig = { |
||||
Slice = "kubernetes.slice"; |
||||
Type = "oneshot"; |
||||
}; |
||||
}; |
||||
|
||||
systemd.services.kubelet = { |
||||
description = "Kubernetes Kubelet Service"; |
||||
wantedBy = [ "kubernetes.target" ]; |
||||
after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ]; |
||||
path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ top.path; |
||||
serviceConfig = { |
||||
Slice = "kubernetes.slice"; |
||||
CPUAccounting = true; |
||||
MemoryAccounting = true; |
||||
ExecStart = ''${top.package}/bin/kubelet \ |
||||
--address=${cfg.address} \ |
||||
--allow-privileged=${boolToString cfg.allowPrivileged} \ |
||||
--authentication-token-webhook \ |
||||
--authentication-token-webhook-cache-ttl="10s" \ |
||||
--authorization-mode=Webhook \ |
||||
${optionalString (cfg.clientCaFile != null) |
||||
"--client-ca-file=${cfg.clientCaFile}"} \ |
||||
${optionalString (cfg.clusterDns != "") |
||||
"--cluster-dns=${cfg.clusterDns}"} \ |
||||
${optionalString (cfg.clusterDomain != "") |
||||
"--cluster-domain=${cfg.clusterDomain}"} \ |
||||
--cni-conf-dir=${cniConfig} \ |
||||
${optionalString (cfg.featureGates != []) |
||||
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
||||
--hairpin-mode=hairpin-veth \ |
||||
--healthz-bind-address=${cfg.healthz.bind} \ |
||||
--healthz-port=${toString cfg.healthz.port} \ |
||||
--hostname-override=${cfg.hostname} \ |
||||
--kubeconfig=${kubeconfig} \ |
||||
${optionalString (cfg.networkPlugin != null) |
||||
"--network-plugin=${cfg.networkPlugin}"} \ |
||||
${optionalString (cfg.nodeIp != null) |
||||
"--node-ip=${cfg.nodeIp}"} \ |
||||
--pod-infra-container-image=pause \ |
||||
${optionalString (cfg.manifests != {}) |
||||
"--pod-manifest-path=/etc/${manifestPath}"} \ |
||||
--port=${toString cfg.port} \ |
||||
--register-node=${boolToString cfg.registerNode} \ |
||||
${optionalString (taints != "") |
||||
"--register-with-taints=${taints}"} \ |
||||
--root-dir=${top.dataDir} \ |
||||
${optionalString (cfg.tlsCertFile != null) |
||||
"--tls-cert-file=${cfg.tlsCertFile}"} \ |
||||
${optionalString (cfg.tlsKeyFile != null) |
||||
"--tls-private-key-file=${cfg.tlsKeyFile}"} \ |
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
||||
${cfg.extraOpts} |
||||
''; |
||||
WorkingDirectory = top.dataDir; |
||||
}; |
||||
}; |
||||
|
||||
# Allways include cni plugins |
||||
services.kubernetes.kubelet.cni.packages = [pkgs.cni-plugins]; |
||||
|
||||
boot.kernelModules = ["br_netfilter"]; |
||||
|
||||
services.kubernetes.kubelet.hostname = with config.networking; |
||||
mkDefault (hostName + optionalString (!isNull domain) ".${domain}"); |
||||
|
||||
services.kubernetes.pki.certs = with top.lib; { |
||||
kubelet = mkCert { |
||||
name = "kubelet"; |
||||
CN = top.kubelet.hostname; |
||||
action = "systemctl restart kubelet.service"; |
||||
|
||||
}; |
||||
kubeletClient = mkCert { |
||||
name = "kubelet-client"; |
||||
CN = "system:node:${top.kubelet.hostname}"; |
||||
fields = { |
||||
O = "system:nodes"; |
||||
}; |
||||
action = "systemctl restart kubelet.service"; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress; |
||||
}) |
||||
|
||||
(mkIf (cfg.enable && cfg.manifests != {}) { |
||||
environment.etc = mapAttrs' (name: manifest: |
||||
nameValuePair "${manifestPath}/${name}.json" { |
||||
text = builtins.toJSON manifest; |
||||
mode = "0755"; |
||||
} |
||||
) cfg.manifests; |
||||
}) |
||||
|
||||
(mkIf (cfg.unschedulable && cfg.enable) { |
||||
services.kubernetes.kubelet.taints.unschedulable = { |
||||
value = "true"; |
||||
effect = "NoSchedule"; |
||||
}; |
||||
}) |
||||
|
||||
]; |
||||
} |
||||
@ -0,0 +1,374 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.pki; |
||||
|
||||
csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON { |
||||
key = { |
||||
algo = "rsa"; |
||||
size = 2048; |
||||
}; |
||||
names = singleton cfg.caSpec; |
||||
}); |
||||
|
||||
csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON { |
||||
key = { |
||||
algo = "rsa"; |
||||
size = 2048; |
||||
}; |
||||
CN = top.masterAddress; |
||||
}); |
||||
|
||||
cfsslAPITokenBaseName = "apitoken.secret"; |
||||
cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}"; |
||||
certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}"; |
||||
cfsslAPITokenLength = 32; |
||||
|
||||
clusterAdminKubeconfig = with cfg.certs.clusterAdmin; |
||||
top.lib.mkKubeConfig "cluster-admin" { |
||||
server = top.apiserverAddress; |
||||
certFile = cert; |
||||
keyFile = key; |
||||
}; |
||||
|
||||
remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}"; |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.pki = with lib.types; { |
||||
|
||||
enable = mkEnableOption "Whether to enable easyCert issuer service."; |
||||
|
||||
certs = mkOption { |
||||
description = "List of certificate specs to feed to cert generator."; |
||||
default = {}; |
||||
type = attrs; |
||||
}; |
||||
|
||||
genCfsslCACert = mkOption { |
||||
description = '' |
||||
Whether to automatically generate cfssl CA certificate and key, |
||||
if they don't exist. |
||||
''; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
genCfsslAPICerts = mkOption { |
||||
description = '' |
||||
Whether to automatically generate cfssl API webserver TLS cert and key, |
||||
if they don't exist. |
||||
''; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
genCfsslAPIToken = mkOption { |
||||
description = '' |
||||
Whether to automatically generate cfssl API-token secret, |
||||
if they doesn't exist. |
||||
''; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
pkiTrustOnBootstrap = mkOption { |
||||
description = "Whether to always trust remote cfssl server upon initial PKI bootstrap."; |
||||
default = true; |
||||
type = bool; |
||||
}; |
||||
|
||||
caCertPathPrefix = mkOption { |
||||
description = '' |
||||
Path-prefrix for the CA-certificate to be used for cfssl signing. |
||||
Suffixes ".pem" and "-key.pem" will be automatically appended for |
||||
the public and private keys respectively. |
||||
''; |
||||
default = "${config.services.cfssl.dataDir}/ca"; |
||||
type = str; |
||||
}; |
||||
|
||||
caSpec = mkOption { |
||||
description = "Certificate specification for the auto-generated CAcert."; |
||||
default = { |
||||
CN = "kubernetes-cluster-ca"; |
||||
O = "NixOS"; |
||||
OU = "services.kubernetes.pki.caSpec"; |
||||
L = "auto-generated"; |
||||
}; |
||||
type = attrs; |
||||
}; |
||||
|
||||
etcClusterAdminKubeconfig = mkOption { |
||||
description = '' |
||||
Symlink a kubeconfig with cluster-admin privileges to environment path |
||||
(/etc/<path>). |
||||
''; |
||||
default = null; |
||||
type = nullOr str; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkIf cfg.enable |
||||
(let |
||||
cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl"; |
||||
cfsslCert = "${cfsslCertPathPrefix}.pem"; |
||||
cfsslKey = "${cfsslCertPathPrefix}-key.pem"; |
||||
in |
||||
{ |
||||
|
||||
services.cfssl = mkIf (top.apiserver.enable) { |
||||
enable = true; |
||||
address = "0.0.0.0"; |
||||
tlsCert = cfsslCert; |
||||
tlsKey = cfsslKey; |
||||
configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON { |
||||
signing = { |
||||
profiles = { |
||||
default = { |
||||
usages = ["digital signature"]; |
||||
auth_key = "default"; |
||||
expiry = "720h"; |
||||
}; |
||||
}; |
||||
}; |
||||
auth_keys = { |
||||
default = { |
||||
type = "standard"; |
||||
key = "file:${cfsslAPITokenPath}"; |
||||
}; |
||||
}; |
||||
})); |
||||
}; |
||||
|
||||
systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable) |
||||
(concatStringsSep "\n" [ |
||||
"set -e" |
||||
(optionalString cfg.genCfsslCACert '' |
||||
if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then |
||||
${cfssl}/bin/cfssl genkey -initca ${csrCA} | \ |
||||
${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix} |
||||
fi |
||||
'') |
||||
(optionalString cfg.genCfsslAPICerts '' |
||||
if [ ! -f "${dataDir}/cfssl.pem" ]; then |
||||
${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \ |
||||
${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix} |
||||
fi |
||||
'') |
||||
(optionalString cfg.genCfsslAPIToken '' |
||||
if [ ! -f "${cfsslAPITokenPath}" ]; then |
||||
head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}" |
||||
fi |
||||
chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}" |
||||
'')]); |
||||
|
||||
systemd.services.kube-certmgr-bootstrap = { |
||||
description = "Kubernetes certmgr bootstrapper"; |
||||
wantedBy = [ "certmgr.service" ]; |
||||
after = [ "cfssl.target" ]; |
||||
script = concatStringsSep "\n" ['' |
||||
set -e |
||||
|
||||
# If there's a cfssl (cert issuer) running locally, then don't rely on user to |
||||
# manually paste it in place. Just symlink. |
||||
# otherwise, create the target file, ready for users to insert the token |
||||
|
||||
if [ -f "${cfsslAPITokenPath}" ]; then |
||||
ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}" |
||||
else |
||||
touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}" |
||||
fi |
||||
'' |
||||
(optionalString (cfg.pkiTrustOnBootstrap) '' |
||||
if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then |
||||
${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \ |
||||
${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile} |
||||
fi |
||||
'') |
||||
]; |
||||
serviceConfig = { |
||||
RestartSec = "10s"; |
||||
Restart = "on-failure"; |
||||
}; |
||||
}; |
||||
|
||||
services.certmgr = { |
||||
enable = true; |
||||
svcManager = "command"; |
||||
specs = |
||||
let |
||||
mkSpec = _: cert: { |
||||
inherit (cert) action; |
||||
authority = { |
||||
inherit remote; |
||||
file.path = cert.caCert; |
||||
root_ca = cert.caCert; |
||||
profile = "default"; |
||||
auth_key_file = certmgrAPITokenPath; |
||||
}; |
||||
certificate = { |
||||
path = cert.cert; |
||||
}; |
||||
private_key = cert.privateKeyOptions; |
||||
request = { |
||||
inherit (cert) CN hosts; |
||||
key = { |
||||
algo = "rsa"; |
||||
size = 2048; |
||||
}; |
||||
names = [ cert.fields ]; |
||||
}; |
||||
}; |
||||
in |
||||
mapAttrs mkSpec cfg.certs; |
||||
}; |
||||
|
||||
#TODO: Get rid of kube-addon-manager in the future for the following reasons |
||||
# - it is basically just a shell script wrapped around kubectl |
||||
# - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount |
||||
# - it is designed to be used with k8s system components only |
||||
# - it would be better with a more Nix-oriented way of managing addons |
||||
systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [{ |
||||
environment.KUBECONFIG = with cfg.certs.addonManager; |
||||
top.lib.mkKubeConfig "addon-manager" { |
||||
server = top.apiserverAddress; |
||||
certFile = cert; |
||||
keyFile = key; |
||||
}; |
||||
} |
||||
|
||||
(optionalAttrs (top.addonManager.bootstrapAddons != {}) { |
||||
serviceConfig.PermissionsStartOnly = true; |
||||
preStart = with pkgs; |
||||
let |
||||
files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v)) |
||||
top.addonManager.bootstrapAddons; |
||||
in |
||||
'' |
||||
export KUBECONFIG=${clusterAdminKubeconfig} |
||||
${kubectl}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files} |
||||
''; |
||||
})]); |
||||
|
||||
environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (!isNull cfg.etcClusterAdminKubeconfig) |
||||
clusterAdminKubeconfig; |
||||
|
||||
environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [ |
||||
(pkgs.writeScriptBin "nixos-kubernetes-node-join" '' |
||||
set -e |
||||
exec 1>&2 |
||||
|
||||
if [ $# -gt 0 ]; then |
||||
echo "Usage: $(basename $0)" |
||||
echo "" |
||||
echo "No args. Apitoken must be provided on stdin." |
||||
echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node." |
||||
exit 1 |
||||
fi |
||||
|
||||
if [ $(id -u) != 0 ]; then |
||||
echo "Run as root please." |
||||
exit 1 |
||||
fi |
||||
|
||||
read -r token |
||||
if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then |
||||
echo "Token must be of length ${toString cfsslAPITokenLength}." |
||||
exit 1 |
||||
fi |
||||
|
||||
echo $token > ${certmgrAPITokenPath} |
||||
chmod 600 ${certmgrAPITokenPath} |
||||
|
||||
echo "Restarting certmgr..." >&1 |
||||
systemctl restart certmgr |
||||
|
||||
echo "Waiting for certs to appear..." >&1 |
||||
|
||||
${optionalString top.kubelet.enable '' |
||||
while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done |
||||
echo "Restarting kubelet..." >&1 |
||||
systemctl restart kubelet |
||||
''} |
||||
|
||||
${optionalString top.proxy.enable '' |
||||
while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done |
||||
echo "Restarting kube-proxy..." >&1 |
||||
systemctl restart kube-proxy |
||||
''} |
||||
|
||||
${optionalString top.flannel.enable '' |
||||
while [ ! -f ${cfg.certs.flannelEtcdClient.cert} ]; do sleep 1; done |
||||
echo "Restarting flannel..." >&1 |
||||
systemctl restart flannel |
||||
''} |
||||
|
||||
echo "Node joined succesfully" |
||||
'')]; |
||||
|
||||
services.etcd = with cfg.certs.etcd; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
trustedCaFile = mkDefault caCert; |
||||
}; |
||||
|
||||
services.flannel.etcd = with cfg.certs.flannelEtcdClient; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
caFile = mkDefault caCert; |
||||
}; |
||||
|
||||
services.kubernetes = { |
||||
|
||||
apiserver = mkIf top.apiserver.enable (with cfg.certs.apiServer; { |
||||
etcd = with cfg.certs.apiserverEtcdClient; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
caFile = mkDefault caCert; |
||||
}; |
||||
clientCaFile = mkDefault caCert; |
||||
tlsCertFile = mkDefault cert; |
||||
tlsKeyFile = mkDefault key; |
||||
serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert; |
||||
kubeletClientCaFile = mkDefault caCert; |
||||
kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert; |
||||
kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key; |
||||
}); |
||||
controllerManager = mkIf top.controllerManager.enable { |
||||
serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key; |
||||
rootCaFile = cfg.certs.controllerManagerClient.caCert; |
||||
kubeconfig = with cfg.certs.controllerManagerClient; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
}; |
||||
}; |
||||
scheduler = mkIf top.scheduler.enable { |
||||
kubeconfig = with cfg.certs.schedulerClient; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
}; |
||||
}; |
||||
kubelet = mkIf top.kubelet.enable { |
||||
clientCaFile = mkDefault cfg.certs.kubelet.caCert; |
||||
tlsCertFile = mkDefault cfg.certs.kubelet.cert; |
||||
tlsKeyFile = mkDefault cfg.certs.kubelet.key; |
||||
kubeconfig = with cfg.certs.kubeletClient; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
}; |
||||
}; |
||||
proxy = mkIf top.proxy.enable { |
||||
kubeconfig = with cfg.certs.kubeProxyClient; { |
||||
certFile = mkDefault cert; |
||||
keyFile = mkDefault key; |
||||
}; |
||||
}; |
||||
}; |
||||
}); |
||||
} |
||||
@ -0,0 +1,80 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.proxy; |
||||
in |
||||
{ |
||||
|
||||
###### interface |
||||
options.services.kubernetes.proxy = with lib.types; { |
||||
|
||||
bindAddress = mkOption { |
||||
description = "Kubernetes proxy listening address."; |
||||
default = "0.0.0.0"; |
||||
type = str; |
||||
}; |
||||
|
||||
enable = mkEnableOption "Whether to enable Kubernetes proxy."; |
||||
|
||||
extraOpts = mkOption { |
||||
description = "Kubernetes proxy extra command line options."; |
||||
default = ""; |
||||
type = str; |
||||
}; |
||||
|
||||
featureGates = mkOption { |
||||
description = "List set of feature gates"; |
||||
default = top.featureGates; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes proxy"; |
||||
|
||||
verbosity = mkOption { |
||||
description = '' |
||||
Optional glog verbosity level for logging statements. See |
||||
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr int; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkIf cfg.enable { |
||||
systemd.services.kube-proxy = { |
||||
description = "Kubernetes Proxy Service"; |
||||
wantedBy = [ "kubernetes.target" ]; |
||||
after = [ "kube-apiserver.service" ]; |
||||
path = with pkgs; [ iptables conntrack_tools ]; |
||||
serviceConfig = { |
||||
Slice = "kubernetes.slice"; |
||||
ExecStart = ''${top.package}/bin/kube-proxy \ |
||||
--bind-address=${cfg.bindAddress} \ |
||||
${optionalString (top.clusterCidr!=null) |
||||
"--cluster-cidr=${top.clusterCidr}"} \ |
||||
${optionalString (cfg.featureGates != []) |
||||
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
||||
--kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \ |
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
||||
${cfg.extraOpts} |
||||
''; |
||||
WorkingDirectory = top.dataDir; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.pki.certs = { |
||||
kubeProxyClient = top.lib.mkCert { |
||||
name = "kube-proxy-client"; |
||||
CN = "system:kube-proxy"; |
||||
action = "systemctl restart kube-proxy.service"; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.proxy.kubeconfig.server = mkDefault top.apiserverAddress; |
||||
}; |
||||
} |
||||
@ -0,0 +1,92 @@ |
||||
{ config, lib, pkgs, ... }: |
||||
|
||||
with lib; |
||||
|
||||
let |
||||
top = config.services.kubernetes; |
||||
cfg = top.scheduler; |
||||
in |
||||
{ |
||||
###### interface |
||||
options.services.kubernetes.scheduler = with lib.types; { |
||||
|
||||
address = mkOption { |
||||
description = "Kubernetes scheduler listening address."; |
||||
default = "127.0.0.1"; |
||||
type = str; |
||||
}; |
||||
|
||||
enable = mkEnableOption "Whether to enable Kubernetes scheduler."; |
||||
|
||||
extraOpts = mkOption { |
||||
description = "Kubernetes scheduler extra command line options."; |
||||
default = ""; |
||||
type = str; |
||||
}; |
||||
|
||||
featureGates = mkOption { |
||||
description = "List set of feature gates"; |
||||
default = top.featureGates; |
||||
type = listOf str; |
||||
}; |
||||
|
||||
kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler"; |
||||
|
||||
leaderElect = mkOption { |
||||
description = "Whether to start leader election before executing main loop."; |
||||
type = bool; |
||||
default = true; |
||||
}; |
||||
|
||||
port = mkOption { |
||||
description = "Kubernetes scheduler listening port."; |
||||
default = 10251; |
||||
type = int; |
||||
}; |
||||
|
||||
verbosity = mkOption { |
||||
description = '' |
||||
Optional glog verbosity level for logging statements. See |
||||
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/> |
||||
''; |
||||
default = null; |
||||
type = nullOr int; |
||||
}; |
||||
|
||||
}; |
||||
|
||||
###### implementation |
||||
config = mkIf cfg.enable { |
||||
systemd.services.kube-scheduler = { |
||||
description = "Kubernetes Scheduler Service"; |
||||
wantedBy = [ "kubernetes.target" ]; |
||||
after = [ "kube-apiserver.service" ]; |
||||
serviceConfig = { |
||||
Slice = "kubernetes.slice"; |
||||
ExecStart = ''${top.package}/bin/kube-scheduler \ |
||||
--address=${cfg.address} \ |
||||
${optionalString (cfg.featureGates != []) |
||||
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ |
||||
--kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \ |
||||
--leader-elect=${boolToString cfg.leaderElect} \ |
||||
--port=${toString cfg.port} \ |
||||
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ |
||||
${cfg.extraOpts} |
||||
''; |
||||
WorkingDirectory = top.dataDir; |
||||
User = "kubernetes"; |
||||
Group = "kubernetes"; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.pki.certs = { |
||||
schedulerClient = top.lib.mkCert { |
||||
name = "kube-scheduler-client"; |
||||
CN = "system:kube-scheduler"; |
||||
action = "systemctl restart kube-scheduler.service"; |
||||
}; |
||||
}; |
||||
|
||||
services.kubernetes.scheduler.kubeconfig.server = mkDefault top.apiserverAddress; |
||||
}; |
||||
} |
||||
@ -1,219 +0,0 @@ |
||||
{ |
||||
pkgs ? import <nixpkgs> {}, |
||||
externalDomain ? "myawesomecluster.cluster.yourdomain.net", |
||||
serviceClusterIp ? "10.0.0.1", |
||||
kubelets, |
||||
... |
||||
}: |
||||
let |
||||
runWithCFSSL = name: cmd: |
||||
let secrets = pkgs.runCommand "${name}-cfss.json" { |
||||
buildInputs = [ pkgs.cfssl pkgs.jq ]; |
||||
outputs = [ "out" "cert" "key" "csr" ]; |
||||
} |
||||
'' |
||||
( |
||||
echo "${cmd}" |
||||
cfssl ${cmd} > tmp |
||||
cat tmp | jq -r .key > $key |
||||
cat tmp | jq -r .cert > $cert |
||||
cat tmp | jq -r .csr > $csr |
||||
|
||||
touch $out |
||||
) 2>&1 | fold -w 80 -s |
||||
''; |
||||
in { |
||||
key = secrets.key; |
||||
cert = secrets.cert; |
||||
csr = secrets.csr; |
||||
}; |
||||
|
||||
writeCFSSL = content: |
||||
pkgs.runCommand content.name { |
||||
buildInputs = [ pkgs.cfssl pkgs.jq ]; |
||||
} '' |
||||
mkdir -p $out |
||||
cd $out |
||||
|
||||
json=${pkgs.lib.escapeShellArg (builtins.toJSON content)} |
||||
|
||||
# for a given $field in the $json, treat the associated value as a |
||||
# file path and substitute the contents thereof into the $json |
||||
# object. |
||||
expandFileField() { |
||||
local field=$1 |
||||
if jq -e --arg field "$field" 'has($field)'; then |
||||
local path="$(echo "$json" | jq -r ".$field")" |
||||
json="$(echo "$json" | jq --arg val "$(cat "$path")" ".$field = \$val")" |
||||
fi |
||||
} |
||||
|
||||
expandFileField key |
||||
expandFileField ca |
||||
expandFileField cert |
||||
|
||||
echo "$json" | cfssljson -bare ${content.name} |
||||
''; |
||||
|
||||
noCSR = content: pkgs.lib.filterAttrs (n: v: n != "csr") content; |
||||
noKey = content: pkgs.lib.filterAttrs (n: v: n != "key") content; |
||||
|
||||
writeFile = content: |
||||
if pkgs.lib.isDerivation content |
||||
then content |
||||
else pkgs.writeText "content" (builtins.toJSON content); |
||||
|
||||
createServingCertKey = { ca, cn, hosts? [], size ? 2048, name ? cn }: |
||||
noCSR ( |
||||
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=server -config=${writeFile ca.config} ${writeFile { |
||||
CN = cn; |
||||
hosts = hosts; |
||||
key = { algo = "rsa"; inherit size; }; |
||||
}}") // { inherit name; } |
||||
); |
||||
|
||||
createClientCertKey = { ca, cn, groups ? [], size ? 2048, name ? cn }: |
||||
noCSR ( |
||||
(runWithCFSSL name "gencert -ca=${writeFile ca.cert} -ca-key=${writeFile ca.key} -profile=client -config=${writeFile ca.config} ${writeFile { |
||||
CN = cn; |
||||
names = map (group: {O = group;}) groups; |
||||
hosts = [""]; |
||||
key = { algo = "rsa"; inherit size; }; |
||||
}}") // { inherit name; } |
||||
); |
||||
|
||||
createSigningCertKey = { C ? "xx", ST ? "x", L ? "x", O ? "x", OU ? "x", CN ? "ca", emailAddress ? "x", expiry ? "43800h", size ? 2048, name ? CN }: |
||||
(noCSR (runWithCFSSL CN "genkey -initca ${writeFile { |
||||
key = { algo = "rsa"; inherit size; }; |
||||
names = [{ inherit C ST L O OU CN emailAddress; }]; |
||||
}}")) // { |
||||
inherit name; |
||||
config.signing = { |
||||
default.expiry = expiry; |
||||
profiles = { |
||||
server = { |
||||
inherit expiry; |
||||
usages = [ |
||||
"signing" |
||||
"key encipherment" |
||||
"server auth" |
||||
]; |
||||
}; |
||||
client = { |
||||
inherit expiry; |
||||
usages = [ |
||||
"signing" |
||||
"key encipherment" |
||||
"client auth" |
||||
]; |
||||
}; |
||||
peer = { |
||||
inherit expiry; |
||||
usages = [ |
||||
"signing" |
||||
"key encipherment" |
||||
"server auth" |
||||
"client auth" |
||||
]; |
||||
}; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
ca = createSigningCertKey {}; |
||||
|
||||
kube-apiserver = createServingCertKey { |
||||
inherit ca; |
||||
cn = "kube-apiserver"; |
||||
hosts = ["kubernetes.default" "kubernetes.default.svc" "localhost" "api.${externalDomain}" serviceClusterIp]; |
||||
}; |
||||
|
||||
kubelet = createServingCertKey { |
||||
inherit ca; |
||||
cn = "kubelet"; |
||||
hosts = ["*.${externalDomain}"]; |
||||
}; |
||||
|
||||
service-accounts = createServingCertKey { |
||||
inherit ca; |
||||
cn = "kube-service-accounts"; |
||||
}; |
||||
|
||||
etcd = createServingCertKey { |
||||
inherit ca; |
||||
cn = "etcd"; |
||||
hosts = ["etcd.${externalDomain}"]; |
||||
}; |
||||
|
||||
etcd-client = createClientCertKey { |
||||
inherit ca; |
||||
cn = "etcd-client"; |
||||
}; |
||||
|
||||
kubelet-client = createClientCertKey { |
||||
inherit ca; |
||||
cn = "kubelet-client"; |
||||
groups = ["system:masters"]; |
||||
}; |
||||
|
||||
apiserver-client = { |
||||
kubelet = hostname: createClientCertKey { |
||||
inherit ca; |
||||
name = "apiserver-client-kubelet-${hostname}"; |
||||
cn = "system:node:${hostname}.${externalDomain}"; |
||||
groups = ["system:nodes"]; |
||||
}; |
||||
|
||||
kube-proxy = createClientCertKey { |
||||
inherit ca; |
||||
name = "apiserver-client-kube-proxy"; |
||||
cn = "system:kube-proxy"; |
||||
groups = ["system:kube-proxy" "system:nodes"]; |
||||
}; |
||||
|
||||
kube-controller-manager = createClientCertKey { |
||||
inherit ca; |
||||
name = "apiserver-client-kube-controller-manager"; |
||||
cn = "system:kube-controller-manager"; |
||||
groups = ["system:masters"]; |
||||
}; |
||||
|
||||
kube-scheduler = createClientCertKey { |
||||
inherit ca; |
||||
name = "apiserver-client-kube-scheduler"; |
||||
cn = "system:kube-scheduler"; |
||||
groups = ["system:kube-scheduler"]; |
||||
}; |
||||
|
||||
admin = createClientCertKey { |
||||
inherit ca; |
||||
cn = "admin"; |
||||
groups = ["system:masters"]; |
||||
}; |
||||
}; |
||||
in { |
||||
master = pkgs.buildEnv { |
||||
name = "master-keys"; |
||||
paths = [ |
||||
(writeCFSSL (noKey ca)) |
||||
(writeCFSSL kube-apiserver) |
||||
(writeCFSSL kubelet-client) |
||||
(writeCFSSL apiserver-client.kube-controller-manager) |
||||
(writeCFSSL apiserver-client.kube-scheduler) |
||||
(writeCFSSL service-accounts) |
||||
(writeCFSSL etcd) |
||||
]; |
||||
}; |
||||
|
||||
worker = pkgs.buildEnv { |
||||
name = "worker-keys"; |
||||
paths = [ |
||||
(writeCFSSL (noKey ca)) |
||||
(writeCFSSL kubelet) |
||||
(writeCFSSL apiserver-client.kube-proxy) |
||||
(writeCFSSL etcd-client) |
||||
] ++ map (hostname: writeCFSSL (apiserver-client.kubelet hostname)) kubelets; |
||||
}; |
||||
|
||||
admin = writeCFSSL apiserver-client.admin; |
||||
} |
||||
@ -1,57 +0,0 @@ |
||||
{ roles, config, pkgs, certs }: |
||||
with pkgs.lib; |
||||
let |
||||
base = { |
||||
inherit roles; |
||||
flannel.enable = true; |
||||
addons.dashboard.enable = true; |
||||
|
||||
caFile = "${certs.master}/ca.pem"; |
||||
apiserver = { |
||||
tlsCertFile = "${certs.master}/kube-apiserver.pem"; |
||||
tlsKeyFile = "${certs.master}/kube-apiserver-key.pem"; |
||||
kubeletClientCertFile = "${certs.master}/kubelet-client.pem"; |
||||
kubeletClientKeyFile = "${certs.master}/kubelet-client-key.pem"; |
||||
serviceAccountKeyFile = "${certs.master}/kube-service-accounts.pem"; |
||||
}; |
||||
etcd = { |
||||
servers = ["https://etcd.${config.networking.domain}:2379"]; |
||||
certFile = "${certs.worker}/etcd-client.pem"; |
||||
keyFile = "${certs.worker}/etcd-client-key.pem"; |
||||
}; |
||||
kubeconfig = { |
||||
server = "https://api.${config.networking.domain}"; |
||||
}; |
||||
kubelet = { |
||||
tlsCertFile = "${certs.worker}/kubelet.pem"; |
||||
tlsKeyFile = "${certs.worker}/kubelet-key.pem"; |
||||
hostname = "${config.networking.hostName}.${config.networking.domain}"; |
||||
kubeconfig = { |
||||
certFile = "${certs.worker}/apiserver-client-kubelet-${config.networking.hostName}.pem"; |
||||
keyFile = "${certs.worker}/apiserver-client-kubelet-${config.networking.hostName}-key.pem"; |
||||
}; |
||||
}; |
||||
controllerManager = { |
||||
serviceAccountKeyFile = "${certs.master}/kube-service-accounts-key.pem"; |
||||
kubeconfig = { |
||||
certFile = "${certs.master}/apiserver-client-kube-controller-manager.pem"; |
||||
keyFile = "${certs.master}/apiserver-client-kube-controller-manager-key.pem"; |
||||
}; |
||||
}; |
||||
scheduler = { |
||||
kubeconfig = { |
||||
certFile = "${certs.master}/apiserver-client-kube-scheduler.pem"; |
||||
keyFile = "${certs.master}/apiserver-client-kube-scheduler-key.pem"; |
||||
}; |
||||
}; |
||||
proxy = { |
||||
kubeconfig = { |
||||
certFile = "${certs.worker}/apiserver-client-kube-proxy.pem"; |
||||
keyFile = "${certs.worker}//apiserver-client-kube-proxy-key.pem"; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
in { |
||||
services.kubernetes = base; |
||||
} |
||||
Loading…
Reference in new issue