kubernetes: update service

nixos/modules/module-list.nix

@@ -157,7 +157,7 @@
   ./services/backup/tarsnap.nix
   ./services/backup/znapzend.nix
   ./services/cluster/fleet.nix
-  ./services/cluster/kubernetes.nix
+  ./services/cluster/kubernetes/default.nix
   ./services/cluster/panamax.nix
   ./services/computing/boinc/client.nix
   ./services/computing/torque/server.nix

nixos/modules/services/cluster/kubernetes/dashboard.nix

@@ -0,0 +1,85 @@
+{ cfg }: {
+    "dashboard-controller" = {
+        "apiVersion" = "extensions/v1beta1";
+        "kind" = "Deployment";
+        "metadata" = {
+            "labels" = {
+                "addonmanager.kubernetes.io/mode" = "Reconcile";
+                "k8s-app" = "kubernetes-dashboard";
+                "kubernetes.io/cluster-service" = "true";
+            };
+            "name" = "kubernetes-dashboard";
+            "namespace" = "kube-system";
+        };
+        "spec" = {
+            "selector" = {
+                "matchLabels" = {
+                    "k8s-app" = "kubernetes-dashboard";
+                };
+            };
+            "template" = {
+                "metadata" = {
+                    "annotations" = {
+                        "scheduler.alpha.kubernetes.io/critical-pod" = "";
+                    };
+                    "labels" = {
+                        "k8s-app" = "kubernetes-dashboard";
+                    };
+                };
+                "spec" = {
+                    "containers" = [{
+                        "image" = "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.0";
+                        "livenessProbe" = {
+                            "httpGet" = {
+                                "path" = "/";
+                                "port" = 9090;
+                            };
+                            "initialDelaySeconds" = 30;
+                            "timeoutSeconds" = 30;
+                        };
+                        "name" = "kubernetes-dashboard";
+                        "ports" = [{
+                            "containerPort" = 9090;
+                        }];
+                        "resources" = {
+                            "limits" = {
+                                "cpu" = "100m";
+                                "memory" = "50Mi";
+                            };
+                            "requests" = {
+                                "cpu" = "100m";
+                                "memory" = "50Mi";
+                            };
+                        };
+                    }];
+                    "tolerations" = [{
+                        "key" = "CriticalAddonsOnly";
+                        "operator" = "Exists";
+                    }];
+                };
+            };
+        };
+    };
+    "dashboard-service" = {
+        "apiVersion" = "v1";
+        "kind" = "Service";
+        "metadata" = {
+            "labels" = {
+                "addonmanager.kubernetes.io/mode" = "Reconcile";
+                "k8s-app" = "kubernetes-dashboard";
+                "kubernetes.io/cluster-service" = "true";
+            };
+            "name" = "kubernetes-dashboard";
+            "namespace" = "kube-system";
+        };
+        "spec" = {
+            "ports" = [{
+                "port" = 80;
+                "targetPort" = 9090;
+            }];
+            "selector" = {
+                "k8s-app" = "kubernetes-dashboard";
+            };
+        };
+    };
+}

nixos/modules/services/cluster/kubernetes/default.nix

@@ -53,6 +53,35 @@ let
     ) cfg.kubelet.manifests;
   };
 
+  addons = pkgs.runCommand "kubernetes-addons" { } ''
+    mkdir -p $out
+    # since we are mounting the addons to the addon manager, they need to be copied
+    ${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon:
+      pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)
+    ) (cfg.addonManager.addons))}
+  '';
+
+  taintOptions = { name, ... }: {
+    options = {
+      key = mkOption {
+        description = "Key of taint.";
+        default = name;
+        type = types.str;
+      };
+      value = mkOption {
+        description = "Value of taint.";
+        type = types.str;
+      };
+      effect = mkOption {
+        description = "Effect of taint.";
+        example = "NoSchedule";
+        type = types.str;
+      };
+    };
+  };
+
+  taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints);
+
 in {
 
   ###### interface
@@ -77,7 +106,7 @@ in {
     };
 
     verbose = mkOption {
-      description = "Kubernetes enable verbose mode for debugging";
+      description = "Kubernetes enable verbose mode for debugging.";
       default = false;
       type = types.bool;
     };
@@ -90,19 +119,19 @@ in {
       };
 
       keyFile = mkOption {
-        description = "Etcd key file";
+        description = "Etcd key file.";
         default = null;
         type = types.nullOr types.path;
       };
 
       certFile = mkOption {
-        description = "Etcd cert file";
+        description = "Etcd cert file.";
         default = null;
         type = types.nullOr types.path;
       };
 
       caFile = mkOption {
-        description = "Etcd ca file";
+        description = "Etcd ca file.";
         default = null;
         type = types.nullOr types.path;
       };
@@ -110,25 +139,25 @@ in {
 
     kubeconfig = {
       server = mkOption {
-        description = "Kubernetes apiserver server address";
+        description = "Kubernetes apiserver server address.";
         default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
         type = types.str;
       };
 
       caFile = mkOption {
-        description = "Certificate authrority file to use to connect to kuberentes apiserver";
+        description = "Certificate authrority file to use to connect to Kubernetes apiserver.";
         type = types.nullOr types.path;
         default = null;
       };
 
       certFile = mkOption {
-        description = "Client certificate file to use to connect to kubernetes";
+        description = "Client certificate file to use to connect to Kubernetes.";
         type = types.nullOr types.path;
         default = null;
       };
 
       keyFile = mkOption {
-        description = "Client key file to use to connect to kubernetes";
+        description = "Client key file to use to connect to Kubernetes.";
         type = types.nullOr types.path;
         default = null;
       };
@@ -142,7 +171,7 @@ in {
 
     apiserver = {
       enable = mkOption {
-        description = "Whether to enable kubernetes apiserver.";
+        description = "Whether to enable Kubernetes apiserver.";
         default = false;
         type = types.bool;
       };
@@ -172,6 +201,14 @@ in {
         type = types.nullOr types.str;
       };
 
+      storageBackend = mkOption {
+        description = ''
+          Kubernetes apiserver storage backend.
+        '';
+        default = "etcd2";
+        type = types.enum ["etcd2" "etcd3"];
+      };
+
       port = mkOption {
         description = "Kubernetes apiserver listening port.";
         default = 8080;
@@ -227,7 +264,7 @@ in {
           Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See
           <link xlink:href="http://kubernetes.io/docs/admin/authorization.html"/>
         '';
-        default = ["ABAC" "RBAC"];
+        default = ["ABAC"];
         type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC"]);
       };
 
@@ -274,7 +311,29 @@ in {
             apiVersion = "abac.authorization.kubernetes.io/v1beta1";
             kind = "Policy";
             spec = {
-              user  = "kube";
+              user  = "kube-worker";
+              namespace = "*";
+              resource = "*";
+              apiGroup = "*";
+              nonResourcePath = "*";
+            };
+          }
+          {
+            apiVersion = "abac.authorization.kubernetes.io/v1beta1";
+            kind = "Policy";
+            spec = {
+              user  = "kube_proxy";
+              namespace = "*";
+              resource = "*";
+              apiGroup = "*";
+              nonResourcePath = "*";
+            };
+          }
+          {
+            apiVersion = "abac.authorization.kubernetes.io/v1beta1";
+            kind = "Policy";
+            spec = {
+              user  = "client";
               namespace = "*";
               resource = "*";
               apiGroup = "*";
@@ -285,7 +344,7 @@ in {
             apiVersion = "abac.authorization.kubernetes.io/v1beta1";
             kind = "Policy";
             spec = {
-              user  = "system:serviceaccount:kube-system:default";
+              group  = "system:serviceaccounts";
               namespace = "*";
               resource = "*";
               apiGroup = "*";
@@ -297,19 +356,19 @@ in {
       };
 
       autorizationRBACSuperAdmin = mkOption {
-        description = "Role based authorization super admin";
+        description = "Role based authorization super admin.";
         default = "admin";
         type = types.str;
       };
 
       allowPrivileged = mkOption {
-        description = "Whether to allow privileged containers on kubernetes.";
+        description = "Whether to allow privileged containers on Kubernetes.";
         default = true;
         type = types.bool;
       };
 
       portalNet = mkOption {
-        description = "Kubernetes CIDR notation IP range from which to assign portal IPs";
+        description = "Kubernetes CIDR notation IP range from which to assign portal IPs.";
         default = "10.10.10.10/24";
         type = types.str;
       };
@@ -319,7 +378,7 @@ in {
           Api runtime configuration. See
           <link xlink:href="http://kubernetes.io/v1.0/docs/admin/cluster-management.html"/>
         '';
-        default = "rbac.authorization.k8s.io/v1alpha1";
+        default = "";
         example = "api/all=false,api/v1=true";
         type = types.str;
       };
@@ -348,25 +407,25 @@ in {
       };
 
       kubeletClientCaFile = mkOption {
-        description = "Path to a cert file for connecting to kubelet";
+        description = "Path to a cert file for connecting to kubelet.";
         default = null;
         type = types.nullOr types.path;
       };
 
       kubeletClientCertFile = mkOption {
-        description = "Client certificate to use for connections to kubelet";
+        description = "Client certificate to use for connections to kubelet.";
         default = null;
         type = types.nullOr types.path;
       };
 
       kubeletClientKeyFile = mkOption {
-        description = "Key to use for connections to kubelet";
+        description = "Key to use for connections to kubelet.";
         default = null;
         type = types.nullOr types.path;
       };
 
       kubeletHttps = mkOption {
-        description = "Whether to use https for connections to kubelet";
+        description = "Whether to use https for connections to kubelet.";
         default = true;
         type = types.bool;
       };
@@ -380,7 +439,7 @@ in {
 
     scheduler = {
       enable = mkOption {
-        description = "Whether to enable kubernetes scheduler.";
+        description = "Whether to enable Kubernetes scheduler.";
         default = false;
         type = types.bool;
       };
@@ -398,7 +457,7 @@ in {
       };
 
       leaderElect = mkOption {
-        description = "Whether to start leader election before executing main loop";
+        description = "Whether to start leader election before executing main loop.";
         type = types.bool;
         default = false;
       };
@@ -412,7 +471,7 @@ in {
 
     controllerManager = {
       enable = mkOption {
-        description = "Whether to enable kubernetes controller manager.";
+        description = "Whether to enable Kubernetes controller manager.";
         default = false;
         type = types.bool;
       };
@@ -430,7 +489,7 @@ in {
       };
 
       leaderElect = mkOption {
-        description = "Whether to start leader election before executing main loop";
+        description = "Whether to start leader election before executing main loop.";
         type = types.bool;
         default = false;
       };
@@ -453,12 +512,6 @@ in {
         type = types.nullOr types.path;
       };
 
-      clusterCidr = mkOption {
-        description = "Kubernetes controller manager CIDR Range for Pods in cluster";
-        default = "10.10.0.0/16";
-        type = types.str;
-      };
-
       extraOpts = mkOption {
         description = "Kubernetes controller manager extra command line options.";
         default = "";
@@ -468,7 +521,7 @@ in {
 
     kubelet = {
       enable = mkOption {
-        description = "Whether to enable kubernetes kubelet.";
+        description = "Whether to enable Kubernetes kubelet.";
         default = false;
         type = types.bool;
       };
@@ -518,13 +571,13 @@ in {
       };
 
       hostname = mkOption {
-        description = "Kubernetes kubelet hostname override";
+        description = "Kubernetes kubelet hostname override.";
         default = config.networking.hostName;
         type = types.str;
       };
 
       allowPrivileged = mkOption {
-        description = "Whether to allow kubernetes containers to request privileged mode.";
+        description = "Whether to allow Kubernetes containers to request privileged mode.";
         default = true;
         type = types.bool;
       };
@@ -536,7 +589,7 @@ in {
       };
 
       clusterDns = mkOption {
-        description = "Use alternative dns.";
+        description = "Use alternative DNS.";
         default = "10.10.0.1";
         type = types.str;
       };
@@ -548,20 +601,20 @@ in {
       };
 
       networkPlugin = mkOption {
-        description = "Network plugin to use by kubernetes";
+        description = "Network plugin to use by Kubernetes.";
         type = types.nullOr (types.enum ["cni" "kubenet"]);
         default = "kubenet";
       };
 
       cni = {
         packages = mkOption {
-          description = "List of network plugin packages to install";
+          description = "List of network plugin packages to install.";
           type = types.listOf types.package;
           default = [];
         };
 
         config = mkOption {
-          description = "Kubernetes CNI configuration";
+          description = "Kubernetes CNI configuration.";
           type = types.listOf types.attrs;
           default = [];
           example = literalExample ''
@@ -588,11 +641,29 @@ in {
       };
 
       manifests = mkOption {
-        description = "List of manifests to bootstrap with kubelet";
+        description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
         type = types.attrsOf types.attrs;
         default = {};
       };
 
+      applyManifests = mkOption {
+        description = "Whether to apply manifests.";
+        default = false;
+        type = types.bool;
+      };
+
+      unschedulable = mkOption {
+        description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
+        default = false;
+        type = types.bool;
+      };
+
+      taints = mkOption {
+        description = ".";
+        default = {};
+        type = types.attrsOf (types.submodule [ taintOptions ]);
+      };
+
       extraOpts = mkOption {
         description = "Kubernetes kubelet extra command line options.";
         default = "";
@@ -602,7 +673,7 @@ in {
 
     proxy = {
       enable = mkOption {
-        description = "Whether to enable kubernetes proxy.";
+        description = "Whether to enable Kubernetes proxy.";
         default = false;
         type = types.bool;
       };
@@ -620,12 +691,74 @@ in {
       };
     };
 
+    addonManager = {
+      enable = mkOption {
+        description = "Whether to enable Kubernetes addon manager.";
+        default = false;
+        type = types.bool;
+      };
+
+      versionTag = mkOption {
+        description = "Version tag of Kubernetes addon manager image.";
+        default = "v6.4-beta.1";
+        type = types.str;
+      };
+
+      addons = mkOption {
+        description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
+        default = { };
+        type = types.attrsOf types.attrs;
+        example = literalExample ''
+          {
+            "my-service" = {
+              "apiVersion" = "v1";
+              "kind" = "Service";
+              "metadata" = {
+                "name" = "my-service";
+                "namespace" = "default";
+              };
+              "spec" = { ... };
+            };
+          }
+          // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; };
+        '';
+      };
+    };
+
+    dns = {
+      enable = mkEnableOption "Kubernetes DNS service.";
+
+      port = mkOption {
+        description = "Kubernetes DNS listening port.";
+        default = 53;
+        type = types.int;
+      };
+
+      domain = mkOption  {
+        description = "Kubernetes DNS domain under which to create names.";
+        default = cfg.kubelet.clusterDomain;
+        type = types.str;
+      };
+
+      extraOpts = mkOption {
+        description = "Kubernetes DNS extra command line options.";
+        default = "";
+        type = types.str;
+      };
+    };
+
     path = mkOption {
-      description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added";
+      description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
       type = types.listOf types.package;
       default = [];
     };
 
+    clusterCidr = mkOption {
+      description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
+      default = "10.10.0.0/16";
+      type = types.str;
+    };
+
   };
 
   ###### implementation
@@ -645,7 +778,10 @@ in {
         serviceConfig = {
           Slice = "kubernetes.slice";
           ExecStart = ''${cfg.package}/bin/kubelet \
-            --pod-manifest-path=${manifests} \
+            ${optionalString cfg.kubelet.applyManifests
+              "--pod-manifest-path=${manifests}"} \
+            ${optionalString (taints != "")
+              "--register-with-taints=${taints}"} \
             --kubeconfig=${kubeconfig} \
             --require-kubeconfig \
             --address=${cfg.kubelet.address} \
@@ -677,15 +813,24 @@ in {
         };
       };
 
+      # Allways include cni plugins
+      services.kubernetes.kubelet.cni.packages = [pkgs.cni];
+    })
+
+    (mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
       environment.etc = mapAttrs' (name: manifest:
         nameValuePair "kubernetes/manifests/${name}.json" {
           text = builtins.toJSON manifest;
           mode = "0755";
         }
       ) cfg.kubelet.manifests;
+    })
 
-      # Allways include cni plugins
-      services.kubernetes.kubelet.cni.packages = [pkgs.cni];
+    (mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) {
+      services.kubernetes.kubelet.taints.unschedulable = {
+        value = "true";
+        effect = "NoSchedule";
+      };
     })
 
     (mkIf cfg.apiserver.enable {
@@ -728,7 +873,7 @@ in {
             --authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \
             ${optionalString (elem "ABAC" cfg.apiserver.authorizationMode)
               "--authorization-policy-file=${
-                pkgs.writeText "kube-auth-policy"
+                pkgs.writeText "kube-auth-policy.jsonl"
                 (concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
               }"
             } \
@@ -743,7 +888,7 @@ in {
               "--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
             ${optionalString cfg.verbose "--v=6"} \
             ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
-            --storage-backend=etcd2 \
+            --storage-backend=${cfg.apiserver.storageBackend} \
             ${cfg.apiserver.extraOpts}
           '';
           WorkingDirectory = cfg.dataDir;
@@ -799,8 +944,8 @@ in {
             ${if (cfg.controllerManager.rootCaFile!=null)
               then "--root-ca-file=${cfg.controllerManager.rootCaFile}"
               else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \
-            ${optionalString (cfg.controllerManager.clusterCidr!=null)
-              "--cluster-cidr=${cfg.controllerManager.clusterCidr}"} \
+            ${optionalString (cfg.clusterCidr!=null)
+              "--cluster-cidr=${cfg.clusterCidr}"} \
             --allocate-node-cidrs=true \
             ${optionalString cfg.verbose "--v=6"} \
             ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
@@ -819,7 +964,7 @@ in {
         description = "Kubernetes Proxy Service";
         wantedBy = [ "kubernetes.target" ];
         after = [ "kube-apiserver.service" ];
-        path = [pkgs.iptables];
+        path = [pkgs.iptables pkgs.conntrack_tools];
         serviceConfig = {
           Slice = "kubernetes.slice";
           ExecStart = ''${cfg.package}/bin/kube-proxy \
@@ -827,6 +972,8 @@ in {
             --bind-address=${cfg.proxy.address} \
             ${optionalString cfg.verbose "--v=6"} \
             ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
+            ${optionalString (cfg.clusterCidr!=null)
+              "--cluster-cidr=${cfg.clusterCidr}"} \
             ${cfg.proxy.extraOpts}
           '';
           WorkingDirectory = cfg.dataDir;
@@ -842,12 +989,18 @@ in {
       virtualisation.docker.enable = mkDefault true;
       services.kubernetes.kubelet.enable = mkDefault true;
       services.kubernetes.kubelet.allowPrivileged = mkDefault true;
+      services.kubernetes.kubelet.applyManifests = mkDefault true;
       services.kubernetes.apiserver.enable = mkDefault true;
       services.kubernetes.scheduler.enable = mkDefault true;
       services.kubernetes.controllerManager.enable = mkDefault true;
+      services.kubernetes.dns.enable = mkDefault true;
       services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
     })
 
+    (mkIf (all (el: el == "master") cfg.roles) {
+      services.kubernetes.kubelet.unschedulable = mkDefault true;
+    })
+
     (mkIf (any (el: el == "node") cfg.roles) {
       virtualisation.docker.enable = mkDefault true;
       virtualisation.docker.logDriver = mkDefault "json-file";
@@ -855,12 +1008,42 @@ in {
       services.kubernetes.proxy.enable = mkDefault true;
     })
 
+    (mkIf cfg.addonManager.enable {
+      services.kubernetes.kubelet.manifests = import ./kube-addon-manager.nix { inherit cfg addons; };
+      environment.etc."kubernetes/addons".source = "${addons}/";
+    })
+
+    (mkIf cfg.dns.enable {
+      systemd.services.kube-dns = {
+        description = "Kubernetes DNS Service";
+        wantedBy = [ "kubernetes.target" ];
+        after = [ "kube-apiserver.service" ];
+        serviceConfig = {
+          Slice = "kubernetes.slice";
+          ExecStart = ''${pkgs.kube-dns}/bin/kube-dns \
+            --kubecfg-file=${kubeconfig} \
+            --dns-port=${toString cfg.dns.port} \
+            --domain=${cfg.dns.domain} \
+            ${optionalString cfg.verbose "--v=6"} \
+            ${optionalString cfg.verbose "--log-flush-frequency=1s"} \
+            ${cfg.dns.extraOpts}
+          '';
+          WorkingDirectory = cfg.dataDir;
+          User = "kubernetes";
+          Group = "kubernetes";
+          AmbientCapabilities = "cap_net_bind_service";
+          SendSIGHUP = true;
+        };
+      };
+    })
+
     (mkIf (
         cfg.apiserver.enable ||
         cfg.scheduler.enable ||
         cfg.controllerManager.enable ||
         cfg.kubelet.enable ||
-        cfg.proxy.enable
+        cfg.proxy.enable ||
+        cfg.dns.enable
     ) {
       systemd.targets.kubernetes = {
         description = "Kubernetes";

nixos/modules/services/cluster/kubernetes/dns.nix

@@ -0,0 +1,240 @@
+{ cfg }: {
+    "kubedns-cm" = {
+        "apiVersion" = "v1";
+        "kind" = "ConfigMap";
+        "metadata" = {
+            "labels" = {
+                "addonmanager.kubernetes.io/mode" = "EnsureExists";
+            };
+            "name" = "kube-dns";
+            "namespace" = "kube-system";
+        };
+    };
+    "kubedns-controller" = {
+        "apiVersion" = "extensions/v1beta1";
+        "kind" = "Deployment";
+        "metadata" = {
+            "labels" = {
+                "addonmanager.kubernetes.io/mode" = "Reconcile";
+                "k8s-app" = "kube-dns";
+                "kubernetes.io/cluster-service" = "true";
+            };
+            "name" = "kube-dns";
+            "namespace" = "kube-system";
+        };
+        "spec" = {
+            "selector" = {
+                "matchLabels" = {
+                    "k8s-app" = "kube-dns";
+                };
+            };
+            "strategy" = {
+                "rollingUpdate" = {
+                    "maxSurge" = "10%";
+                    "maxUnavailable" = 0;
+                };
+            };
+            "template" = {
+                "metadata" = {
+                    "annotations" = {
+                        "scheduler.alpha.kubernetes.io/critical-pod" = "";
+                    };
+                    "labels" = {
+                        "k8s-app" = "kube-dns";
+                    };
+                };
+                "spec" = {
+                    "containers" = [{
+                        "args" = ["--domain=${cfg.dns.domain}."
+                            "--dns-port=10053"
+                            "--config-dir=/kube-dns-config"
+                            "--kube-master-url=${cfg.kubeconfig.server}"
+                            "--v=2"
+                        ];
+                        "env" = [{
+                            "name" = "PROMETHEUS_PORT";
+                            "value" = "10055";
+                        }];
+                        "image" = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1";
+                        "livenessProbe" = {
+                            "failureThreshold" = 5;
+                            "httpGet" = {
+                                "path" = "/healthcheck/kubedns";
+                                "port" = 10054;
+                                "scheme" = "HTTP";
+                            };
+                            "initialDelaySeconds" = 60;
+                            "successThreshold" = 1;
+                            "timeoutSeconds" = 5;
+                        };
+                        "name" = "kubedns";
+                        "ports" = [{
+                            "containerPort" = 10053;
+                            "name" = "dns-local";
+                            "protocol" = "UDP";
+                        } {
+                            "containerPort" = 10053;
+                            "name" = "dns-tcp-local";
+                            "protocol" = "TCP";
+                        } {
+                            "containerPort" = 10055;
+                            "name" = "metrics";
+                            "protocol" = "TCP";
+                        }];
+                        "readinessProbe" = {
+                            "httpGet" = {
+                                "path" = "/readiness";
+                                "port" = 8081;
+                                "scheme" = "HTTP";
+                            };
+                            "initialDelaySeconds" = 3;
+                            "timeoutSeconds" = 5;
+                        };
+                        "resources" = {
+                            "limits" = {
+                                "memory" = "170Mi";
+                            };
+                            "requests" = {
+                                "cpu" = "100m";
+                                "memory" = "70Mi";
+                            };
+                        };
+                        "volumeMounts" = [{
+                            "mountPath" = "/kube-dns-config";
+                            "name" = "kube-dns-config";
+                        }];
+                    } {
+                        "args" = ["-v=2"
+                            "-logtostderr"
+                            "-configDir=/etc/k8s/dns/dnsmasq-nanny"
+                            "-restartDnsmasq=true"
+                            "--"
+                            "-k"
+                            "--cache-size=1000"
+                            "--log-facility=-"
+                            "--server=/${cfg.dns.domain}/127.0.0.1#10053"
+                            "--server=/in-addr.arpa/127.0.0.1#10053"
+                            "--server=/ip6.arpa/127.0.0.1#10053"
+                        ];
+                        "image" = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1";
+                        "livenessProbe" = {
+                            "failureThreshold" = 5;
+                            "httpGet" = {
+                                "path" = "/healthcheck/dnsmasq";
+                                "port" = 10054;
+                                "scheme" = "HTTP";
+                            };
+                            "initialDelaySeconds" = 60;
+                            "successThreshold" = 1;
+                            "timeoutSeconds" = 5;
+                        };
+                        "name" = "dnsmasq";
+                        "ports" = [{
+                            "containerPort" = 53;
+                            "name" = "dns";
+                            "protocol" = "UDP";
+                        } {
+                            "containerPort" = 53;
+                            "name" = "dns-tcp";
+                            "protocol" = "TCP";
+                        }];
+                        "resources" = {
+                            "requests" = {
+                                "cpu" = "150m";
+                                "memory" = "20Mi";
+                            };
+                        };
+                        "volumeMounts" = [{
+                            "mountPath" = "/etc/k8s/dns/dnsmasq-nanny";
+                            "name" = "kube-dns-config";
+                        }];
+                    } {
+                        "args" = ["--v=2"
+                            "--logtostderr"
+                            "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cfg.dns.domain},5,A"
+                            "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cfg.dns.domain},5,A"
+                        ];
+                        "image" = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1";
+                        "livenessProbe" = {
+                            "failureThreshold" = 5;
+                            "httpGet" = {
+                                "path" = "/metrics";
+                                "port" = 10054;
+                                "scheme" = "HTTP";
+                            };
+                            "initialDelaySeconds" = 60;
+                            "successThreshold" = 1;
+                            "timeoutSeconds" = 5;
+                        };
+                        "name" = "sidecar";
+                        "ports" = [{
+                            "containerPort" = 10054;
+                            "name" = "metrics";
+                            "protocol" = "TCP";
+                        }];
+                        "resources" = {
+                            "requests" = {
+                                "cpu" = "10m";
+                                "memory" = "20Mi";
+                            };
+                        };
+                    }];
+                    "dnsPolicy" = "Default";
+                    "serviceAccountName" = "kube-dns";
+                    "tolerations" = [{
+                        "key" = "CriticalAddonsOnly";
+                        "operator" = "Exists";
+                    }];
+                    "volumes" = [{
+                        "configMap" = {
+                            "name" = "kube-dns";
+                            "optional" = true;
+                        };
+                        "name" = "kube-dns-config";
+                    }];
+                };
+            };
+        };
+    };
+    "kubedns-sa" = {
+        "apiVersion" = "v1";
+        "kind" = "ServiceAccount";
+        "metadata" = {
+            "labels" = {
+                "addonmanager.kubernetes.io/mode" = "Reconcile";
+                "kubernetes.io/cluster-service" = "true";
+            };
+            "name" = "kube-dns";
+            "namespace" = "kube-system";
+        };
+    };
+    "kubedns-svc" = {
+        "apiVersion" = "v1";
+        "kind" = "Service";
+        "metadata" = {
+            "labels" = {
+                "addonmanager.kubernetes.io/mode" = "Reconcile";
+                "k8s-app" = "kube-dns";
+                "kubernetes.io/cluster-service" = "true";
+                "kubernetes.io/name" = "KubeDNS";
+            };
+            "name" = "kube-dns";
+            "namespace" = "kube-system";
+        };
+        "spec" = {
+            "clusterIP" = "${cfg.dns.serverIp}";
+            "ports" = [{
+                "name" = "dns";
+                "port" = 53;
+                "protocol" = "UDP";
+            } {
+                "name" = "dns-tcp";
+                "port" = 53;
+                "protocol" = "TCP";
+            }];
+            "selector" = {
+                "k8s-app" = "kube-dns";
+            };
+        };
+    };
+}

nixos/modules/services/cluster/kubernetes/kube-addon-manager.nix

@@ -0,0 +1,54 @@
+{ cfg, addons }: {
+    "kube-addon-manager" = {
+        "apiVersion" = "v1";
+        "kind" = "Pod";
+        "metadata" = {
+            "labels" = {
+                "component" = "kube-addon-manager";
+            };
+            "name" = "kube-addon-manager";
+            "namespace" = "kube-system";
+        };
+        "spec" = {
+            "containers" = [{
+                "command" = ["/bin/bash"
+                    "-c"
+                    "/opt/kube-addons.sh | tee /var/log/kube-addon-manager.log"
+                ];
+                "env" = [{
+                    "name" = "KUBECTL_OPTS";
+                    "value" = "--server=${cfg.kubeconfig.server}";
+                }];
+                "image" = "gcr.io/google-containers/kube-addon-manager:${cfg.addonManager.versionTag}";
+                "name" = "kube-addon-manager";
+                "resources" = {
+                    "requests" = {
+                        "cpu" = "5m";
+                        "memory" = "50Mi";
+                    };
+                };
+                "volumeMounts" = [{
+                    "mountPath" = "/etc/kubernetes/addons/";
+                    "name" = "addons";
+                    "readOnly" = true;
+                } {
+                    "mountPath" = "/var/log";
+                    "name" = "varlog";
+                    "readOnly" = false;
+                }];
+            }];
+            "hostNetwork" = true;
+            "volumes" = [{
+                "hostPath" = {
+                    "path" = "${addons}/";
+                };
+                "name" = "addons";
+            } {
+                "hostPath" = {
+                    "path" = "/var/log";
+                };
+                "name" = "varlog";
+            }];
+        };
+    };
+}