This packages k3s as a single self-contained binary (as upstream k3s does), but without having to download any impure already-built binaries. The upstream packaging scripts are used. Due to k3s's rather complicated packaging arrangement, this ends up being a rather long derivation.wip/yesman
parent
d6a8d0ca5b
commit
e370711ad0
@ -0,0 +1,233 @@ |
||||
with import <nixpkgs> {}; |
||||
|
||||
{ stdenv, lib, makeWrapper, fetchFromGitHub, fetchurl, fetchzip }: |
||||
|
||||
with lib; |
||||
|
||||
# k3s is a kinda weird derivation. One of the main points of k3s is the |
||||
# simplicity of it being one binary that can perform several tasks. |
||||
# However, when you have a good package manager (like nix), that doesn't |
||||
# actually make much of a difference; you don't really care if it's one binary |
||||
# or 10 since with a good package manager, installing and running it is |
||||
# identical. |
||||
# Since upstream k3s packages itself as one large binary with several |
||||
# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s |
||||
# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with |
||||
# some exceptions. |
||||
# K3s also carries patches to some packages (such as containerd and cni |
||||
# plugins), so we intentionally use the k3s versions of those binaries for k3s, |
||||
# even if the upstream version of those binaries exist in nixpkgs already. In |
||||
# the end, that means we have a thick k3s binary that behaves like the upstream |
||||
# one for the most part. |
||||
# However, k3s also bundles several pieces of unpatched software, from the |
||||
# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. |
||||
# Those pieces of software we entirely ignore upstream's handling of, and just |
||||
# make sure they're in the path if desired. |
||||
let |
||||
k3sVersion = "1.17.3+k3s1"; # k3s git tag |
||||
traefikChartVersion = "1.81.0"; # taken from ./scripts/version.sh at the above k3s tag |
||||
k3sRootVersion = "0.3.0"; # taken from .s/cripts/version.sh at the above k3s tag |
||||
# bundled into the k3s binary |
||||
traefikChart = fetchurl { |
||||
url = "https://kubernetes-charts.storage.googleapis.com/traefik-${traefikChartVersion}.tgz"; |
||||
sha256 = "1aqpzgjlvqhil0g3angz94zd4xbl4iq0qmpjcy5aq1xv9qciwdi9"; |
||||
}; |
||||
# so, k3s is a complicated thing to package |
||||
# This derivation attempts to avoid including any random binaries from the |
||||
# internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which |
||||
# we don't care about doing, we can add those as build or runtime |
||||
# dependencies using a real package manager). |
||||
# In addition to those binaries, it's also configuration though (right now |
||||
# mostly strongswan configuration), and k3s does use those files. |
||||
# As such, we download it in order to grab 'etc' and bundle it into the final |
||||
# k3s binary. |
||||
k3sRoot = fetchzip { |
||||
# Note: marked as apache 2.0 license |
||||
url = "https://github.com/rancher/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; |
||||
sha256 = "12xafn5jivl8lqdcs25b28xrc4mf7yf1xif5np169nvvxgvmpdxp"; |
||||
stripRoot=false; |
||||
}; |
||||
k3sPlugins = buildGoPackage rec { |
||||
name = "k3s-cni-plugins"; |
||||
version = "0.7.6-k3s1"; # from ./scripts/version.sh 'VERSION_CNIPLUGINS'; update when k3s's repo is updated. |
||||
|
||||
goPackagePath = "github.com/containernetworking/plugins"; |
||||
subPackages = [ "." ]; |
||||
|
||||
src = fetchFromGitHub { |
||||
owner = "rancher"; |
||||
repo = "plugins"; |
||||
rev = "v${version}"; |
||||
sha256 = "0ax72z1ziann352bp6khfds8vlf3bbkqckrkpx4l4jxgqks45izs"; |
||||
}; |
||||
|
||||
meta = { |
||||
description = "k3s-cni-plugins"; |
||||
license = licenses.asl20; |
||||
homepage = https://k3s.io; |
||||
maintainers = []; |
||||
platforms = platforms.linux; |
||||
}; |
||||
}; |
||||
# Grab this separately from a build because it's used by both stages of the |
||||
# k3s build. |
||||
k3sRepo = fetchgit { |
||||
url = "https://github.com/rancher/k3s"; |
||||
rev = "v${k3sVersion}"; |
||||
leaveDotGit = true; # for version / build date below |
||||
sha256 = "0qahyc0mf9glxj49va6d20mcncqg4svfic2iz8b1lqid5c4g68mm"; |
||||
}; |
||||
# Stage 1 of the k3s build: |
||||
# Let's talk about how k3s is structured. |
||||
# One of the ideas of k3s is that there's the single "k3s" binary which can |
||||
# do everything you need, from running a k3s server, to being a worker node, |
||||
# to running kubectl. |
||||
# The way that actually works is that k3s is a single go binary that contains |
||||
# a bunch of bindata that it unpacks at runtime into directories (either the |
||||
# user's home directory or /var/lib/rancher if run as root). |
||||
# This bindata includes both binaries and configuration. |
||||
# In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. |
||||
# First, we build all the binaries that get packed into the thick k3s binary |
||||
# (and output them from one derivation so they'll all be suitably patched up). |
||||
# Then, we bundle those binaries into our thick k3s binary and use that as |
||||
# the final single output. |
||||
# This approach was chosen because it ensures the bundled binaries all are |
||||
# correctly built to run with nix (we can lean on the existing buildGoPackage |
||||
# stuff), and we can again lean on that tooling for the final k3s binary too. |
||||
# Other alternatives would be to manually run the |
||||
# strip/patchelf/remove-references step ourselves in the installPhase of the |
||||
# derivation when we've built all the binaries, but haven't bundled them in |
||||
# with generated bindata yet. |
||||
k3sBuildStage1 = buildGoPackage rec { |
||||
name = "k3s-build-1"; |
||||
version = "${k3sVersion}"; |
||||
|
||||
goPackagePath = "github.com/rancher/k3s"; |
||||
|
||||
src = k3sRepo; |
||||
|
||||
patches = [ ./patches/00-k3s.patch ]; |
||||
|
||||
nativeBuildInputs = [ pkgconfig autoPatchelfHook breakpointHook ]; |
||||
buildInputs = [ git runc libseccomp ]; |
||||
|
||||
buildPhase = '' |
||||
pushd go/src/${goPackagePath} |
||||
|
||||
patchShebangs ./scripts/build ./scripts/version.sh |
||||
mkdir -p bin |
||||
./scripts/build |
||||
|
||||
popd |
||||
''; |
||||
|
||||
installPhase = '' |
||||
pushd go/src/${goPackagePath} |
||||
|
||||
mkdir -p "$bin/bin" |
||||
install -m 0755 -t "$bin/bin" ./bin/* |
||||
|
||||
popd |
||||
''; |
||||
|
||||
meta = { |
||||
description = "The various binaries that get packaged into the final k3s binary."; |
||||
license = licenses.asl20; |
||||
homepage = https://k3s.io; |
||||
maintainers = []; |
||||
platforms = platforms.linux; |
||||
}; |
||||
}; |
||||
k3sBuild = buildGoPackage rec { |
||||
name = "k3s-build"; |
||||
version = "${k3sVersion}"; |
||||
|
||||
goPackagePath = "github.com/rancher/k3s"; |
||||
|
||||
src = k3sRepo; |
||||
|
||||
patches = [ ./patches/00-k3s.patch ]; |
||||
|
||||
nativeBuildInputs = [ pkgconfig autoPatchelfHook breakpointHook ]; |
||||
buildInputs = [ git k3sBuildStage1 ]; |
||||
|
||||
# In order to build the thick k3s binary (which is what |
||||
# ./scripts/package-cli does), we need to get all the binaries that script |
||||
# expects in place. |
||||
buildPhase = '' |
||||
pushd go/src/${goPackagePath} |
||||
|
||||
patchShebangs ./scripts/build ./scripts/version.sh ./scripts/package-cli |
||||
|
||||
mkdir -p bin |
||||
|
||||
install -m 0755 -t ./bin ${k3sBuildStage1}/bin/* |
||||
install -m 0755 -T "${k3sPlugins}/bin/plugins" ./bin/cni |
||||
# Note: use the already-nixpkgs-bundled k3s rather than the one bundled |
||||
# in k3s because the k3s one is completely unmodified from upstream |
||||
# (unlike containerd, cni, etc) |
||||
install -m 0755 -T "${runc}/bin/runc" ./bin/runc |
||||
cp -R "${k3sRoot}/etc" ./etc |
||||
mkdir -p "build/static/charts" |
||||
cp "${traefikChart}" "build/static/charts/traefik-${traefikChartVersion}.tgz" |
||||
|
||||
./scripts/package-cli |
||||
|
||||
popd |
||||
''; |
||||
|
||||
installPhase = '' |
||||
pushd go/src/${goPackagePath} |
||||
|
||||
mkdir -p "$bin/bin" |
||||
install -m 0755 -t "$bin/bin" ./dist/artifacts/k3s |
||||
|
||||
popd |
||||
''; |
||||
|
||||
meta = { |
||||
description = "The k3s go binary which is used by the final wrapped output below."; |
||||
license = licenses.asl20; |
||||
homepage = https://k3s.io; |
||||
maintainers = []; |
||||
platforms = platforms.linux; |
||||
}; |
||||
}; |
||||
in |
||||
stdenv.mkDerivation rec { |
||||
name = "k3s"; |
||||
|
||||
# Important utilities used by the kubelet, see |
||||
# https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 |
||||
# Note the list in that issue is stale and some aren't relevant for k3s. |
||||
k3sRuntimeDeps = [ |
||||
socat iptables iproute bridge-utils ethtool utillinux ipset conntrack-tools |
||||
]; |
||||
|
||||
buildInputs = [ |
||||
k3sBuild makeWrapper |
||||
] ++ k3sRuntimeDeps; |
||||
|
||||
unpackPhase = "true"; |
||||
|
||||
# And, one final derivation (you thought the last one was it, right?) |
||||
# We got the binary we wanted above, but it doesn't have all the runtime |
||||
# dependencies k8s wants, including mount utilities for kubelet, networking |
||||
# tools for cni/kubelet stuff, etc |
||||
# Use a wrapper script to reference all the binaries that k3s tries to |
||||
# execute, but that we didn't bundle with it. |
||||
installPhase = '' |
||||
mkdir -p "$out/bin" |
||||
makeWrapper ${k3sBuild}/bin/k3s "$out/bin/k3s" \ |
||||
--prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ |
||||
--prefix PATH : "$out/bin" |
||||
''; |
||||
|
||||
meta = { |
||||
description = "A lightweight Kubernetes distribution."; |
||||
license = licenses.asl20; |
||||
homepage = https://k3s.io; |
||||
maintainers = with maintainers; [ euank ]; |
||||
platforms = platforms.linux; |
||||
}; |
||||
} |
@ -0,0 +1,74 @@ |
||||
diff --git a/main.go b/main.go
|
||||
index 62908bb7bb..0527222887 100644
|
||||
--- a/main.go
|
||||
+++ b/main.go
|
||||
@@ -1,5 +1,5 @@
|
||||
//go:generate go run pkg/codegen/cleanup/main.go
|
||||
-//go:generate /bin/rm -rf pkg/generated
|
||||
+//go:generate rm -rf pkg/generated
|
||||
//go:generate go run pkg/codegen/main.go
|
||||
//go:generate go fmt pkg/deploy/zz_generated_bindata.go
|
||||
//go:generate go fmt pkg/static/zz_generated_bindata.go
|
||||
diff --git a/scripts/build b/scripts/build
|
||||
index 72d3c07ece..3e5455b262 100755
|
||||
--- a/scripts/build
|
||||
+++ b/scripts/build
|
||||
@@ -10,7 +10,8 @@ PKG_CONTAINERD="github.com/containerd/containerd"
|
||||
PKG_RANCHER_CONTAINERD="github.com/rancher/containerd"
|
||||
PKG_CRICTL="github.com/kubernetes-sigs/cri-tools"
|
||||
|
||||
-buildDate=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
+# Deterministic build date
|
||||
+buildDate="$(date -d "$(git log -1 --format=%ai)" -u "+%Y-%m-%dT%H:%M:%SZ")"
|
||||
|
||||
VENDOR_PREFIX="${PKG}/vendor/"
|
||||
VERSIONFLAGS="
|
||||
@@ -82,17 +83,7 @@ cleanup() {
|
||||
}
|
||||
|
||||
INSTALLBIN=$(pwd)/bin
|
||||
-if [ ! -x ${INSTALLBIN}/cni ]; then
|
||||
-(
|
||||
- echo Building cni
|
||||
- TMPDIR=$(mktemp -d)
|
||||
- trap cleanup EXIT
|
||||
- WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
|
||||
- git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
|
||||
- cd $WORKDIR
|
||||
- GOPATH=$TMPDIR CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni
|
||||
-)
|
||||
-fi
|
||||
+# skip building cni, use our separately built one
|
||||
# echo Building agent
|
||||
# CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go
|
||||
echo Building server
|
||||
@@ -108,9 +99,8 @@ ln -s containerd ./bin/ctr
|
||||
#CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go
|
||||
# echo Building containerd
|
||||
# CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
|
||||
-echo Building runc
|
||||
-make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static
|
||||
-cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc
|
||||
+
|
||||
+# skip building runc; use our packaged one
|
||||
|
||||
echo Building containerd-shim
|
||||
make -C ./vendor/github.com/containerd/containerd bin/containerd-shim
|
||||
diff --git a/scripts/package-cli b/scripts/package-cli
|
||||
index 4c66ce32df..6d1e0c03cb 100755
|
||||
--- a/scripts/package-cli
|
||||
+++ b/scripts/package-cli
|
||||
@@ -55,10 +55,10 @@ LDFLAGS="
|
||||
-X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8}
|
||||
-w -s
|
||||
"
|
||||
-STATIC="-extldflags '-static'"
|
||||
if [ "$DQLITE" = "true" ]; then
|
||||
DQLITE_TAGS="dqlite"
|
||||
fi
|
||||
-CGO_ENABLED=0 go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
|
||||
+go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go
|
||||
|
||||
-./scripts/build-upload ${CMD_NAME} ${COMMIT}
|
||||
+# for nixos, don't upload it
|
||||
+# ./scripts/build-upload ${CMD_NAME} ${COMMIT}
|
Loading…
Reference in new issue