nvidia-docker: init at 2.0.3

nvidia_x11 and persistenced were modified to provide binaries which can be mounted inside a docker-container to be executed there. most ldconfig-based discovery of bundled nvidia libraries is patched out ldconfig itself is patched to be able to deal with patchelf'ed libraries See https://sourceware.org/bugzilla/show_bug.cgi?id=23964
5 years ago · d6c2fe76ed
parent 8913ba2a8d
commit d6c2fe76ed
7 changed files with 289 additions and 0 deletions
--- a/pkgs/applications/virtualization/nvidia-docker/config.toml
+++ b/pkgs/applications/virtualization/nvidia-docker/config.toml
@ -0,0 +1,13 @@
+disable-require = false
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-runtime-hook.log"
+ldcache = "/tmp/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+#user = "root:video"
+ldconfig = "@@glibcbin@/bin/ldconfig"
--- a/pkgs/applications/virtualization/nvidia-docker/default.nix
+++ b/pkgs/applications/virtualization/nvidia-docker/default.nix
@ -0,0 +1,84 @@
+{ stdenv, lib, fetchFromGitHub, fetchpatch, callPackage, makeWrapper
+, buildGoPackage, runc, libelf, libcap, libseccomp, glibc }:
+
+with lib; let
+
+  glibc-ldconf = glibc.overrideAttrs (oldAttrs: {
+    # ldconfig needs help reading libraries that have been patchelf-ed, as the
+    # .dynstr section is no longer in the first LOAD segment. See also
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=23964 and
+    # https://github.com/NixOS/patchelf/issues/44
+    patches = oldAttrs.patches ++ [ (fetchpatch {
+      name = "ldconfig-patchelf.patch";
+      url = "https://sourceware.org/bugzilla/attachment.cgi?id=11444";
+      sha256 = "0nzzmq7pli37iyjrgcmvcy92piiwjybpw245ds7q43pbgdm7lc3s";
+    })];
+  });
+
+  libnvidia-container = callPackage ./libnvc.nix { };
+
+  nvidia-container-runtime = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "nvidia-container-runtime";
+    rev = "runtime-v2.0.0";
+    sha256 = "0jcj5xxbg7x7gyhbb67h3ds6vly62gx7j02zm6lg102h34jajj7a";
+  };
+
+  nvidia-container-runtime-hook = buildGoPackage rec {
+    name = "nvidia-container-runtime-hook-${version}";
+    version = "1.4.0";
+
+    goPackagePath = "nvidia-container-runtime-hook";
+
+    src = "${nvidia-container-runtime}/hook/nvidia-container-runtime-hook";
+  };
+
+  nvidia-runc = runc.overrideAttrs (oldAttrs: rec {
+    name = "nvidia-runc";
+    version = "1.0.0-rc6";
+    src = fetchFromGitHub {
+      owner = "opencontainers";
+      repo = "runc";
+      rev = "v${version}";
+      sha256 = "1jwacb8xnmx5fr86gximhbl9dlbdwj3rpf27hav9q1si86w5pb1j";
+    };
+    patches = [ "${nvidia-container-runtime}/runtime/runc/3f2f8b84a77f73d38244dd690525642a72156c64/0001-Add-prestart-hook-nvidia-container-runtime-hook-to-t.patch" ];
+  });
+
+in stdenv.mkDerivation rec {
+  name = "nvidia-docker-${version}";
+  version = "2.0.3";
+
+  src = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "nvidia-docker";
+    rev = "v${version}";
+    sha256 = "1vx5m591mnvcb9vy0196x5lh3r8swjsk0fnlv5h62m7m4m07v6wx";
+  };
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  buildPhase = ''
+    mkdir bin
+    cp nvidia-docker bin
+    cp ${libnvidia-container}/bin/nvidia-container-cli bin
+    cp ${nvidia-container-runtime-hook}/bin/nvidia-container-runtime-hook bin
+    cp ${nvidia-runc}/bin/runc bin/nvidia-container-runtime
+  '';
+
+  installPhase = ''
+    mkdir -p $out/{bin,etc}
+    cp -r bin $out
+    wrapProgram $out/bin/nvidia-container-cli \
+      --prefix LD_LIBRARY_PATH : /run/opengl-driver/lib:/run/opengl-driver-32/lib
+    cp ${./config.toml} $out/etc/config.toml
+    substituteInPlace $out/etc/config.toml --subst-var-by glibcbin ${lib.getBin glibc-ldconf}
+  '';
+
+  meta = {
+    homepage = https://github.com/NVIDIA/nvidia-docker;
+    description = "NVIDIA container runtime for Docker";
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+  };
+}
--- a/pkgs/applications/virtualization/nvidia-docker/libnvc-ldconfig-and-path-fixes.patch
+++ b/pkgs/applications/virtualization/nvidia-docker/libnvc-ldconfig-and-path-fixes.patch
@ -0,0 +1,130 @@
+diff --git a/src/ldcache.c b/src/ldcache.c
+index 38bab05..e1abc89 100644
+--- a/src/ldcache.c
+++ b/src/ldcache.c
+@@ -108,40 +108,27 @@ ldcache_close(struct ldcache *ctx)
+ 
+ int
+ ldcache_resolve(struct ldcache *ctx, uint32_t arch, const char *root, const char * const libs[],
+-    char *paths[], size_t size, ldcache_select_fn select, void *select_ctx)
+    char *paths[], size_t size, const char* version)
+ {
+         char path[PATH_MAX];
+-        struct header_libc6 *h;
+-        int override;
+        char dir[PATH_MAX];
+        char lib[PATH_MAX];
+ 
+-        h = (struct header_libc6 *)ctx->ptr;
+         memset(paths, 0, size * sizeof(*paths));
+ 
+-        for (uint32_t i = 0; i < h->nlibs; ++i) {
+-                int32_t flags = h->libs[i].flags;
+-                char *key = (char *)ctx->ptr + h->libs[i].key;
+-                char *value = (char *)ctx->ptr + h->libs[i].value;
+-
+-                if (!(flags & LD_ELF) || (flags & LD_ARCH_MASK) != arch)
+        for (size_t j = 0; j < size; ++j) {
+                snprintf(dir, 100, "/run/opengl-driver%s/lib",
+                    arch == LD_I386_LIB32 ? "-32" : "");
+                if (!strncmp(libs[j], "libvdpau_nvidia.so", 100))
+                  strcat(dir, "/vdpau");
+                snprintf(lib, 100, "%s/%s.%s", dir, libs[j], version);
+                if (path_resolve_full(ctx->err, path, "/", lib) < 0)
+                        return (-1);
+                if (!file_exists(ctx->err, path))
+                         continue;
+-
+-                for (size_t j = 0; j < size; ++j) {
+-                        if (!str_has_prefix(key, libs[j]))
+-                                continue;
+-                        if (path_resolve(ctx->err, path, root, value) < 0)
+-                                return (-1);
+-                        if (paths[j] != NULL && str_equal(paths[j], path))
+-                                continue;
+-                        if ((override = select(ctx->err, select_ctx, root, paths[j], path)) < 0)
+-                                return (-1);
+-                        if (override) {
+-                                free(paths[j]);
+-                                paths[j] = xstrdup(ctx->err, path);
+-                                if (paths[j] == NULL)
+-                                        return (-1);
+-                        }
+-                        break;
+-                }
+                paths[j] = xstrdup(ctx->err, path);
+                if (paths[j] == NULL)
+                        return (-1);
+         }
+         return (0);
+ }
+diff --git a/src/ldcache.h b/src/ldcache.h
+index 33d78dd..2b087db 100644
+--- a/src/ldcache.h
+++ b/src/ldcache.h
+@@ -50,6 +50,6 @@ void ldcache_init(struct ldcache *, struct error *, const char *);
+ int  ldcache_open(struct ldcache *);
+ int  ldcache_close(struct ldcache *);
+ int  ldcache_resolve(struct ldcache *, uint32_t, const char *, const char * const [],
+-    char *[], size_t, ldcache_select_fn, void *);
+    char *[], size_t, const char*);
+ 
+ #endif /* HEADER_LDCACHE_H */
+diff --git a/src/nvc_info.c b/src/nvc_info.c
+index cc96542..3fe7612 100644
+--- a/src/nvc_info.c
+++ b/src/nvc_info.c
+@@ -163,15 +163,13 @@ find_library_paths(struct error *err, struct nvc_driver_info *info, const char *
+         if (path_resolve_full(err, path, root, ldcache) < 0)
+                 return (-1);
+         ldcache_init(&ld, err, path);
+-        if (ldcache_open(&ld) < 0)
+-                return (-1);
+ 
+         info->nlibs = size;
+         info->libs = array_new(err, size);
+         if (info->libs == NULL)
+                 goto fail;
+         if (ldcache_resolve(&ld, LIB_ARCH, root, libs,
+-            info->libs, info->nlibs, select_libraries, info) < 0)
+            info->libs, info->nlibs, info->nvrm_version) < 0)
+                 goto fail;
+ 
+         info->nlibs32 = size;
+@@ -179,13 +177,11 @@ find_library_paths(struct error *err, struct nvc_driver_info *info, const char *
+         if (info->libs32 == NULL)
+                 goto fail;
+         if (ldcache_resolve(&ld, LIB32_ARCH, root, libs,
+-            info->libs32, info->nlibs32, select_libraries, info) < 0)
+            info->libs32, info->nlibs32, info->nvrm_version) < 0)
+                 goto fail;
+         rv = 0;
+ 
+  fail:
+-        if (ldcache_close(&ld) < 0)
+-                return (-1);
+         return (rv);
+ }
+ 
+@@ -199,7 +195,7 @@ find_binary_paths(struct error *err, struct nvc_driver_info *info, const char *r
+         char path[PATH_MAX];
+         int rv = -1;
+ 
+-        if ((env = secure_getenv("PATH")) == NULL) {
+        if ((env = "/run/nvidia-docker/bin:/run/nvidia-docker/extras/bin") == NULL) {
+                 error_setx(err, "environment variable PATH not found");
+                 return (-1);
+         }
+diff --git a/src/nvc_ldcache.c b/src/nvc_ldcache.c
+index d41a24d..65b7878 100644
+--- a/src/nvc_ldcache.c
+++ b/src/nvc_ldcache.c
+@@ -331,7 +331,7 @@ nvc_ldcache_update(struct nvc_context *ctx, const struct nvc_container *cnt)
+         if (validate_args(ctx, cnt != NULL) < 0)
+                 return (-1);
+ 
+-        argv = (char * []){cnt->cfg.ldconfig, cnt->cfg.libs_dir, cnt->cfg.libs32_dir, NULL};
+        argv = (char * []){cnt->cfg.ldconfig, "-f", "/tmp/ld.so.conf.nvidia-host", "-C", "/tmp/ld.so.cache.nvidia-host", cnt->cfg.libs_dir, cnt->cfg.libs32_dir, NULL};
+         if (*argv[0] == '@') {
+                 /*
+                  * We treat this path specially to be relative to the host filesystem.
--- a/pkgs/applications/virtualization/nvidia-docker/libnvc.nix
+++ b/pkgs/applications/virtualization/nvidia-docker/libnvc.nix
@ -0,0 +1,53 @@
+{ stdenv, lib, fetchFromGitHub, libelf, libcap, libseccomp }:
+
+with lib; let
+
+  modp-ver = "396.51";
+
+  nvidia-modprobe = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "nvidia-modprobe";
+    rev = modp-ver;
+    sha256 = "1fw2qwc84k64agw6fx2v0mjf88aggph9c6qhs4cv7l3gmflv8qbk";
+  };
+
+in stdenv.mkDerivation rec {
+  name = "libnvidia-container-${version}";
+  version = "1.0.0";
+
+  src = fetchFromGitHub {
+    owner = "NVIDIA";
+    repo = "libnvidia-container";
+    rev = "v${version}";
+    sha256 = "1ws6mfsbgxhzlb5w1r8qqg2arvxkr21n59i4cqsyz3h5jsqsflbw";
+  };
+
+  # locations of nvidia-driver libraries are not resolved via ldconfig which
+  # doesn't get used on NixOS. Additional support binaries like nvidia-smi are
+  # not resolved via the environment PATH but via the derivation output path.
+  patches = [ ./libnvc-ldconfig-and-path-fixes.patch ];
+
+  makeFlags = [
+    "WITH_LIBELF=yes"
+    "prefix=$(out)"
+  ];
+
+  postPatch = ''
+    sed -i 's/^REVISION :=.*/REVISION = ${src.rev}/' mk/common.mk
+    sed -i 's/^COMPILER :=.*/COMPILER = $(CC)/' mk/common.mk
+
+    mkdir -p deps/src/nvidia-modprobe-${modp-ver}
+    cp -r ${nvidia-modprobe}/* deps/src/nvidia-modprobe-${modp-ver}
+    chmod -R u+w deps/src
+    touch deps/src/nvidia-modprobe-${modp-ver}/.download_stamp
+  '';
+
+  buildInputs = [ libelf libcap libseccomp ];
+
+  meta = {
+    homepage = https://github.com/NVIDIA/libnvidia-container;
+    description = "NVIDIA container runtime library";
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+  };
+}
--- a/pkgs/os-specific/linux/nvidia-x11/builder.sh
+++ b/pkgs/os-specific/linux/nvidia-x11/builder.sh
@ -129,6 +129,8 @@ installPhase() {
        for i in nvidia-cuda-mps-control nvidia-cuda-mps-server nvidia-smi nvidia-debugdump; do
            if [ -e "$i" ]; then
                install -Dm755 $i $bin/bin/$i
+                # unmodified binary backup for mounting in containers
+                install -Dm755 $i $bin/origBin/$i
                patchelf --interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
                    --set-rpath $out/lib:$libPath $bin/bin/$i
            fi
--- a/pkgs/os-specific/linux/nvidia-x11/persistenced.nix
+++ b/pkgs/os-specific/linux/nvidia-x11/persistenced.nix
@ -16,6 +16,11 @@ stdenv.mkDerivation rec {
  installFlags = [ "PREFIX=$(out)" ];

  postFixup = ''
+    # Save a copy of persistenced for mounting in containers
+    mkdir $out/origBin
+    cp $out/{bin,origBin}/nvidia-persistenced
+    patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/origBin/nvidia-persistenced
+
    patchelf --set-rpath "$(patchelf --print-rpath $out/bin/nvidia-persistenced):${nvidia_x11}/lib" \
      $out/bin/nvidia-persistenced
  '';
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -11837,6 +11837,8 @@ in

  nv-codec-headers = callPackage ../development/libraries/nv-codec-headers { };

+  nvidia-docker = callPackage ../applications/virtualization/nvidia-docker { };
+
  nvidia-texture-tools = callPackage ../development/libraries/nvidia-texture-tools { };

  nvidia-video-sdk = callPackage ../development/libraries/nvidia-video-sdk { };