poetry2nix: init at 1.1.0

pkgs/development/tools/poetry2nix/poetry2nix/README.md

@@ -0,0 +1,6 @@
+Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix
+
+The update procedure is as-follows:
+1. Send your change to the upstream poetry2nix repository
+2. Get it approved with tests passing
+3. Run the update script in pkgs/development/tools/poetry2nix

pkgs/development/tools/poetry2nix/poetry2nix/bin/poetry2nix

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+from concurrent.futures import ThreadPoolExecutor
+import subprocess
+import textwrap
+import argparse
+import toml
+import json
+import sys
+
+
+argparser = argparse.ArgumentParser(description="Generate overrides for git hashes",)
+argparser.add_argument(
+    "--lock", default="poetry.lock", help="Path to input poetry.lock",
+)
+argparser.add_argument(
+    "--out", default="poetry-git-overlay.nix", help="Output file",
+)
+
+
+def fetch_git(pkg):
+    return (
+        pkg["name"],
+        subprocess.run(
+            [
+                "nix-prefetch-git",
+                "--fetch-submodules",
+                "--url",
+                pkg["source"]["url"],
+                "--rev",
+                pkg["source"]["reference"],
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        ),
+    )
+
+
+def indent(expr, spaces=2):
+    i = " " * spaces
+    return "\n".join([(i if l != "" else "") + l for l in expr.split("\n")])
+
+
+if __name__ == "__main__":
+    args = argparser.parse_args()
+
+    with open(args.lock) as lockf:
+        lock = toml.load(lockf)
+
+    pkgs = []
+    for pkg in lock["package"]:
+        if "source" in pkg:
+            pkgs.append(pkg)
+
+    with ThreadPoolExecutor() as e:
+        futures = []
+
+        for pkg in pkgs:
+            futures.append(e.submit(fetch_git, pkg))
+
+        lines = [
+            "{ pkgs }:",
+            "self: super: {",
+        ]
+
+        for f in futures:
+            drv_name, p = f.result()
+            if p.returncode != 0:
+                sys.stderr.buffer.write(p.stderr)
+                sys.stderr.buffer.flush()
+                exit(p.returncode)
+
+            meta = json.loads(p.stdout.decode())
+            lines.append(
+                indent(
+                    textwrap.dedent(
+                        """
+              %s = super.%s.overrideAttrs (
+                _: {
+                  src = pkgs.fetchgit {
+                    url = "%s";
+                    rev = "%s";
+                    sha256 = "%s";
+                  };
+                }
+              );"""
+                        % (drv_name, drv_name, meta["url"], meta["rev"], meta["sha256"])
+                    )
+                )
+            )
+
+        lines.extend(["", "}", ""])
+
+        expr = "\n".join(lines)
+
+    with open(args.out, "w") as f:
+        f.write(expr)
+
+    print(f"Wrote {args.out}")

pkgs/development/tools/poetry2nix/poetry2nix/cli.nix

@@ -0,0 +1,52 @@
+{ pkgs ? import <nixpkgs> {}
+, lib ? pkgs.lib
+, version
+}:
+
+let
+  inherit (pkgs) python3;
+
+in
+pkgs.stdenv.mkDerivation {
+  pname = "poetry2nix";
+  inherit version;
+
+  buildInputs = [
+    (python3.withPackages (ps: [ ps.toml ]))
+  ];
+
+  nativeBuildInputs = [
+    pkgs.makeWrapper
+  ];
+
+  src = ./bin;
+
+  dontConfigure = true;
+
+  buildPhase = ''
+    runHook preBuild
+    ${python3.pkgs.black}/bin/black --quiet --check poetry2nix
+    patchShebangs poetry2nix
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    mkdir -p $out/bin
+    mv poetry2nix $out/bin
+
+    wrapProgram $out/bin/poetry2nix --prefix PATH ":" ${lib.makeBinPath [
+    pkgs.nix-prefetch-git
+  ]}
+
+    runHook postInstall
+  '';
+
+  meta = {
+    homepage = "https://github.com/nix-community/poetry2nix";
+    description = "CLI to supplement sha256 hashes for git dependencies";
+    license = lib.licenses.mit;
+    maintainers = [ lib.maintainers.adisbladis ];
+  };
+
+}

pkgs/development/tools/poetry2nix/poetry2nix/default.nix

@@ -0,0 +1,252 @@
+{ pkgs ? import <nixpkgs> {}
+, lib ? pkgs.lib
+, poetry ? null
+, poetryLib ? import ./lib.nix { inherit lib pkgs; }
+}:
+
+let
+  inherit (poetryLib) isCompatible readTOML;
+
+  # Poetry2nix version
+  version = "1.1.0";
+
+  /* The default list of poetry2nix override overlays */
+  defaultPoetryOverrides = (import ./overrides.nix { inherit pkgs lib; });
+
+  mkEvalPep508 = import ./pep508.nix {
+    inherit lib;
+    stdenv = pkgs.stdenv;
+  };
+
+  getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn;
+
+  getAttrDefault = attribute: set: default: (
+    if builtins.hasAttr attribute set
+    then builtins.getAttr attribute set
+    else default
+  );
+
+  # Map SPDX identifiers to license names
+  spdxLicenses = lib.listToAttrs (lib.filter (pair: pair.name != null) (builtins.map (v: { name = if lib.hasAttr "spdxId" v then v.spdxId else null; value = v; }) (lib.attrValues lib.licenses)));
+  # Get license by id falling back to input string
+  getLicenseBySpdxId = spdxId: getAttrDefault spdxId spdxLicenses spdxId;
+
+  #
+  # Returns an attrset { python, poetryPackages } for the given lockfile
+  #
+  mkPoetryPython =
+    { poetrylock
+    , poetryPkg
+    , overrides ? [ defaultPoetryOverrides ]
+    , meta ? {}
+    , python ? pkgs.python3
+    , pwd ? null
+    }@attrs: let
+      lockData = readTOML poetrylock;
+      lockFiles = lib.getAttrFromPath [ "metadata" "files" ] lockData;
+
+      specialAttrs = [ "poetrylock" "overrides" ];
+      passedAttrs = builtins.removeAttrs attrs specialAttrs;
+
+      evalPep508 = mkEvalPep508 python;
+
+      # Filter packages by their PEP508 markers
+      partitions = let
+        supportsPythonVersion = pkgMeta: if pkgMeta ? marker then (evalPep508 pkgMeta.marker) else true;
+      in
+        lib.partition supportsPythonVersion lockData.package;
+
+      compatible = partitions.right;
+      incompatible = partitions.wrong;
+
+      # Create an overriden version of pythonPackages
+      #
+      # We need to avoid mixing multiple versions of pythonPackages in the same
+      # closure as python can only ever have one version of a dependency
+      baseOverlay = self: super:
+        let
+          getDep = depName: if builtins.hasAttr depName self then self."${depName}" else throw "foo";
+
+          lockPkgs = builtins.listToAttrs (
+            builtins.map (
+              pkgMeta: rec {
+                name = pkgMeta.name;
+                value = self.mkPoetryDep (
+                  pkgMeta // {
+                    inherit pwd;
+                    source = getAttrDefault "source" pkgMeta null;
+                    files = lockFiles.${name};
+                    pythonPackages = self;
+                  }
+                );
+              }
+            ) compatible
+          );
+        in
+          lockPkgs;
+
+      overlays = builtins.map getFunctorFn (
+        [
+          (
+            self: super: {
+              mkPoetryDep = self.callPackage ./mk-poetry-dep.nix {
+                inherit pkgs lib python poetryLib;
+              };
+              poetry = poetryPkg;
+            }
+          )
+          # Null out any filtered packages, we don't want python.pkgs from nixpkgs
+          (self: super: builtins.listToAttrs (builtins.map (x: { name = x.name; value = null; }) incompatible))
+          # Create poetry2nix layer
+          baseOverlay
+        ] ++ # User provided overrides
+        overrides
+      );
+
+      packageOverrides = lib.foldr lib.composeExtensions (self: super: {}) overlays;
+
+      py = python.override { inherit packageOverrides; self = py; };
+    in
+      {
+        python = py;
+        poetryPackages = map (pkg: py.pkgs.${pkg.name}) compatible;
+      };
+
+  /* Returns a package with a python interpreter and all packages specified in the poetry.lock lock file.
+
+     Example:
+       poetry2nix.mkPoetryEnv { poetrylock = ./poetry.lock; python = python3; }
+  */
+  mkPoetryEnv =
+    { poetrylock
+    , overrides ? [ defaultPoetryOverrides ]
+    , meta ? {}
+    , pwd ? null
+    , python ? pkgs.python3
+    }:
+      let
+        poetryPkg = poetry.override { inherit python; };
+        py = mkPoetryPython (
+          {
+            inherit poetryPkg poetrylock overrides meta python pwd;
+          }
+        );
+      in
+        py.python.withPackages (_: py.poetryPackages);
+
+  /* Creates a Python application from pyproject.toml and poetry.lock */
+  mkPoetryApplication =
+    { src
+    , pyproject
+    , poetrylock
+    , overrides ? [ defaultPoetryOverrides ]
+    , meta ? {}
+    , python ? pkgs.python3
+    , pwd ? null
+    , ...
+    }@attrs: let
+      poetryPkg = poetry.override { inherit python; };
+
+      py = (
+        mkPoetryPython {
+          inherit poetryPkg poetrylock overrides meta python pwd;
+        }
+      ).python;
+
+      pyProject = readTOML pyproject;
+
+      specialAttrs = [ "pyproject" "poetrylock" "overrides" ];
+      passedAttrs = builtins.removeAttrs attrs specialAttrs;
+
+      getDeps = depAttr: let
+        deps = getAttrDefault depAttr pyProject.tool.poetry {};
+        depAttrs = builtins.map (d: lib.toLower d) (builtins.attrNames deps);
+      in
+        builtins.map (dep: py.pkgs."${dep}") depAttrs;
+
+      getInputs = attr: getAttrDefault attr attrs [];
+      mkInput = attr: extraInputs: getInputs attr ++ extraInputs;
+
+      buildSystemPkgs = poetryLib.getBuildSystemPkgs {
+        inherit pyProject;
+        pythonPackages = py.pkgs;
+      };
+
+    in
+      py.pkgs.buildPythonApplication (
+        passedAttrs // {
+          pname = pyProject.tool.poetry.name;
+          version = pyProject.tool.poetry.version;
+
+          format = "pyproject";
+
+          nativeBuildInputs = [ pkgs.yj ];
+          buildInputs = mkInput "buildInputs" buildSystemPkgs;
+          propagatedBuildInputs = mkInput "propagatedBuildInputs" (getDeps "dependencies") ++ ([ py.pkgs.setuptools ]);
+          checkInputs = mkInput "checkInputs" (getDeps "dev-dependencies");
+
+          passthru = {
+            python = py;
+          };
+
+          postPatch = (getAttrDefault "postPatch" passedAttrs "") + ''
+            # Tell poetry not to resolve the path dependencies. Any version is
+            # fine !
+            yj -tj < pyproject.toml | python ${./pyproject-without-path.py} > pyproject.json
+            yj -jt < pyproject.json > pyproject.toml
+            rm pyproject.json
+          '';
+
+          meta = meta // {
+            inherit (pyProject.tool.poetry) description homepage;
+            license = getLicenseBySpdxId (getAttrDefault "license" pyProject.tool.poetry "unknown");
+          };
+
+        }
+      );
+
+  /* Poetry2nix CLI used to supplement SHA-256 hashes for git dependencies  */
+  cli = import ./cli.nix { inherit pkgs lib version; };
+
+  /* Poetry2nix documentation  */
+  doc = pkgs.stdenv.mkDerivation {
+    pname = "poetry2nix-docs";
+    inherit version;
+
+    src = pkgs.runCommandNoCC "poetry2nix-docs-src" {} ''
+      mkdir -p $out
+      cp ${./default.nix} $out/default.nix
+    '';
+
+    buildInputs = [
+      pkgs.nixdoc
+    ];
+
+    buildPhase = ''
+      nixdoc --category poetry2nix --description "Poetry2nix functions" --file ./default.nix > poetry2nix.xml
+    '';
+
+    installPhase = ''
+      mkdir -p $out
+      cp poetry2nix.xml $out/
+    '';
+
+  };
+
+in
+{
+  inherit mkPoetryEnv mkPoetryApplication cli doc;
+
+  /*
+  The default list of poetry2nix override overlays
+
+  Can be overriden by calling defaultPoetryOverrides.overrideOverlay which takes an overlay function
+  */
+  defaultPoetryOverrides = {
+    __functor = defaultPoetryOverrides;
+    overrideOverlay = fn: self: super: let
+      defaultSet = defaultPoetryOverrides self super;
+      customSet = fn self super;
+    in defaultSet // customSet;
+  };
+}

pkgs/development/tools/poetry2nix/poetry2nix/extensions.json

@@ -0,0 +1,14 @@
+[
+  "tar",
+  "tar.bz2",
+  "tar.gz",
+  "tar.lz",
+  "tar.lzma",
+  "tar.xz",
+  "tbz",
+  "tgz",
+  "tlz",
+  "txz",
+  "whl",
+  "zip"
+]

pkgs/development/tools/poetry2nix/poetry2nix/lib.nix

@@ -0,0 +1,79 @@
+{ lib, pkgs }:
+let
+  inherit (import ./semver.nix { inherit lib; }) satisfiesSemver;
+
+  # Returns true if pythonVersion matches with the expression in pythonVersions
+  isCompatible = pythonVersion: pythonVersions:
+    let
+      operators = {
+        "||" = cond1: cond2: cond1 || cond2;
+        "," = cond1: cond2: cond1 && cond2; # , means &&
+      };
+      # split string at "," and "||"
+      tokens = builtins.filter (x: x != "") (builtins.split "(,|\\|\\|)" pythonVersions);
+      combine = acc: v:
+        let
+          isOperator = builtins.typeOf v == "list";
+          operator = if isOperator then (builtins.elemAt v 0) else acc.operator;
+        in
+          if isOperator then (acc // { inherit operator; }) else {
+            inherit operator;
+            state = operators."${operator}" acc.state (satisfiesSemver pythonVersion v);
+          };
+      initial = { operator = ","; state = true; };
+    in
+      (builtins.foldl' combine initial tokens).state;
+
+  readTOML = path: builtins.fromTOML (builtins.readFile path);
+
+  #
+  # Returns the appropriate manylinux dependencies and string representation for the file specified
+  #
+  getManyLinuxDeps = f:
+    let
+      ml = pkgs.pythonManylinuxPackages;
+    in
+      if lib.strings.hasInfix "manylinux1" f then { pkg = [ ml.manylinux1 ]; str = "1"; }
+      else if lib.strings.hasInfix "manylinux2010" f then { pkg = [ ml.manylinux2010 ]; str = "2010"; }
+      else if lib.strings.hasInfix "manylinux2014" f then { pkg = [ ml.manylinux2014 ]; str = "2014"; }
+      else { pkg = []; str = null; };
+
+  # Fetch the artifacts from the PyPI index. Since we get all
+  # info we need from the lock file we don't use nixpkgs' fetchPyPi
+  # as it modifies casing while not providing anything we don't already
+  # have.
+  #
+  # Args:
+  #   pname: package name
+  #   file: filename including extension
+  #   hash: SRI hash
+  #   kind: Language implementation and version tag https://www.python.org/dev/peps/pep-0427/#file-name-convention
+  fetchFromPypi = lib.makeOverridable (
+    { pname, file, hash, kind }:
+      pkgs.fetchurl {
+        url = "https://files.pythonhosted.org/packages/${kind}/${lib.toLower (builtins.substring 0 1 file)}/${pname}/${file}";
+        inherit hash;
+      }
+  );
+
+  getBuildSystemPkgs =
+    { pythonPackages
+    , pyProject
+    }: let
+      buildSystem = lib.getAttrFromPath [ "build-system" "build-backend" ] pyProject;
+      drvAttr = builtins.elemAt (builtins.split "\\.|:" buildSystem) 0;
+    in
+      if buildSystem == "" then [] else (
+        [ pythonPackages.${drvAttr} or (throw "unsupported build system ${buildSystem}") ]
+      );
+
+in
+{
+  inherit
+    fetchFromPypi
+    getManyLinuxDeps
+    isCompatible
+    readTOML
+    getBuildSystemPkgs
+    ;
+}

pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix

@@ -0,0 +1,100 @@
+{ autoPatchelfHook
+, pkgs
+, lib
+, python
+, buildPythonPackage
+, pythonPackages
+, poetryLib
+}:
+{ name
+, version
+, files
+, source
+, dependencies ? {}
+, pythonPackages
+, python-versions
+, pwd
+, supportedExtensions ? lib.importJSON ./extensions.json
+, ...
+}: let
+
+  inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi;
+
+  inherit (import ./pep425.nix {
+    inherit lib python;
+    inherit (pkgs) stdenv;
+  }) selectWheel
+    ;
+
+  fileCandidates = let
+    supportedRegex = ("^.*?(" + builtins.concatStringsSep "|" supportedExtensions + ")");
+    matchesVersion = fname: builtins.match ("^.*" + builtins.replaceStrings [ "." ] [ "\\." ] version + ".*$") fname != null;
+    hasSupportedExtension = fname: builtins.match supportedRegex fname != null;
+  in
+    builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file) files;
+
+  toPath = s: pwd + "/${s}";
+
+  isSource = source != null;
+  isGit = isSource && source.type == "git";
+  isLocal = isSource && source.type == "directory";
+
+  localDepPath = toPath source.url;
+  pyProject = poetryLib.readTOML (localDepPath + "/pyproject.toml");
+
+  buildSystemPkgs = poetryLib.getBuildSystemPkgs {
+    inherit pythonPackages pyProject;
+  };
+
+  fileInfo = let
+    isBdist = f: lib.strings.hasSuffix "whl" f.file;
+    isSdist = f: ! isBdist f;
+    binaryDist = selectWheel fileCandidates;
+    sourceDist = builtins.filter isSdist fileCandidates;
+    lockFileEntry = if (builtins.length sourceDist) > 0 then builtins.head sourceDist else builtins.head binaryDist;
+  in
+    rec {
+      inherit (lockFileEntry) file hash;
+      name = file;
+      format = if lib.strings.hasSuffix ".whl" name then "wheel" else "setuptools";
+      kind = if format == "setuptools" then "source" else (builtins.elemAt (lib.strings.splitString "-" name) 2);
+    };
+
+in
+buildPythonPackage {
+  pname = name;
+  version = version;
+
+  doCheck = false; # We never get development deps
+  dontStrip = true;
+  format = if isLocal then "pyproject" else if isGit then "setuptools" else fileInfo.format;
+
+  nativeBuildInputs = if (!isSource && (getManyLinuxDeps fileInfo.name).str != null) then [ autoPatchelfHook ] else [];
+  buildInputs = if !isSource then (getManyLinuxDeps fileInfo.name).pkg else [];
+
+  propagatedBuildInputs =
+    let
+      # Some dependencies like django gets the attribute name django
+      # but dependencies try to access Django
+      deps = builtins.map (d: lib.toLower d) (builtins.attrNames dependencies);
+    in
+      (builtins.map (n: pythonPackages.${n}) deps) ++ (if isLocal then buildSystemPkgs else []);
+
+  meta = {
+    broken = ! isCompatible python.version python-versions;
+    license = [];
+  };
+
+  # We need to retrieve kind from the interpreter and the filename of the package
+  # Interpreters should declare what wheel types they're compatible with (python type + ABI)
+  # Here we can then choose a file based on that info.
+  src = if isGit then (
+    builtins.fetchGit {
+      inherit (source) url;
+      rev = source.reference;
+    }
+  ) else if isLocal then (localDepPath) else fetchFromPypi {
+    pname = name;
+    inherit (fileInfo) file hash kind;
+  };
+}

pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix

@@ -0,0 +1,405 @@
+{ pkgs ? import <nixpkgs> {}
+, lib ? pkgs.lib
+, stdenv ? pkgs.stdenv
+}:
+
+self: super:
+
+let
+
+  addSetupTools = drv: if drv == null then null else drv.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.setuptools_scm
+      ];
+    }
+  );
+
+  getAttrDefault = attribute: set: default:
+    if builtins.hasAttr attribute set
+    then builtins.getAttr attribute set
+    else default;
+
+in
+{
+
+  asciimatics = super.asciimatics.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.setuptools_scm
+      ];
+    }
+  );
+
+  av = super.av.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [
+        pkgs.pkgconfig
+      ];
+      buildInputs = old.buildInputs ++ [ pkgs.ffmpeg_4 ];
+    }
+  );
+
+  bcrypt = super.bcrypt.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ pkgs.libffi ];
+    }
+  );
+
+  cffi = super.cffi.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ pkgs.libffi ];
+    }
+  );
+
+  cftime = super.cftime.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.cython
+      ];
+    }
+  );
+
+  configparser = addSetupTools super.configparser;
+
+  cbor2 = addSetupTools super.cbor2;
+
+  cryptography = super.cryptography.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ pkgs.openssl ];
+    }
+  );
+
+  django = (
+    super.django.overrideAttrs (
+      old: {
+        propagatedNativeBuildInputs = (getAttrDefault "propagatedNativeBuildInputs" old [])
+        ++ [ pkgs.gettext ];
+      }
+    )
+  );
+
+  django-bakery = super.django-bakery.overrideAttrs (
+    old: {
+      configurePhase = ''
+        if ! test -e LICENSE; then
+          touch LICENSE
+        fi
+      '' + (getAttrDefault "configurePhase" old "");
+    }
+  );
+
+  # Environment markers are not always included (depending on how a dep was defined)
+  enum34 = if self.pythonAtLeast "3.4" then null else super.enum34;
+
+  grandalf = super.grandalf.overrideAttrs (
+    old: {
+      postPatch = ''
+        substituteInPlace setup.py --replace "setup_requires=['pytest-runner',]," "setup_requires=[]," || true
+      '';
+    }
+  );
+
+  horovod = super.horovod.overrideAttrs (
+    old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ];
+    }
+  );
+
+  hypothesis = addSetupTools super.hypothesis;
+
+  importlib-metadata = addSetupTools super.importlib-metadata;
+
+  inflect = super.inflect.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.setuptools_scm
+      ];
+    }
+  );
+
+  jsonschema = addSetupTools super.jsonschema;
+
+  keyring = addSetupTools super.keyring;
+
+  lap = super.lap.overrideAttrs (
+    old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [
+        self.numpy
+      ];
+    }
+  );
+
+  llvmlite = super.llvmlite.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.llvm ];
+
+      # Disable static linking
+      # https://github.com/numba/llvmlite/issues/93
+      postPatch = ''
+        substituteInPlace ffi/Makefile.linux --replace "-static-libstdc++" ""
+
+        substituteInPlace llvmlite/tests/test_binding.py --replace "test_linux" "nope"
+      '';
+
+      # Set directory containing llvm-config binary
+      preConfigure = ''
+        export LLVM_CONFIG=${pkgs.llvm}/bin/llvm-config
+      '';
+
+      __impureHostDeps = pkgs.stdenv.lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ];
+
+      passthru = old.passthru // { llvm = pkgs.llvm; };
+    }
+  );
+
+  lockfile = super.lockfile.overrideAttrs (
+    old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.pbr ];
+    }
+  );
+
+  lxml = super.lxml.overrideAttrs (
+    old: {
+      nativeBuildInputs = with pkgs; old.nativeBuildInputs ++ [ pkgconfig libxml2.dev libxslt.dev ];
+      buildInputs = with pkgs; old.buildInputs ++ [ libxml2 libxslt ];
+    }
+  );
+
+  markupsafe = super.markupsafe.overrideAttrs (
+    old: {
+      src = old.src.override { pname = builtins.replaceStrings [ "markupsafe" ] [ "MarkupSafe" ] old.pname; };
+    }
+  );
+
+  matplotlib = super.matplotlib.overrideAttrs (
+    old: {
+      NIX_CFLAGS_COMPILE = stdenv.lib.optionalString stdenv.isDarwin "-I${pkgs.libcxx}/include/c++/v1";
+
+      XDG_RUNTIME_DIR = "/tmp";
+
+      nativeBuildInputs = old.nativeBuildInputs ++ [
+        pkgs.pkgconfig
+      ];
+
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [
+        pkgs.libpng
+        pkgs.freetype
+      ];
+
+      inherit (super.matplotlib) patches;
+    }
+  );
+
+  mccabe = super.mccabe.overrideAttrs (
+    old: {
+      postPatch = ''
+        substituteInPlace setup.py --replace "setup_requires=['pytest-runner']," "setup_requires=[]," || true
+      '';
+    }
+  );
+
+  netcdf4 = super.netcdf4.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.cython
+      ];
+
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [
+        pkgs.zlib
+        pkgs.netcdf
+        pkgs.hdf5
+        pkgs.curl
+        pkgs.libjpeg
+      ];
+
+      # Variables used to configure the build process
+      USE_NCCONFIG = "0";
+      HDF5_DIR = lib.getDev pkgs.hdf5;
+      NETCDF4_DIR = pkgs.netcdf;
+      CURL_DIR = pkgs.curl.dev;
+      JPEG_DIR = pkgs.libjpeg.dev;
+    }
+  );
+
+  numpy = super.numpy.overrideAttrs (
+    old: let
+      blas = pkgs.openblasCompat;
+      blasImplementation = lib.nameFromURL blas.name "-";
+      cfg = pkgs.writeTextFile {
+        name = "site.cfg";
+        text = (
+          lib.generators.toINI {} {
+            ${blasImplementation} = {
+              include_dirs = "${blas}/include";
+              library_dirs = "${blas}/lib";
+            } // lib.optionalAttrs (blasImplementation == "mkl") {
+              mkl_libs = "mkl_rt";
+              lapack_libs = "";
+            };
+          }
+        );
+      };
+    in
+      {
+        nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ];
+        buildInputs = old.buildInputs ++ [ blas ];
+        enableParallelBuilding = true;
+        preBuild = ''
+          ln -s ${cfg} site.cfg
+        '';
+        passthru = old.passthru // {
+          blas = blas;
+          inherit blasImplementation cfg;
+        };
+      }
+  );
+
+  pillow = super.pillow.overrideAttrs (
+    old: {
+      nativeBuildInputs = [ pkgs.pkgconfig ] ++ old.nativeBuildInputs;
+      buildInputs = with pkgs; [ freetype libjpeg zlib libtiff libwebp tcl lcms2 ] ++ old.buildInputs;
+    }
+  );
+
+  pluggy = addSetupTools super.pluggy;
+
+  psycopg2 = super.psycopg2.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ];
+    }
+  );
+
+  psycopg2-binary = super.psycopg2-binary.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.postgresql ];
+    }
+  );
+
+  py = addSetupTools super.py;
+
+  pyarrow = super.pyarrow.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.cython
+      ];
+    }
+  );
+
+  pycairo = (
+    drv: (
+      drv.overridePythonAttrs (
+        _: {
+          format = "other";
+        }
+      )
+    ).overrideAttrs (
+      old: {
+
+        nativeBuildInputs = old.nativeBuildInputs ++ [
+          pkgs.meson
+          pkgs.ninja
+          pkgs.pkgconfig
+        ];
+
+        propagatedBuildInputs = old.propagatedBuildInputs ++ [
+          pkgs.cairo
+          pkgs.xlibsWrapper
+        ];
+
+        mesonFlags = [ "-Dpython=${if self.isPy3k then "python3" else "python"}" ];
+      }
+    )
+  ) super.pycairo;
+
+  pycocotools = super.pycocotools.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.cython
+        self.numpy
+      ];
+    }
+  );
+
+  pygobject = super.pygobject.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkgconfig ];
+      buildInputs = old.buildInputs ++ [ pkgs.glib pkgs.gobject-introspection ];
+    }
+  );
+
+  pyopenssl = super.pyopenssl.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ pkgs.openssl ];
+    }
+  );
+
+  pytest = addSetupTools super.pytest;
+
+  pytest-mock = addSetupTools super.pytest-mock;
+
+  python-dateutil = addSetupTools super.python-dateutil;
+
+  python-prctl = super.python-prctl.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [
+        self.setuptools_scm
+        pkgs.libcap
+      ];
+    }
+  );
+
+  scaleapi = super.scaleapi.overrideAttrs (
+    old: {
+      postPatch = ''
+        substituteInPlace setup.py --replace "install_requires = ['requests>=2.4.2', 'enum34']" "install_requires = ['requests>=2.4.2']" || true
+      '';
+    }
+  );
+
+  scipy = super.scipy.overrideAttrs (
+    old: {
+      nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gfortran ];
+      setupPyBuildFlags = [ "--fcompiler='gnu95'" ];
+      enableParallelBuilding = true;
+      buildInputs = old.buildInputs ++ [ self.numpy.blas ];
+      preConfigure = ''
+        sed -i '0,/from numpy.distutils.core/s//import setuptools;from numpy.distutils.core/' setup.py
+        export NPY_NUM_BUILD_JOBS=$NIX_BUILD_CORES
+      '';
+      preBuild = ''
+        ln -s ${self.numpy.cfg} site.cfg
+      '';
+    }
+  );
+
+  shapely = super.shapely.overrideAttrs (
+    old: {
+      buildInputs = old.buildInputs ++ [ pkgs.geos self.cython ];
+      inherit (super.shapely) patches GEOS_LIBRARY_PATH;
+    }
+  );
+
+  six = addSetupTools super.six;
+
+  urwidtrees = super.urwidtrees.overrideAttrs (
+    old: {
+      propagatedBuildInputs = old.propagatedBuildInputs ++ [
+        self.urwid
+      ];
+    }
+  );
+
+  # TODO: Figure out getting rid of this hack
+  wheel = (
+    pkgs.python3.pkgs.override {
+      python = self.python;
+    }
+  ).wheel.overridePythonAttrs (
+    _: {
+      inherit (super.wheel) pname name version src;
+    }
+  );
+
+  zipp = addSetupTools super.zipp;
+}

pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix

@@ -0,0 +1,106 @@
+{ lib, stdenv, python, isLinux ? stdenv.isLinux }:
+
+let
+  inherit (lib.strings) hasSuffix hasInfix splitString removeSuffix;
+
+  # The 'cpxy" as determined by `python.version`
+  #
+  # e.g "2.7.17" -> "cp27"
+  #     "3.5.9"  -> "cp35"
+  pythonTag =
+    let
+      ver = builtins.splitVersion python.version;
+      major = builtins.elemAt ver 0;
+      minor = builtins.elemAt ver 1;
+    in
+      "cp${major}${minor}";
+
+  abiTag = "${pythonTag}m";
+
+  #
+  # Parses wheel file returning an attribute set
+  #
+  toWheelAttrs = str:
+    let
+      entries = splitString "-" str;
+      p = removeSuffix ".whl" (builtins.elemAt entries 4);
+    in
+      {
+        pkgName = builtins.elemAt entries 0;
+        pkgVer = builtins.elemAt entries 1;
+        pyVer = builtins.elemAt entries 2;
+        abi = builtins.elemAt entries 3;
+        platform = p;
+      };
+
+  #
+  # Builds list of acceptable osx wheel files
+  #
+  # <versions>   accepted versions in descending order of preference
+  # <candidates> list of wheel files to select from
+  findBestMatches = versions: candidates:
+    let
+      v = lib.lists.head versions;
+      vs = lib.lists.tail versions;
+    in
+      if (builtins.length versions == 0)
+      then []
+      else (builtins.filter (x: hasInfix v x.file) candidates) ++ (findBestMatches vs candidates);
+
+  # pyver = "cpXX"
+  # x     = "cpXX" | "py2" | "py3" | "py2.py3"
+  isPyVersionCompatible = pyver: x:
+    let
+      normalize = y: ''cp${lib.strings.removePrefix "cp" (lib.strings.removePrefix "py" y)}'';
+      isCompat = p: x: lib.strings.hasPrefix (normalize x) p;
+    in
+      lib.lists.any (isCompat pyver) (lib.strings.splitString "." x);
+
+  #
+  # Selects the best matching wheel file from a list of files
+  #
+  selectWheel = files:
+    let
+      filesWithoutSources = (builtins.filter (x: hasSuffix ".whl" x.file) files);
+
+      isPyAbiCompatible = pyabi: x: x == "none" || pyabi == x;
+
+      withPython = ver: abi: x: (isPyVersionCompatible ver x.pyVer) && (isPyAbiCompatible abi x.abi);
+
+      withPlatform = if isLinux
+      then (
+        x: x.platform == "manylinux1_${stdenv.platform.kernelArch}"
+        || x.platform == "manylinux2010_${stdenv.platform.kernelArch}"
+        || x.platform == "manylinux2014_${stdenv.platform.kernelArch}"
+        || x.platform == "any"
+      )
+      else (x: hasInfix "macosx" x.platform || x.platform == "any");
+
+      filterWheel = x:
+        let
+          f = toWheelAttrs x.file;
+        in
+          (withPython pythonTag abiTag f) && (withPlatform f);
+
+      filtered = builtins.filter filterWheel filesWithoutSources;
+
+      choose = files:
+        let
+          osxMatches = [ "10_12" "10_11" "10_10" "10_9" "any" ];
+          linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ];
+          chooseLinux = x: lib.singleton (builtins.head (findBestMatches linuxMatches x));
+          chooseOSX = x: lib.singleton (builtins.head (findBestMatches osxMatches x));
+        in
+          if isLinux
+          then chooseLinux files
+          else chooseOSX files;
+
+    in
+      if (builtins.length filtered == 0)
+      then []
+      else choose (filtered);
+
+in
+{
+  inherit selectWheel toWheelAttrs isPyVersionCompatible;
+}

pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix

@@ -0,0 +1,218 @@
+{ lib, stdenv }: python:
+
+let
+
+  # Like builtins.substring but with stop being offset instead of length
+  substr = start: stop: s: builtins.substring start (stop - start) s;
+
+  # Strip leading/trailing whitespace from string
+  stripStr = s: lib.elemAt (builtins.split "^ *" (lib.elemAt (builtins.split " *$" s) 0)) 2;
+
+  findSubExpressionsFun = acc: c: (
+    if c == "(" then (
+      let
+        posNew = acc.pos + 1;
+        isOpen = acc.openP == 0;
+        startPos = if isOpen then posNew else acc.startPos;
+      in
+        acc // {
+          inherit startPos;
+          exprs = acc.exprs ++ [ (substr acc.exprPos (acc.pos - 1) acc.expr) ];
+          pos = posNew;
+          openP = acc.openP + 1;
+        }
+    ) else if c == ")" then (
+      let
+        openP = acc.openP - 1;
+        exprs = findSubExpressions (substr acc.startPos acc.pos acc.expr);
+      in
+        acc // {
+          inherit openP;
+          pos = acc.pos + 1;
+          exprs = if openP == 0 then acc.exprs ++ [ exprs ] else acc.exprs;
+          exprPos = if openP == 0 then acc.pos + 1 else acc.exprPos;
+        }
+    ) else acc // { pos = acc.pos + 1; }
+  );
+
+  # Make a tree out of expression groups (parens)
+  findSubExpressions = expr: let
+    acc = builtins.foldl' findSubExpressionsFun {
+      exprs = [];
+      expr = expr;
+      pos = 0;
+      openP = 0;
+      exprPos = 0;
+      startPos = 0;
+    } (lib.stringToCharacters expr);
+    tailExpr = (substr acc.exprPos acc.pos expr);
+    tailExprs = if tailExpr != "" then [ tailExpr ] else [];
+  in
+    acc.exprs ++ tailExprs;
+
+  parseExpressions = exprs: let
+    splitCond = (
+      s: builtins.map
+        (x: stripStr (if builtins.typeOf x == "list" then (builtins.elemAt x 0) else x))
+        (builtins.split " (and|or) " (s + " "))
+    );
+
+    mapfn = expr: (
+      if (builtins.match "^ ?$" expr != null) then null  # Filter empty
+      else if (builtins.elem expr [ "and" "or" ]) then {
+        type = "bool";
+        value = expr;
+      }
+      else {
+        type = "expr";
+        value = expr;
+      }
+    );
+
+    parse = expr: builtins.filter (x: x != null) (builtins.map mapfn (splitCond expr));
+
+  in
+    builtins.foldl' (
+      acc: v: acc ++ (
+        if builtins.typeOf v == "string" then parse v else [ (parseExpressions v) ]
+      )
+    ) [] exprs;
+
+  # Transform individual expressions to structured expressions
+  # This function also performs variable substitution, replacing environment markers with their explicit values
+  transformExpressions = exprs: let
+    variables = {
+      os_name = "posix"; # TODO: Check other platforms
+      sys_platform = (
+        if stdenv.isLinux then "linux"
+        else if stdenv.isDarwin then "darwin"
+        else throw "Unsupported platform"
+      );
+      platform_machine = stdenv.platform.kernelArch;
+      platform_python_implementation = "CPython"; # Only CPython supported for now
+      platform_release = ""; # Field not reproducible
+      platform_system = (
+        if stdenv.isLinux then "Linux"
+        else if stdenv.isDarwin then "Darwin"
+        else throw "Unsupported platform"
+      );
+      platform_version = ""; # Field not reproducible
+      python_version = python.passthru.pythonVersion;
+      python_full_version = python.version;
+      implementation_name = "cpython"; # Only cpython supported for now
+      implementation_version = python.version;
+      extra = "";
+    };
+
+    substituteVar = value: if builtins.hasAttr value variables then (builtins.toJSON variables."${value}") else value;
+
+    processVar = value: builtins.foldl' (acc: v: v acc) value [
+      stripStr
+      substituteVar
+    ];
+
+  in
+    if builtins.typeOf exprs == "set" then (
+      if exprs.type == "expr" then (
+        let
+          mVal = ''[a-zA-Z0-9\'"_\. ]+'';
+          mOp = "in|[!=<>]+";
+          e = stripStr exprs.value;
+          m = builtins.map stripStr (builtins.match ''^(${mVal}) *(${mOp}) *(${mVal})$'' e);
+        in
+          {
+            type = "expr";
+            value = {
+              op = builtins.elemAt m 1;
+              values = [
+                (processVar (builtins.elemAt m 0))
+                (processVar (builtins.elemAt m 2))
+              ];
+            };
+          }
+      ) else exprs
+    ) else builtins.map transformExpressions exprs;
+
+  # Recursively eval all expressions
+  evalExpressions = exprs: let
+    unmarshal = v: (
+      # TODO: Handle single quoted values
+      if v == "True" then true
+      else if v == "False" then false
+      else builtins.fromJSON v
+    );
+    hasElem = needle: haystack: builtins.elem needle (builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " haystack));
+    # TODO: Implement all operators
+    op = {
+      "<=" = x: y: (unmarshal x) <= (unmarshal y);
+      "<" = x: y: (unmarshal x) < (unmarshal y);
+      "!=" = x: y: x != y;
+      "==" = x: y: x == y;
+      ">=" = x: y: (unmarshal x) >= (unmarshal y);
+      ">" = x: y: (unmarshal x) > (unmarshal y);
+      "~=" = null;
+      "===" = null;
+      "in" = x: y: let
+        values = builtins.filter (x: builtins.typeOf x == "string") (builtins.split " " (unmarshal y));
+      in
+        builtins.elem (unmarshal x) values;
+    };
+  in
+    if builtins.typeOf exprs == "set" then (
+      if exprs.type == "expr" then (
+        let
+          expr = exprs;
+          result = (op."${expr.value.op}") (builtins.elemAt expr.value.values 0) (builtins.elemAt expr.value.values 1);
+        in
+          {
+            type = "value";
+            value = result;
+          }
+      ) else exprs
+    ) else builtins.map evalExpressions exprs;
+
+  # Now that we have performed an eval all that's left to do is to concat the graph into a single bool
+  reduceExpressions = exprs: let
+    cond = {
+      "and" = x: y: x && y;
+      "or" = x: y: x || y;
+    };
+    reduceExpressionsFun = acc: v: (
+      if builtins.typeOf v == "set" then (
+        if v.type == "value" then (
+          acc // {
+            value = cond."${acc.cond}" acc.value v.value;
+          }
+        ) else if v.type == "bool" then (
+          acc // {
+            cond = v.value;
+          }
+        ) else throw "Unsupported type"
+      ) else if builtins.typeOf v == "list" then (
+        let
+          ret = builtins.foldl' reduceExpressionsFun {
+            value = true;
+            cond = "and";
+          } v;
+        in
+          acc // {
+            value = cond."${acc.cond}" acc.value ret.value;
+          }
+      ) else throw "Unsupported type"
+    );
+  in
+    (
+      builtins.foldl' reduceExpressionsFun {
+        value = true;
+        cond = "and";
+      } exprs
+    ).value;
+
+in
+e: builtins.foldl' (acc: v: v acc) e [
+  findSubExpressions
+  parseExpressions
+  transformExpressions
+  evalExpressions
+  reduceExpressions
+]

pkgs/development/tools/poetry2nix/poetry2nix/pyproject-without-path.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# Patch out path dependencies from a pyproject.json file
+
+import json
+import sys
+
+data = json.load(sys.stdin)
+
+for dep in data['tool']['poetry']['dependencies'].values():
+    if isinstance(dep, dict):
+        try:
+            del dep['path'];
+        except KeyError:
+            pass
+        else:
+            dep['version'] = '*'
+
+json.dump(data, sys.stdout, indent=4)

pkgs/development/tools/poetry2nix/poetry2nix/semver.nix

@@ -0,0 +1,80 @@
+{ lib }:
+
+let
+  inherit (builtins) elemAt match;
+
+  # Replace a list entry at defined index with set value
+  ireplace = idx: value: list: let
+    inherit (builtins) genList length;
+  in
+    genList (i: if i == idx then value else (elemAt list i)) (length list);
+
+  operators = let
+    matchWildCard = s: match "([^\*])(\.[\*])" s;
+    mkComparison = ret: version: v: builtins.compareVersions version v == ret;
+    mkIdxComparison = idx: version: v: let
+      ver = builtins.splitVersion v;
+      minor = builtins.toString (lib.toInt (elemAt ver idx) + 1);
+      upper = builtins.concatStringsSep "." (ireplace idx minor ver);
+    in
+      operators.">=" version v && operators."<" version upper;
+    dropWildcardPrecision = f: version: constraint: let
+      m = matchWildCard constraint;
+      hasWildcard = m != null;
+      c = if hasWildcard then (elemAt m 0) else constraint;
+      v =
+        if hasWildcard then (builtins.substring 0 (builtins.stringLength c) version)
+        else version;
+    in
+      f v c;
+  in
+    {
+      # Prefix operators
+      "==" = dropWildcardPrecision (mkComparison 0);
+      ">" = dropWildcardPrecision (mkComparison 1);
+      "<" = dropWildcardPrecision (mkComparison (-1));
+      "!=" = v: c: ! operators."==" v c;
+      ">=" = v: c: operators."==" v c || operators.">" v c;
+      "<=" = v: c: operators."==" v c || operators."<" v c;
+      # Semver specific operators
+      "~" = mkIdxComparison 1; #
+      "^" = mkIdxComparison 0;
+      # Infix operators
+      "-" = version: v: operators.">=" version v.vl && operators."<=" version v.vu;
+    };
+
+  re = {
+    operators = "([=><!~\^]+)";
+    version = "([0-9\.\*x]+)";
+  };
+
+  parseConstraint = constraint: let
+    constraintStr = builtins.replaceStrings [ " " ] [ "" ] constraint;
+    # The common prefix operators
+    mPre = match "${re.operators} *${re.version}" constraintStr;
+    # There is also an infix operator to match ranges
+    mIn = match "${re.version} *(-) *${re.version}" constraintStr;
+  in
+    (
+      if mPre != null then {
+        op = elemAt mPre 0;
+        v = elemAt mPre 1;
+      }
+        # Infix operators are range matches
+      else if mIn != null then {
+        op = elemAt mIn 1;
+        v = {
+          vl = (elemAt mIn 0);
+          vu = (elemAt mIn 2);
+        };
+      }
+      else throw "Constraint \"${constraintStr}\" could not be parsed"
+    );
+
+  satisfiesSemver = version: constraint: let
+    inherit (parseConstraint constraint) op v;
+  in
+    if constraint == "*" then true else operators."${op}" version v;
+
+in
+{ inherit satisfiesSemver; }

pkgs/development/tools/poetry2nix/update

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+pwd=$(pwd)
+workdir=$(mktemp -d)
+
+function cleanup {
+    cd "$pwd"
+    rm -rf $workdir
+}
+trap cleanup EXIT
+
+cd "$workdir"
+
+curl -L -s https://github.com/nix-community/poetry2nix/archive/master.tar.gz | tar -xz
+mv poetry2nix-master/* .
+
+mkdir build
+cp *.nix *.json *.py build/
+cp -r bin build/
+rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix
+
+cat > build/README.md << EOF
+Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix
+
+The update procedure is as-follows:
+1. Send your change to the upstream poetry2nix repository
+2. Get it approved with tests passing
+3. Run the update script in pkgs/development/tools/poetry2nix
+EOF
+
+rm -rf "$pwd/poetry2nix"
+mv build "$pwd/poetry2nix"

pkgs/top-level/all-packages.nix

@@ -9296,7 +9296,12 @@ in
   svg2tikz = python27Packages.svg2tikz;
 
   pew = callPackage ../development/tools/pew {};
+
   poetry = with python3Packages; toPythonApplication poetry;
+  poetry2nix = callPackage ../development/tools/poetry2nix/poetry2nix {
+    inherit pkgs lib;
+  };
+
   pipenv = callPackage ../development/tools/pipenv {};
 
   pipewire = callPackage ../development/libraries/pipewire {};