From 1d820c22244634ad2936c73ac8680f5086d4e8e5 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 12 Jan 2022 17:53:58 +1300 Subject: [PATCH] poetry2nix: 1.24.1 -> 1.25.0 --- .../tools/poetry2nix/poetry2nix/default.nix | 44 +++++- .../poetry2nix/fetch_from_legacy.py | 40 ++--- .../hooks/pyproject-without-special-deps.py | 7 +- .../tools/poetry2nix/poetry2nix/overrides.nix | 144 ++++++++++++++++-- .../tools/poetry2nix/poetry2nix/pep425.nix | 10 +- 5 files changed, 203 insertions(+), 42 deletions(-) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index 07e0063d6c5..c9b70c83bfe 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -5,7 +5,7 @@ }: let # Poetry2nix version - version = "1.24.1"; + version = "1.26.0"; inherit (poetryLib) isCompatible readTOML moduleName; @@ -122,10 +122,10 @@ lib.makeScope pkgs.newScope (self: { # Example: { my-app = ./src; } , editablePackageSources ? { } , __isBootstrap ? false # Hack: Always add Poetry as a build input unless bootstrapping + , pyProject ? readTOML pyproject }@attrs: let poetryPkg = poetry.override { inherit python; }; - pyProject = readTOML pyproject; scripts = pyProject.tool.poetry.scripts or { }; hasScripts = scripts != { }; @@ -133,9 +133,11 @@ lib.makeScope pkgs.newScope (self: { inherit python scripts; }; - hasEditable = editablePackageSources != { }; + editablePackageSources' = lib.filterAttrs (name: path: path != null) editablePackageSources; + hasEditable = editablePackageSources' != { }; editablePackage = self.mkPoetryEditablePackage { - inherit pyProject python editablePackageSources; + inherit pyProject python; + editablePackageSources = editablePackageSources'; }; poetryLock = readTOML poetrylock; @@ -190,7 +192,10 @@ lib.makeScope pkgs.newScope (self: { (lib.reverseList compatible) ); in - lockPkgs; + lockPkgs // { + # Create a dummy null package for the current project in case any dependencies depend on the root project (issue #307) + ${pyProject.tool.poetry.name} = null; + }; overlays = builtins.map getFunctorFn ( @@ -264,14 +269,34 @@ lib.makeScope pkgs.newScope (self: { , extraPackages ? ps: [ ] }: let + inherit (lib) elem hasAttr; + + pyProject = readTOML pyproject; + + # Automatically add dependencies with develop = true as editable packages, but only if path dependencies + getEditableDeps = set: lib.mapAttrs + (name: value: projectDir + "/${value.path}") + (lib.filterAttrs (name: dep: dep.develop or false && hasAttr "path" dep) set); + + editablePackageSources' = ( + (getEditableDeps (pyProject.tool.poetry."dependencies" or { })) + // (getEditableDeps (pyProject.tool.poetry."dev-dependencies" or { })) + // editablePackageSources + ); + poetryPython = self.mkPoetryPackages { - inherit pyproject poetrylock overrides python pwd preferWheels editablePackageSources; + inherit pyproject poetrylock overrides python pwd preferWheels pyProject; + editablePackageSources = editablePackageSources'; }; inherit (poetryPython) poetryPackages; + # Don't add editable sources to the environment since they will sometimes fail to build and are not useful in the development env + editableAttrs = lib.attrNames editablePackageSources'; + envPkgs = builtins.filter (drv: ! lib.elem (drv.pname or drv.name or "") editableAttrs) poetryPackages; + in - poetryPython.python.withPackages (ps: poetryPackages ++ (extraPackages ps)); + poetryPython.python.withPackages (ps: envPkgs ++ (extraPackages ps)); /* Creates a Python application from pyproject.toml and poetry.lock @@ -282,7 +307,10 @@ lib.makeScope pkgs.newScope (self: { */ mkPoetryApplication = { projectDir ? null - , src ? self.cleanPythonSources { src = projectDir; } + , src ? ( + # Assume that a project which is the result of a derivation is already adequately filtered + if lib.isDerivation projectDir then projectDir else self.cleanPythonSources { src = projectDir; } + ) , pyproject ? projectDir + "/pyproject.toml" , poetrylock ? projectDir + "/poetry.lock" , overrides ? self.defaultPoetryOverrides diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/fetch_from_legacy.py b/pkgs/development/tools/poetry2nix/poetry2nix/fetch_from_legacy.py index d59c3a7763a..8858b64ec3e 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/fetch_from_legacy.py +++ b/pkgs/development/tools/poetry2nix/poetry2nix/fetch_from_legacy.py @@ -63,30 +63,36 @@ context.verify_mode = ssl.CERT_NONE req = urllib.request.Request(index_url) if username and password: import base64 - password_b64 = base64.b64encode(bytes(f"{username}:{password}", "utf-8")).decode("utf-8") - req.add_header("Authorization", f"Basic {password_b64}") -response = urllib.request.urlopen( - req, - context=context) + + password_b64 = base64.b64encode(":".join((username, password)).encode()).decode( + "utf-8" + ) + req.add_header("Authorization", "Basic {}".format(password_b64)) +response = urllib.request.urlopen(req, context=context) index = response.read() parser = Pep503() parser.feed(str(index)) if package_filename not in parser.sources: - print("The file %s has not be found in the index %s" % ( - package_filename, index_url)) + print( + "The file %s has not be found in the index %s" % (package_filename, index_url) + ) exit(1) package_file = open(package_filename, "wb") # Sometimes the href is a relative path -if urlparse(parser.sources[package_filename]).netloc == '': +if urlparse(parser.sources[package_filename]).netloc == "": parsed_url = urlparse(index_url) - package_url = urlunparse(( - parsed_url.scheme, - parsed_url.netloc, - parser.sources[package_filename], - None, None, None, - )) + package_url = urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path + "/" + parser.sources[package_filename], + None, + None, + None, + ) + ) else: package_url = parser.sources[package_filename] @@ -106,10 +112,8 @@ print("Downloading %s" % real_package_url) req = urllib.request.Request(real_package_url) if username and password: - req.add_unredirected_header("Authorization", f"Basic {password_b64}") -response = urllib.request.urlopen( - req, - context=context) + req.add_unredirected_header("Authorization", "Basic {}".format(password_b64)) +response = urllib.request.urlopen(req, context=context) with response as r: shutil.copyfileobj(r, package_file) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py index af9816cf831..9f79f9afab5 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py +++ b/pkgs/development/tools/poetry2nix/poetry2nix/hooks/pyproject-without-special-deps.py @@ -22,7 +22,12 @@ def main(input, output, fields_to_remove): if any_removed: dep["version"] = "*" - json.dump(data, output, separators=(",", ":")) + # Set ensure_ascii to False because TOML is valid UTF-8 so text that can't + # be represented in ASCII is perfectly legitimate + # HACK: Setting ensure_asscii to False breaks Python2 for some dependencies (like cachy==0.3.0) + json.dump( + data, output, separators=(",", ":"), ensure_ascii=sys.version_info.major < 3 + ) if __name__ == "__main__": diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 6039e50d04e..6e35069a817 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -98,12 +98,26 @@ self: super: ''; }); + backports-functools-lru-cache = super.backports-functools-lru-cache.overridePythonAttrs (old: { + postPatch = '' + substituteInPlace setup.py --replace \ + 'setuptools.setup()' \ + 'setuptools.setup(version="${old.version}")' + ''; + }); + bcrypt = super.bcrypt.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.libffi ]; } ); + bjoern = super.bjoern.overridePythonAttrs ( + old: { + buildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.libev ]; + } + ); + black = super.black.overridePythonAttrs ( old: { dontPreferSetupPy = true; @@ -247,6 +261,36 @@ self: super: buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools ]; }); + dbus-python = super.dbus-python.overridePythonAttrs (old: { + outputs = [ "out" "dev" ]; + + postPatch = old.postPatch or "" + '' + substituteInPlace ./configure --replace /usr/bin/file ${pkgs.file}/bin/file + substituteInPlace ./dbus-python.pc.in --replace 'Cflags: -I''${includedir}' 'Cflags: -I''${includedir}/dbus-1.0' + ''; + + configureFlags = (old.configureFlags or [ ]) ++ [ + "PYTHON_VERSION=${lib.versions.major self.python.version}" + ]; + + preConfigure = lib.concatStringsSep "\n" [ + (old.preConfigure or "") + (if (lib.versionAtLeast stdenv.hostPlatform.darwinMinVersion "11" && stdenv.isDarwin) then '' + MACOSX_DEPLOYMENT_TARGET=10.16 + '' else "") + ]; + + preBuild = old.preBuild or "" + '' + make distclean + ''; + + nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ pkgs.pkg-config ]; + buildInputs = old.buildInputs or [ ] ++ [ pkgs.dbus pkgs.dbus-glib ] + # My guess why it's sometimes trying to -lncurses. + # It seems not to retain the dependency anyway. + ++ lib.optional (! self.python ? modules) pkgs.ncurses; + }); + dcli = super.dcli.overridePythonAttrs (old: { propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.setuptools ]; }); @@ -373,6 +417,13 @@ self: super: } ); + fastapi = super.fastapi.overridePythonAttrs ( + old: { + # Note: requires full flit, not just flit-core + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.flit ]; + } + ); + fastecdsa = super.fastecdsa.overridePythonAttrs (old: { buildInputs = old.buildInputs ++ [ pkgs.gmp.dev ]; }); @@ -504,6 +555,13 @@ self: super: propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.pyparsing ]; }); + icecream = super.icecream.overridePythonAttrs (old: { + # # ERROR: Could not find a version that satisfies the requirement executing>=0.3.1 (from icecream) (from versions: none) + postPatch = '' + substituteInPlace setup.py --replace 'executing>=0.3.1' 'executing' + ''; + }); + imagecodecs = super.imagecodecs.overridePythonAttrs ( old: { patchPhase = '' @@ -569,9 +627,9 @@ self: super: # disable the removal of pyproject.toml, required because of setuptools_scm dontPreferSetupPy = true; - postPatch = old.postPatch or "" + '' + postPatch = old.postPatch or "" + (lib.optionalString ((old.format or "") != "wheel") '' substituteInPlace setup.py --replace 'setuptools.setup()' 'setuptools.setup(version="${old.version}")' - ''; + ''); } ); @@ -867,13 +925,6 @@ self: super: buildInputs = oa.buildInputs ++ [ self.pbr ]; }); - moto = super.moto.overridePythonAttrs ( - old: { - buildInputs = (old.buildInputs or [ ]) ++ - [ self.sshpubkeys ]; - } - ); - mpi4py = super.mpi4py.overridePythonAttrs ( old: let @@ -988,8 +1039,18 @@ self: super: ); opencv-python = super.opencv-python.overridePythonAttrs ( - old: rec { - buildInputs = (old.buildInputs or [ ]) ++ [ self.scikit-build ]; + old: { + nativeBuildInputs = [ pkgs.cmake ] ++ old.nativeBuildInputs; + buildInputs = [ self.scikit-build ] ++ (old.buildInputs or [ ]); + dontUseCmakeConfigure = true; + } + ); + + opencv-contrib-python = super.opencv-contrib-python.overridePythonAttrs ( + old: { + nativeBuildInputs = [ pkgs.cmake ] ++ old.nativeBuildInputs; + buildInputs = [ self.scikit-build ] ++ (old.buildInputs or [ ]); + dontUseCmakeConfigure = true; } ); @@ -1007,6 +1068,13 @@ self: super: } ); + pantalaimon = super.pantalaimon.overridePythonAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ pkgs.installShellFiles ]; + postInstall = old.postInstall or "" + '' + installManPage docs/man/*.[1-9] + ''; + }); + paramiko = super.paramiko.overridePythonAttrs (old: { doCheck = false; # requires networking }); @@ -1077,6 +1145,10 @@ self: super: } ); + prettytable = super.prettytable.overridePythonAttrs (old: { + propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.setuptools ]; + }); + psycopg2 = super.psycopg2.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) @@ -1277,7 +1349,11 @@ self: super: } ); - pytezos = super.pytezos.override (old: { + pytaglib = super.pytaglib.overridePythonAttrs (old: { + buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.taglib ]; + }); + + pytezos = super.pytezos.overridePythonAttrs (old: { buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.libsodium ]; }); @@ -1323,6 +1399,7 @@ self: super: pkgs.qt5.qtsvg pkgs.qt5.qtdeclarative pkgs.qt5.qtwebchannel + pkgs.qt5.qt3d # self.pyqt5-sip self.sip ] @@ -1472,6 +1549,12 @@ self: super: } ); + python-olm = super.python-olm.overridePythonAttrs ( + old: { + buildInputs = old.buildInputs or [ ] ++ [ pkgs.olm ]; + } + ); + python-snappy = super.python-snappy.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.snappy ]; @@ -1496,6 +1579,13 @@ self: super: } ); + pyudev = super.pyudev.overridePythonAttrs (old: { + postPatch = '' + substituteInPlace src/pyudev/_ctypeslib/utils.py \ + --replace "find_library(name)" "'${pkgs.lib.getLib pkgs.systemd}/lib/libudev.so'" + ''; + }); + pyusb = super.pyusb.overridePythonAttrs ( old: { postPatch = '' @@ -1559,6 +1649,12 @@ self: super: } ); + requests-mock = super.requests-mock.overridePythonAttrs ( + old: { + propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ super.pbr ]; + } + ); + requests-unixsocket = super.requests-unixsocket.overridePythonAttrs ( old: { nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ self.pbr ]; @@ -1616,6 +1712,18 @@ self: super: } else old ); + scikit-image = super.scikit-image.overridePythonAttrs ( + old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ + self.cython + self.pythran + self.packaging + self.wheel + self.numpy + ]; + } + ); + scikit-learn = super.scikit-learn.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ [ @@ -1673,7 +1781,7 @@ self: super: tables = super.tables.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ [ self.pywavelets ]; - HDF5_DIR = "${pkgs.hdf5}"; + HDF5_DIR = lib.getDev pkgs.hdf5; nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.pkg-config ]; propagatedBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ pkgs.hdf5 self.numpy self.numexpr ]; } @@ -2126,6 +2234,10 @@ self: super: buildInputs = (old.buildInputs or [ ]) ++ [ self.pbr ]; }); + pysqlite = super.pysqlite.overridePythonAttrs (old: { + buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.sqlite ]; + }); + selinux = super.selinux.overridePythonAttrs (old: { buildInputs = (old.buildInputs or [ ]) ++ [ self.setuptools-scm-git-archive ]; }); @@ -2147,6 +2259,12 @@ self: super: sourceRoot = "."; }); + wcwidth = super.wcwidth.overridePythonAttrs (old: { + propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ + lib.optional self.isPy27 (self.backports-functools-lru-cache or self.backports_functools_lru_cache) + ; + }); + wtforms = super.wtforms.overridePythonAttrs (old: { buildInputs = (old.buildInputs or [ ]) ++ [ self.Babel ]; }); diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix index 1f6978b98a2..56f894c2e75 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix @@ -84,7 +84,13 @@ let ) else (x: x.platform == "any") - else (x: hasInfix "macosx" x.platform || x.platform == "any"); + else + if stdenv.isDarwin + then + if stdenv.targetPlatform.isAarch64 + then (x: x.platform == "any" || (hasInfix "macosx" x.platform && lib.lists.any (e: hasSuffix e x.platform) [ "arm64" "aarch64" ])) + else (x: x.platform == "any" || (hasInfix "macosx" x.platform && hasSuffix "x86_64" x.platform)) + else (x: x.platform == "any"); filterWheel = x: let f = toWheelAttrs x.file; @@ -93,7 +99,7 @@ let filtered = builtins.filter filterWheel filesWithoutSources; choose = files: let - osxMatches = [ "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ]; + osxMatches = [ "12_0" "11_0" "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ]; linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "any" ]; chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x); chooseOSX = x: lib.take 1 (findBestMatches osxMatches x);