Python: improve cross-compilation

This changeset allows for cross-compilation of Python packages. Packages
built with buildPythonPackage are not allowed to refer to the build
machine. Executables that have shebangs will refer to the host.

pkgs/development/interpreters/python/build-python-package-common.nix

@@ -1,7 +1,6 @@
 # This function provides generic bits to install a Python wheel.
 
 { python
-, bootstrapped-pip
 }:
 
 { buildInputs ? []
@@ -10,7 +9,7 @@
 , ... } @ attrs:
 
 attrs // {
-  buildInputs = buildInputs ++ [ bootstrapped-pip ];
+  buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
 
   configurePhase = attrs.configurePhase or ''
     runHook preConfigure
@@ -24,7 +23,7 @@ attrs // {
     export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
 
     pushd dist
-    ${bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
+    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
     popd
 
     runHook postInstall

pkgs/development/interpreters/python/build-python-package-setuptools.nix

@@ -2,7 +2,6 @@
 
 { lib
 , python
-, bootstrapped-pip
 }:
 
 {
@@ -26,13 +25,13 @@ in attrs // {
   buildPhase = attrs.buildPhase or ''
     runHook preBuild
     cp ${setuppy} nix_run_setup
-    ${python.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
+    ${python.pythonForBuild.interpreter} nix_run_setup ${lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags))} bdist_wheel
     runHook postBuild
   '';
 
   installCheckPhase = attrs.checkPhase or ''
     runHook preCheck
-    ${python.interpreter} nix_run_setup test
+    ${python.pythonForBuild.interpreter} nix_run_setup test
     runHook postCheck
   '';
 
@@ -47,9 +46,9 @@ in attrs // {
     if test -e setup.py; then
       tmp_path=$(mktemp -d)
       export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.sitePackages}
-      ${bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
+      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
+      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
+      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
     fi
     ${postShellHook}
   '';

pkgs/development/interpreters/python/build-python-package.nix

@@ -10,17 +10,16 @@
 , ensureNewerSourcesForZipFilesHook
 , toPythonModule
 , namePrefix
-, bootstrapped-pip
 , flit
 , writeScript
 , update-python-libraries
 }:
 
 let
-  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python bootstrapped-pip; };
+  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
   flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
   wheel-specific = import ./build-python-package-wheel.nix { };
-  common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
+  common = import ./build-python-package-common.nix { inherit python; };
   mkPythonDerivation = import ./mk-python-derivation.nix {
     inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
     inherit toPythonModule namePrefix writeScript update-python-libraries;

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -32,6 +32,9 @@ assert x11Support -> tcl != null
 with stdenv.lib;
 
 let
+
+  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
     implementation = "cpython";

pkgs/development/interpreters/python/cpython/default.nix

@@ -21,6 +21,7 @@
 , sourceVersion
 , sha256
 , passthruFun
+, bash
 }:
 
 assert x11Support -> tcl != null
@@ -46,7 +47,8 @@ let
   nativeBuildInputs = [
     nukeReferences
   ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-    buildPackages.stdenv.cc crossPython
+    buildPackages.stdenv.cc
+    pythonForBuild
   ];
 
   buildInputs = filter (p: p != null) [
@@ -56,11 +58,11 @@ let
 
   hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
 
-  crossPython = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
 
-  pythonForBuild = if stdenv.hostPlatform == stdenv.buildPlatform then
+  pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
     "$out/bin/python"
-  else crossPython.interpreter;
+  else pythonForBuild.interpreter;
 
 in with passthru; stdenv.mkDerivation {
   pname = "python3";
@@ -215,14 +217,25 @@ in with passthru; stdenv.mkDerivation {
     # We rebuild three times, once for each optimization level
     # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
     # This is automatically used when `SOURCE_DATE_EPOCH` is set.
-    find $out -name "*.py" | ${pythonForBuild}     -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | ${pythonForBuild} -O  -m compileall -q -f -x "lib2to3" -i -
-    find $out -name "*.py" | ${pythonForBuild} -OO -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter}     -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
+  '';
+
+  preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+    # Ensure patch-shebangs uses shebangs of host interpreter.
+    export PATH=${stdenv.lib.makeBinPath [ "$out" bash ]}:$PATH
   '';
 
   # Enforce that we don't have references to the OpenSSL -dev package, which we
   # explicitly specify in our configure flags above.
-  disallowedReferences = [ openssl.dev ];
+  disallowedReferences = [
+    openssl.dev
+  ] ++ stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+    # Ensure we don't have references to build-time packages.
+    # These typically end up in shebangs.
+    pythonForBuild buildPackages.bash
+  ];
 
   inherit passthru;
 

pkgs/development/interpreters/python/default.nix

@@ -21,7 +21,6 @@ with pkgs;
         overrides = packageOverrides;
       };
     in rec {
-        sitePackages = "lib/${libPrefix}/site-packages";
         isPy27 = pythonVersion == "2.7";
         isPy33 = pythonVersion == "3.3"; # TODO: remove
         isPy34 = pythonVersion == "3.4"; # TODO: remove
@@ -35,7 +34,7 @@ with pkgs;
         withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;};
         pkgs = pythonPackages;
         interpreter = "${self}/bin/${executable}";
-        inherit executable implementation libPrefix pythonVersion;
+        inherit executable implementation libPrefix pythonVersion sitePackages;
         inherit sourceVersion;
         pythonAtLeast = lib.versionAtLeast pythonVersion;
         pythonOlder = lib.versionOlder pythonVersion;
@@ -112,8 +111,8 @@ in {
     inherit passthruFun;
   };
 
-  pypy3 = callPackage ./pypy {
-    self = pypy3;
+  pypy35 = callPackage ./pypy {
+    self = pypy35;
     sourceVersion = {
       major = "6";
       minor = "0";

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -77,7 +77,7 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
 
   buildInputs = [ wrapPython ]
     ++ lib.optional (lib.hasSuffix "zip" (attrs.src.name or "")) unzip
-    ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
+#     ++ lib.optional catchConflicts setuptools # If we no longer propagate setuptools
     ++ buildInputs
     ++ pythonPath;
 
@@ -100,9 +100,12 @@ let self = toPythonModule (python.stdenv.mkDerivation (builtins.removeAttrs attr
     # Check if we have two packages with the same name in the closure and fail.
     # If this happens, something went wrong with the dependencies specs.
     # Intentionally kept in a subdirectory, see catch_conflicts/README.md.
-    ${python.interpreter} ${./catch_conflicts}/catch_conflicts.py
+    ${python.pythonForBuild.interpreter} ${./catch_conflicts}/catch_conflicts.py
   '' + attrs.postFixup or '''';
 
+  # Python packages built through cross-compilation are always for the host platform.
+  disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ];
+
   meta = {
     # default to python's platforms
     platforms = python.meta.platforms;

pkgs/development/interpreters/python/wrap-python.nix

@@ -9,7 +9,8 @@ makeSetupHook {
       deps = makeWrapper;
       substitutions.sitePackages = python.sitePackages;
       substitutions.executable = python.interpreter;
-      substitutions.python = python;
+      substitutions.python = python.pythonForBuild;
+      substitutions.pythonHost = python;
       substitutions.magicalSedExpression = let
         # Looks weird? Of course, it's between single quoted shell strings.
         # NOTE: Order DOES matter here, so single character quotes need to be

pkgs/development/interpreters/python/wrap.sh

@@ -16,8 +16,8 @@ buildPythonPath() {
     declare -A pythonPathsSeen=()
     program_PYTHONPATH=
     program_PATH=
-    pythonPathsSeen["@python@"]=1
-    addToSearchPath program_PATH @python@/bin
+    pythonPathsSeen["@pythonHost@"]=1
+    addToSearchPath program_PATH @pythonHost@/bin
     for path in $pythonPath; do
         _addToPythonPath $path
     done
@@ -53,7 +53,13 @@ wrapPythonProgramsIn() {
             # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
             # Python to use besides one with this hook anyway.
             if head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'; then
-                sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#! @executable@^"
+                sed -i "$f" -e "1 s^.*/env[ ]*\(python\|pypy\)[^ ]*^#!@executable@^"
+            fi
+
+            if head -n1 "$f" | grep -q '#!.*'; then
+                # Cross-compilation hack: ensure shebangs are for the host
+                echo "Rewriting $(head -n 1 $f) to #!@pythonHost@"
+                sed -i "$f" -e "1 s^#!@python@^#!@pythonHost@^"
             fi
 
             # catch /python and /.python-wrapped

pkgs/development/python-modules/cypari2/default.nix

@@ -1,5 +1,4 @@
 { stdenv
-, bootstrapped-pip
 , buildPythonPackage
 , python
 , fetchPypi
@@ -24,11 +23,11 @@ buildPythonPackage rec {
   # That is because while the default install phase succeeds to build the package,
   # it fails to generate the file "auto_paridecl.pxd".
   installPhase = ''
-    mkdir -p "$out/lib/${python.libPrefix}/site-packages"
-    export PYTHONPATH="$out/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
+    mkdir -p "$out/lib/${python.sitePackages}"
+    export PYTHONPATH="$out/lib/${python.sitePackages}:$PYTHONPATH"
 
     # install "." instead of "*.whl"
-    ${bootstrapped-pip}/bin/pip install --no-index --prefix=$out --no-cache --build=tmpdir .
+    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install --no-index --prefix=$out --no-cache --build=tmpdir .
   '';
 
   buildInputs = [

pkgs/development/python-modules/pyyaml/default.nix

@@ -1,4 +1,4 @@
-{ lib, buildPythonPackage, fetchPypi, libyaml }:
+{ lib, buildPythonPackage, fetchPypi, libyaml, buildPackages }:
 
 buildPythonPackage rec {
   pname = "PyYAML";
@@ -9,6 +9,8 @@ buildPythonPackage rec {
     sha256 = "3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf";
   };
 
+  nativeBuildInputs = [ buildPackages.stdenv.cc ];
+
   propagatedBuildInputs = [ libyaml ];
 
   meta = with lib; {

pkgs/development/python-modules/setuptools/default.nix

@@ -17,19 +17,24 @@ stdenv.mkDerivation rec {
     sha256 = "86bb4d8e1b0fabad1f4642b64c335b673e53e7a381de03c9a89fe678152c4c64";
   };
 
-  nativeBuildInputs = [ unzip wrapPython ];
-  buildInputs = [ python ];
+  nativeBuildInputs = [ unzip wrapPython python.pythonForBuild ];
   doCheck = false;  # requires pytest
   installPhase = ''
       dst=$out/${python.sitePackages}
       mkdir -p $dst
       export PYTHONPATH="$dst:$PYTHONPATH"
-      ${python.interpreter} setup.py install --prefix=$out
+      ${python.pythonForBuild.interpreter} setup.py install --prefix=$out
       wrapPythonPrograms
   '';
 
   pythonPath = [];
 
+  dontPatchShebangs = true;
+
+  # Python packages built through cross-compilation are always for the host platform.
+  disallowedReferences = stdenv.lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ python.pythonForBuild ];
+
+
   meta = with stdenv.lib; {
     description = "Utilities to facilitate the installation of Python packages";
     homepage = https://pypi.python.org/pypi/setuptools;

pkgs/top-level/all-packages.nix

@@ -7937,6 +7937,7 @@ in
   python3 = python37;
   pypy = pypy2;
   pypy2 = pypy27;
+  pypy3 = pypy35;
 
   # Python interpreter that is build with all modules, including tkinter.
   # These are for compatibility and should not be used inside Nixpkgs.
@@ -7954,7 +7955,7 @@ in
   python3Packages = python3.pkgs;
 
   pythonInterpreters = callPackage ./../development/interpreters/python {};
-  inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy3;
+  inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy35;
 
   # Python package sets.
   python27Packages = lib.hiPrioSet (recurseIntoAttrs python27.pkgs);

pkgs/top-level/python-packages.nix

@@ -43,7 +43,6 @@ let
       else ff;
 
   buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
-    inherit bootstrapped-pip;
     flit = self.flit;
     # We want Python libraries to be named like e.g. "python3.6-${name}"
     inherit namePrefix;
@@ -51,7 +50,6 @@ let
   }));
 
   buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
-    inherit bootstrapped-pip;
     flit = self.flit;
     namePrefix = "";
     toPythonModule = x: x; # Application does not provide modules.