python3Packages.apache-airflow: 1.10.5 -> 2.1.1

pkgs/development/python-modules/apache-airflow/default.nix

@@ -1,165 +1,202 @@
 { lib
 , stdenv
+, python
 , buildPythonPackage
 , fetchFromGitHub
-, fetchpatch
+, writeText
 , alembic
+, argcomplete
+, attrs
+, blinker
 , cached-property
-, configparser
+, cattrs
+, clickclick
 , colorlog
 , croniter
+, cryptography
 , dill
 , flask
 , flask-appbuilder
-, flask-admin
 , flask-caching
 , flask_login
-, flask-swagger
 , flask_wtf
-, flask-bcrypt
-, funcsigs
-, future
 , GitPython
+, graphviz
 , gunicorn
+, httpx
 , iso8601
-, json-merge-patch
+, importlib-resources
+, importlib-metadata
+, inflection
+, itsdangerous
 , jinja2
-, ldap3
-, lxml
+, jsonschema
 , lazy-object-proxy
+, lockfile
 , markdown
+, markupsafe
+, marshmallow-oneofschema
+, numpy
+, openapi-spec-validator
 , pandas
 , pendulum
 , psutil
 , pygments
+, pyjwt
 , python-daemon
 , python-dateutil
-, requests
+, python-nvd3
+, python-slugify
+, python3-openid
+, pyyaml
+, rich
 , setproctitle
-, snakebite
 , sqlalchemy
+, sqlalchemy-jsonfield
+, swagger-ui-bundle
 , tabulate
 , tenacity
 , termcolor
-, text-unidecode
-, thrift
-, tzlocal
 , unicodecsv
-, zope_deprecation
-, nose
-, pythonOlder
-, pythonAtLeast
+, werkzeug
+, pytest
+, freezegun
+, mkYarnPackage
 }:
+let
 
-buildPythonPackage rec {
-  pname = "apache-airflow";
-  version = "1.10.5";
-  # Upstream does not yet support python 3.8
-  # https://github.com/apache/airflow/issues/8674
-  disabled = pythonOlder "3.5" || pythonAtLeast "3.8";
+  version = "2.1.1rc1";
 
-  src = fetchFromGitHub rec {
+  airflow-src = fetchFromGitHub rec {
     owner = "apache";
     repo = "airflow";
     rev = version;
-    sha256 = "14fmhfwx977c9jdb2kgm93i6acx43l45ggj30rb37r68pzpb6l6h";
+    sha256 = "1vzzmcfgqni9rkf7ggh8mswnm3ffwaishcz1ysrwx0a96ilhm9q2";
   };
 
-  patches = [
-       # Not yet accepted: https://github.com/apache/airflow/pull/6562
-     (fetchpatch {
-       name = "avoid-warning-from-abc.collections";
-       url = "https://patch-diff.githubusercontent.com/raw/apache/airflow/pull/6562.patch";
-       sha256 = "0swpay1qlb7f9kgc56631s1qd9k82w4nw2ggvkm7jvxwf056k61z";
-     })
-       # Not yet accepted: https://github.com/apache/airflow/pull/6561
-     (fetchpatch {
-       name = "pendulum2-compatibility";
-       url = "https://patch-diff.githubusercontent.com/raw/apache/airflow/pull/6561.patch";
-       sha256 = "17hw8qyd4zxvib9zwpbn32p99vmrdz294r31gnsbkkcl2y6h9knk";
-     })
-  ];
+  # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
+  # This replicates this shell script, fixing bugs in yarn.lock and package.json
+
+  airflow-frontend = mkYarnPackage {
+    name = "airflow-frontend";
+
+    src = "${airflow-src}/airflow/www";
+    packageJSON = ./package.json;
+    yarnLock = ./yarn.lock;
+    yarnNix = ./yarn.nix;
+
+    distPhase = "true";
+
+    configurePhase = ''
+      cp -r $node_modules node_modules
+    '';
+
+    buildPhase = ''
+      yarn --offline build
+      find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
+    '';
+
+    installPhase = ''
+      mkdir -p $out/static/
+      cp -r static/dist $out/static
+    '';
+  };
+
+in
+buildPythonPackage rec {
+  pname = "apache-airflow";
+  inherit version;
+  src = airflow-src;
 
   propagatedBuildInputs = [
     alembic
+    argcomplete
+    attrs
+    blinker
     cached-property
+    cattrs
+    clickclick
     colorlog
-    configparser
     croniter
+    cryptography
     dill
     flask
-    flask-admin
     flask-appbuilder
-    flask-bcrypt
     flask-caching
     flask_login
-    flask-swagger
     flask_wtf
-    funcsigs
-    future
     GitPython
+    graphviz
     gunicorn
+    httpx
     iso8601
-    json-merge-patch
+    importlib-resources
+    importlib-metadata
+    inflection
+    itsdangerous
     jinja2
-    ldap3
-    lxml
+    jsonschema
     lazy-object-proxy
+    lockfile
     markdown
+    markupsafe
+    marshmallow-oneofschema
+    numpy
+    openapi-spec-validator
     pandas
     pendulum
     psutil
     pygments
+    pyjwt
     python-daemon
     python-dateutil
-    requests
+    python-nvd3
+    python-slugify
+    python3-openid
+    pyyaml
+    rich
     setproctitle
     sqlalchemy
+    sqlalchemy-jsonfield
+    swagger-ui-bundle
     tabulate
     tenacity
     termcolor
-    text-unidecode
-    thrift
-    tzlocal
     unicodecsv
-    zope_deprecation
+    werkzeug
+  ];
+
+  buildInputs = [
+    airflow-frontend
   ];
 
   checkInputs = [
-    snakebite
-    nose
+    freezegun
+    pytest
   ];
 
+  INSTALL_PROVIDERS_FROM_SOURCES = "true";
+
   postPatch = ''
-    substituteInPlace setup.py \
+    substituteInPlace setup.cfg \
+      --replace "importlib_resources~=1.4" "importlib_resources" \
+      --replace "importlib_metadata~=1.7" "importlib_metadata" \
+      --replace "tenacity~=6.2.0" "tenacity" \
+      --replace "pyjwt<2" "pyjwt" \
       --replace "flask>=1.1.0, <2.0" "flask" \
-      --replace "jinja2>=2.10.1, <2.11.0" "jinja2" \
-      --replace "pandas>=0.17.1, <1.0.0" "pandas" \
-      --replace "flask-caching>=1.3.3, <1.4.0" "flask-caching" \
-      --replace "flask-appbuilder>=1.12.5, <2.0.0" "flask-appbuilder" \
-      --replace "flask-admin==1.5.3" "flask-admin" \
       --replace "flask-login>=0.3, <0.5" "flask-login" \
-      --replace "cached_property~=1.5" "cached_property" \
-      --replace "dill>=0.2.2, <0.3" "dill" \
-      --replace "configparser>=3.5.0, <3.6.0" "configparser" \
-      --replace "colorlog==4.0.2" "colorlog" \
-      --replace "funcsigs==1.0.0" "funcsigs" \
-      --replace "flask-swagger==0.2.13" "flask-swagger" \
-      --replace "python-daemon>=2.1.1, <2.2" "python-daemon" \
-      --replace "alembic>=1.0, <2.0" "alembic" \
-      --replace "markdown>=2.5.2, <3.0" "markdown" \
-      --replace "future>=0.16.0, <0.17" "future" \
-      --replace "tenacity==4.12.0" "tenacity" \
-      --replace "text-unidecode==1.2" "text-unidecode" \
-      --replace "tzlocal>=1.4,<2.0.0" "tzlocal" \
-      --replace "sqlalchemy~=1.3" "sqlalchemy" \
-      --replace "gunicorn>=19.5.0, <20.0" "gunicorn"
-
-    # dumb-init is only needed for CI and Docker, not relevant for NixOS.
-    substituteInPlace setup.py \
-      --replace "'dumb-init>=1.2.2'," ""
-
-    substituteInPlace tests/core.py \
+      --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \
+      --replace "jinja2>=2.10.1, <2.12.0" "jinja2" \
+      --replace "attrs>=20.0, <21.0" "attrs" \
+      --replace "cattrs~=1.1, <1.7.0" "cattrs" \
+      --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \
+      --replace "docutils<0.17" "docutils" \
+      --replace "sqlalchemy>=1.3.18, <1.4" "sqlalchemy" \
+      --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \
+      --replace "werkzeug~=1.0, >=1.0.1" "werkzeug" \
+      --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous"
+
+    substituteInPlace tests/core/test_core.py \
       --replace "/bin/bash" "${stdenv.shell}"
   '';
 
@@ -174,11 +211,14 @@ buildPythonPackage rec {
    export PATH=$PATH:$out/bin
 
    airflow version
-   airflow initdb
-   airflow resetdb -y
-   nosetests tests.core.CoreTest
-   ## all tests
-   # nosetests --cover-package=airflow
+   airflow db init
+   airflow db reset -y
+
+   pytest tests/core/test_core.py
+  '';
+
+  postInstall = ''
+    cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static
   '';
 
   meta = with lib; {

pkgs/development/python-modules/apache-airflow/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "airflow-frontend",
+  "version": "2.1.1rc1",
+  "description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development",
+    "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress",
+    "build": "NODE_ENV=production webpack --colors --progress",
+    "lint": "eslint --ignore-path=.eslintignore --ext .js,.html .",
+    "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ."
+  },
+  "author": "Apache",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apache/airflow.git"
+  },
+  "homepage": "https://airflow.apache.org/",
+  "keywords": [
+    "big",
+    "data",
+    "workflow",
+    "airflow",
+    "d3",
+    "nerds",
+    "database",
+    "flask"
+  ],
+  "devDependencies": {
+    "babel": "^6.23.0",
+    "babel-core": "^6.26.3",
+    "babel-eslint": "^10.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-plugin-css-modules-transform": "^1.6.1",
+    "babel-polyfill": "^6.26.0",
+    "clean-webpack-plugin": "^3.0.0",
+    "copy-webpack-plugin": "^6.0.3",
+    "css-loader": "^3.4.2",
+    "eslint": "^7.5.0",
+    "eslint-config-airbnb-base": "^14.2.0",
+    "eslint-plugin-html": "^6.0.2",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-standard": "^4.0.1",
+    "file-loader": "^6.0.0",
+    "imports-loader": "^1.1.0",
+    "mini-css-extract-plugin": "1.6.0",
+    "moment-locales-webpack-plugin": "^1.2.0",
+    "optimize-css-assets-webpack-plugin": "6.0.0",
+    "style-loader": "^1.2.1",
+    "stylelint": "^13.6.1",
+    "stylelint-config-standard": "^20.0.0",
+    "url-loader": "4.1.0",
+    "webpack": "^4.16.3",
+    "webpack-cli": "^3.1.0",
+    "webpack-manifest-plugin": "^2.2.0"
+  },
+  "dependencies": {
+    "bootstrap-3-typeahead": "^4.0.2",
+    "codemirror": "^5.59.1",
+    "d3": "^3.4.4",
+    "d3-shape": "^2.1.0",
+    "d3-tip": "^0.9.1",
+    "dagre-d3": "^0.6.4",
+    "datatables.net": "^1.10.23",
+    "datatables.net-bs": "^1.10.23",
+    "eonasdan-bootstrap-datetimepicker": "^4.17.47",
+    "jquery": ">=3.4.0",
+    "jshint": "^2.12.0",
+    "moment-timezone": "^0.5.28",
+    "nvd3": "^1.8.6",
+    "redoc": "^2.0.0-rc.48",
+    "url-search-params-polyfill": "^8.1.0"
+  },
+  "resolutions": {
+    "lodash": "^4.17.21"
+  }
+}