arrow-cpp: 0.10.0 -> 0.11.0 (#48066)

* arrow-cpp: 0.10.0 -> 0.11.0

* arrow-cpp: enable testing

pkgs/development/libraries/arrow-cpp/default.nix

@@ -1,42 +1,75 @@
-{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, python, rapidjson, snappy, zlib, zstd }:
+{ stdenv, symlinkJoin, fetchurl, fetchFromGitHub, boost, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl, python, rapidjson, snappy, thrift, which, zlib, zstd }:
+
+let
+  parquet-testing = fetchFromGitHub {
+    owner = "apache";
+    repo = "parquet-testing";
+    rev = "46ae2605c2de306f5740587107dcf333a527f2d1";
+    sha256 = "07ps745gas2zcfmg56m3vwl63yyzmalnxwb5dc40vd004cx5hdik";
+  };
+in
 
 stdenv.mkDerivation rec {
   name = "arrow-cpp-${version}";
-  version = "0.10.0";
+  version = "0.11.0";
 
   src = fetchurl {
     url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
-    sha256 = "0bc4krapz1kzdm16npzmgdz7zvg9lip6rnqbwph8vfn7zji0fcll";
+    sha256 = "0pc5pqr0dbnx8s1ji102dhw9bbrsq3ml4ac3mmi2022yfyizlf0q";
   };
 
   sourceRoot = "apache-arrow-${version}/cpp";
 
-  # patch to fix python-test
-  patches = [ ./darwin.patch ];
+  patches = [
+    # fix ARROW-3467
+    ./double-conversion_cmake.patch
+
+    # patch to fix python-test
+    ./darwin.patch
+    ];
 
   nativeBuildInputs = [ cmake ];
-  buildInputs = [ boost python.pkgs.python python.pkgs.numpy ];
+  buildInputs = [ boost double-conversion glog python.pkgs.python python.pkgs.numpy ];
 
   preConfigure = ''
+    substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
     substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
+    substituteInPlace cmake_modules/FindGLOG.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
     substituteInPlace cmake_modules/FindLz4.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
     substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
+
+    patchShebangs build-support/
   '';
 
   BROTLI_HOME = symlinkJoin { name="brotli-wrap"; paths = [ brotli.lib brotli.dev ]; };
+  DOUBLE_CONVERSION_HOME = double-conversion;
   FLATBUFFERS_HOME = flatbuffers;
-  GTEST_HOME = gtest;
   GFLAGS_HOME = gflags;
+  GLOG_HOME = glog;
+  GTEST_HOME = gtest;
   LZ4_HOME = symlinkJoin { name="lz4-wrap"; paths = [ lz4 lz4.dev ]; };
   RAPIDJSON_HOME = rapidjson;
   SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; };
-  ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; };
+  THRIFT_HOME = thrift;
+  ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib zlib.dev ]; };
   ZSTD_HOME = zstd;
 
   cmakeFlags = [
     "-DARROW_PYTHON=ON"
+    "-DARROW_PARQUET=ON"
   ];
 
+  doInstallCheck = true;
+  PARQUET_TEST_DATA = if doInstallCheck then "${parquet-testing}/data" else null;
+  installCheckInputs = [ perl which ];
+  installCheckPhase = (stdenv.lib.optionalString stdenv.isDarwin ''
+    for f in release/*-test; do
+      install_name_tool -add_rpath "$out"/lib  "$f"
+    done
+  '') + ''
+    ctest -L unittest -V
+  '';
+
   meta = {
     description = "A  cross-language development platform for in-memory data";
     homepage = https://arrow.apache.org/;

pkgs/development/libraries/arrow-cpp/double-conversion_cmake.patch

@@ -0,0 +1,43 @@
+diff --git a/CMakeLists.txt b/cpp/CMakeLists.txt
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -615,7 +615,7 @@ endif(UNIX)
+ set(ARROW_LINK_LIBS)
+ 
+ # Libraries to link statically with libarrow.so
+-set(ARROW_STATIC_LINK_LIBS double-conversion)
++set(ARROW_STATIC_LINK_LIBS ${DOUBLE_CONVERSION_TARGET})
+ 
+ if (ARROW_WITH_BROTLI)
+   SET(ARROW_STATIC_LINK_LIBS
+@@ -694,7 +694,7 @@ else ()
+   set(ARROW_MIN_TEST_LIBS
+     arrow_shared
+     ${ARROW_LINK_LIBS}
+-    double-conversion
++    ${DOUBLE_CONVERSION_TARGET}
+     ${BOOST_SYSTEM_LIBRARY}
+     ${BOOST_FILESYSTEM_LIBRARY}
+     ${BOOST_REGEX_LIBRARY}
+diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+--- a/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cmake_modules/ThirdpartyToolchain.cmake
+@@ -469,14 +469,16 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "")
+   set(DOUBLE_CONVERSION_VENDORED 1)
+ else()
+   find_package(double-conversion REQUIRED)
++  set(DOUBLE_CONVERSION_TARGET double-conversion::double-conversion)
+   set(DOUBLE_CONVERSION_VENDORED 0)
+ endif()
+ 
+ include_directories(SYSTEM ${DOUBLE_CONVERSION_INCLUDE_DIR})
+-ADD_THIRDPARTY_LIB(double-conversion
+-  STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
+ 
+ if (DOUBLE_CONVERSION_VENDORED)
++  ADD_THIRDPARTY_LIB(double-conversion
++    STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB})
++  set(DOUBLE_CONVERSION_TARGET double-conversion)
+   add_dependencies(arrow_dependencies double-conversion_ep)
+ endif()
+ 

pkgs/development/libraries/parquet-cpp/default.nix

@@ -1,39 +0,0 @@
-{ stdenv, symlinkJoin, fetchurl, arrow-cpp, boost, cmake, gtest, snappy, thrift, zlib }:
-
-stdenv.mkDerivation rec {
-  name = "parquet-cpp-${version}";
-  version = "1.5.0";
-
-  src = fetchurl {
-    url = "https://github.com/apache/parquet-cpp/archive/apache-${name}.tar.gz";
-    sha256 = "19nwqahc0igr0jfprbf2m86rmzz6zicw4z7b8z832wbsyc904wli";
-  };
-
-  patches = [ ./api.patch ];
-
-  nativeBuildInputs = [ cmake ];
-  buildInputs = [ boost ];
-
-  preConfigure = ''
-    substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
-    substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
-  '';
-
-  ARROW_HOME = arrow-cpp;
-  THRIFT_HOME = thrift;
-  GTEST_HOME = gtest;
-  SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; };
-  ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; };
-
-  cmakeFlags = [
-    "-DPARQUET_BUILD_BENCHMARKS=OFF"
-  ];
-
-  meta = {
-    description = "A C++ library to read and write the Apache Parquet columnar data format";
-    homepage = http://parquet.apache.org;
-    license = stdenv.lib.licenses.asl20;
-    platforms = stdenv.lib.platforms.unix;
-    maintainers = with stdenv.lib.maintainers; [ veprbl ];
-  };
-}

pkgs/development/python-modules/pyarrow/default.nix

@@ -1,8 +1,7 @@
-{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, JPype1, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }:
+{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, JPype1, numpy, pandas, pytest, pytestrunner, pkgconfig, setuptools_scm, six }:
 
 let
-  _arrow-cpp = arrow-cpp.override { inherit python;};
-  _parquet-cpp = parquet-cpp.override { arrow-cpp = _arrow-cpp; };
+  _arrow-cpp = arrow-cpp.override { inherit python; };
 in
 
 buildPythonPackage rec {
@@ -17,18 +16,11 @@ buildPythonPackage rec {
   checkInputs = [ pandas pytest pytestrunner JPype1 ];
 
   PYARROW_BUILD_TYPE = "release";
-  PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib";
-
-  preBuild = ''
-    substituteInPlace CMakeLists.txt --replace "\''${ARROW_ABI_VERSION}" '"0.0.0"'
-    substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"'
-
-    # fix the hardcoded value
-    substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${_parquet-cpp.version}")'
-  '';
+  PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib";
 
   preCheck = ''
     rm pyarrow/tests/test_hdfs.py
+    rm pyarrow/tests/test_cuda.py
 
     # fails: "ArrowNotImplementedError: Unsupported numpy type 22"
     substituteInPlace pyarrow/tests/test_feather.py --replace "test_timedelta_with_nulls" "_disabled"
@@ -47,7 +39,7 @@ buildPythonPackage rec {
   '';
 
   ARROW_HOME = _arrow-cpp;
-  PARQUET_HOME = _parquet-cpp;
+  PARQUET_HOME = _arrow-cpp;
 
   setupPyBuildFlags = ["--with-parquet" ];
 

pkgs/top-level/aliases.nix

@@ -207,6 +207,7 @@ mapAliases ({
   openssh_with_kerberos = openssh; # added 2018-01-28
   owncloudclient = owncloud-client;  # added 2016-08
   p11_kit = p11-kit; # added 2018-02-25
+  parquet-cpp = arrow-cpp; # added 2018-09-08
   pass-otp = pass.withExtensions (ext: [ext.pass-otp]); # added 2018-05-04
   pgp-tools = signing-party; # added 2017-03-26
   pidgin-with-plugins = pidgin; # added 2016-06

pkgs/top-level/all-packages.nix

@@ -11600,8 +11600,6 @@ with pkgs;
 
   paperkey = callPackage ../tools/security/paperkey { };
 
-  parquet-cpp = callPackage ../development/libraries/parquet-cpp {};
-
   pangoxsl = callPackage ../development/libraries/pangoxsl { };
 
   pcaudiolib = callPackage ../development/libraries/pcaudiolib {