Merge pull request #4 from apache/master

Arrow Master 1/29/2019

Author: mikepigott

.gitignore

@@ -30,6 +30,8 @@ arrow-src.tar.gz
 .build_cache_dir
 dependency-reduced-pom.xml
 MANIFEST
+compile_commands.json
+build.ninja
 
 # Generated Visual Studio files
 *.vcxproj

.travis.yml

@@ -16,7 +16,7 @@
 # under the License.
 
 sudo: required
-dist: trusty
+dist: xenial
 
 services:
   - docker
@@ -38,16 +38,13 @@ before_install:
   - |
       if [ $TRAVIS_OS_NAME == "linux" ]; then
         sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
-        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
         sudo apt-get update -qq
       fi
   - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
 
 
 matrix:
   fast_finish: true
-  allow_failures:
-  - jdk: oraclejdk9
   include:
   - name: "Lint C++, Python, R"
     os: linux
@@ -61,20 +58,44 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
-  - name: "C++ unit tests, code coverage with gcc 4.8"
+  - name: "C++ unit tests w/ Valgrind, clang 6.0"
+    language: cpp
+    os: linux
+    env:
+    - ARROW_TRAVIS_VALGRIND=1
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_PLASMA=1
+    - ARROW_TRAVIS_ORC=1
+    - ARROW_TRAVIS_PARQUET=1
+    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    before_script:
+    - if [ $ARROW_CI_CPP_AFFECTED != "1" ]; then exit; fi
+    - export CC="clang-6.0"
+    - export CXX="clang++-6.0"
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    # If either C++ or Python changed, we must install the C++ libraries
+    - git submodule update --init
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
+  # Separating Valgrind and C++ coverage makes individual jobs shorter
+  - name: "C++ unit tests w/ gcc 5.4, coverage"
     compiler: gcc
     language: cpp
     os: linux
     jdk: openjdk8
     env:
-    - ARROW_TRAVIS_VALGRIND=1
     - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_ORC=1
     - ARROW_TRAVIS_COVERAGE=1
     - ARROW_TRAVIS_PARQUET=1
     - ARROW_TRAVIS_GANDIVA=1
     - ARROW_TRAVIS_GANDIVA_JAVA=1
+    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     before_script:
     - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
@@ -87,7 +108,40 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
-  - name: "Python 2.7 and 3.6 unit tests, coverage with gcc 4.8"
+  - name: "C++ unit tests w/ gcc 4.8, trusty"
+    dist: trusty
+    compiler: gcc
+    language: cpp
+    os: linux
+    jdk: openjdk8
+    env:
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_PLASMA=1
+    - ARROW_TRAVIS_ORC=1
+    - ARROW_TRAVIS_PARQUET=1
+    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_GANDIVA_JAVA=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    before_install:
+    - ulimit -c unlimited -S
+    - |
+        if [ $TRAVIS_OS_NAME == "linux" ]; then
+          sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
+          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+          sudo apt-get update -qq
+        fi
+    - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
+    before_script:
+    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    # If either C++ or Python changed, we must install the C++ libraries
+    - git submodule update --init
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
+    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
+  - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage"
     compiler: gcc
     language: cpp
     os: linux
@@ -98,10 +152,10 @@ matrix:
     - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_TRAVIS_COVERAGE=1
     - ARROW_TRAVIS_PYTHON_DOCS=1
-    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - ARROW_TRAVIS_PYTHON_JVM=1
-    - ARROW_TRAVIS_PYTHON_GANDIVA=1
     - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
     # TODO(wesm): Run the benchmarks outside of Travis
     # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
     before_script:
@@ -111,6 +165,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1
+    - ARROW_TRAVIS_PYTHON_GANDIVA=1
     # Only run Plasma tests with valgrind in one of the Python builds because
     # they are slow
     - export PLASMA_VALGRIND=0
@@ -148,17 +203,18 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh
-  - name: "[OS X] Python w/ XCode 6.4"
+  - name: "[OS X] Python w/ XCode 7.3"
     compiler: clang
     language: cpp
-    osx_image: xcode6.4
+    osx_image: xcode7.3
     os: osx
     cache:
     addons:
     env:
     - ARROW_TRAVIS_USE_TOOLCHAIN=1
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    - MACOSX_DEPLOYMENT_TARGET="10.9"
     before_script:
     script:
     - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
@@ -181,18 +237,22 @@ matrix:
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
     - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
-  - name: "Java w/ Oracle JDK 9"
+  - name: "Java w/ OpenJDK 9"
     language: java
     os: linux
-    jdk: oraclejdk9
+    jdk: openjdk9
+    before_script:
+    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+  - name: "Java w/ OpenJDK 11"
+    language: java
+    os: linux
+    jdk: openjdk11
     before_script:
     - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
-    addons:
-      apt:
-        packages:
-          - oracle-java9-installer
   - name: "Integration w/ OpenJDK 8"
     language: java
     os: linux
@@ -201,13 +261,13 @@ matrix:
     env:
     - ARROW_TRAVIS_PLASMA=1
     - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1
-    - CC="clang-6.0"
-    - CXX="clang++-6.0"
     before_script:
     - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
+    - export CC="clang-6.0"
+    - export CXX="clang++-6.0"
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - nvm install 10.1
+    - nvm install 11.6
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
@@ -217,14 +277,14 @@ matrix:
     language: node_js
     os: linux
     node_js:
-    - '10.1'
+    - '11.6'
     before_script:
     - if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
-  - name: "C++ & GLib & Ruby w/ gcc 4.9"
+  - name: "C++ & GLib & Ruby w/ gcc 5.4"
     compiler: gcc
     language: cpp
     os: linux
@@ -234,8 +294,6 @@ matrix:
     - ARROW_TRAVIS_USE_VENDORED_BOOST=1
     - ARROW_TRAVIS_PARQUET=1
     - ARROW_TRAVIS_PLASMA=1
-    - BUILD_TORCH_EXAMPLE=no
-    - MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9"
     before_script:
     - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
@@ -286,7 +344,7 @@ matrix:
     - pushd ${TRAVIS_BUILD_DIR}/rust
     # Run coverage for codecov.io
     - mkdir -p target/kcov
-    - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=stable cargo coverage --verbose
+    - RUST_BACKTRACE=1 RUSTUP_TOOLCHAIN=nightly cargo coverage --verbose
     - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
   - name: Go
     language: go
@@ -305,6 +363,9 @@ matrix:
     language: r
     cache: packages
     latex: false
+    dist: trusty
+    env:
+    - ARROW_TRAVIS_PARQUET=1
     before_install:
     # Have to copy-paste this here because of how R's build steps work
     - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
@@ -317,7 +378,6 @@ matrix:
         fi
     - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
-    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib
     - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig
     - pushd ${TRAVIS_BUILD_DIR}/r