Merge branch 'apache:master' into flight-jdbc-driver

Author: jduo

.dockerignore

@@ -28,6 +28,7 @@
 !ci/**
 !c_glib/Gemfile
 !dev/archery/setup.py
+!dev/release/setup-*.sh
 !docs/requirements*.txt
 !python/requirements*.txt
 !python/manylinux1/**

.env

@@ -61,7 +61,8 @@ GO=1.16
 HDFS=3.2.1
 JDK=8
 KARTOTHEK=latest
-LLVM=12
+# LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
+LLVM=13
 MAVEN=3.5.4
 NODE=16
 NUMPY=latest

.github/workflows/cpp.yml

@@ -126,6 +126,7 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
@@ -141,6 +142,8 @@ jobs:
       ARROW_WITH_SNAPPY: ON
       ARROW_WITH_ZLIB: ON
       ARROW_WITH_ZSTD: ON
+      # System Abseil installed by Homebrew uses C++ 17
+      CMAKE_CXX_STANDARD: 17
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -153,6 +156,9 @@ jobs:
           rm -f /usr/local/bin/2to3
           brew update --preinstall
           brew bundle --file=cpp/Brewfile
+      - name: Install Google Cloud Storage Testbench
+        shell: bash
+        run: ci/scripts/install_gcs_testbench.sh default
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -268,6 +274,9 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      # google-could-cpp uses _dupenv_s() but it can't be used with msvcrt.
+      # We need to use ucrt to use _dupenv_s().
+      # ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
       ARROW_JEMALLOC: OFF

.github/workflows/go.yml

@@ -152,7 +152,7 @@ jobs:
           fetch-depth: 0
           submodules: recursive
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+        run: go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -180,7 +180,7 @@ jobs:
           fetch-depth: 0
           submodules: recursive
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+        run: go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -213,7 +213,7 @@ jobs:
         shell: bash
         run: brew install apache-arrow
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+        run: go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -268,7 +268,7 @@ jobs:
         with:
           go-version: '1.17'
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+        run: go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)

.github/workflows/r.yml

@@ -236,7 +236,7 @@ jobs:
     name: AMD64 Windows R ${{ matrix.config.rversion }} RTools ${{ matrix.config.rtools }}
     runs-on: windows-2019
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 60
+    timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
@@ -329,12 +329,37 @@ jobs:
         if: ${{ matrix.config.rtools == 35 }}
         shell: Rscript {0}
         run: install.packages("cpp11", type = "source")
+      - name: Prune dependencies (on R 3.6)
+        if: ${{ matrix.config.rtools == 35 }}
+        shell: Rscript {0}
+        run: |
+            # To prevent the build from timing out, let's prune some optional deps (and their possible version requirements)
+            setwd("r")
+            # create a backup to use later
+            file.copy("DESCRIPTION", "DESCRIPTION.bak")
+            d <- read.dcf("DESCRIPTION")
+            to_prune <- c("duckdb", "DBI", "dbplyr", "decor", "knitr", "rmarkdown", "pkgload", "reticulate")
+            pattern <- paste0("\\n?", to_prune, "( \\([^,]*\\))?,?", collapse = "|")
+            d[,"Suggests"] <- gsub(pattern, "", d[,"Suggests"])
+            write.dcf(d, "DESCRIPTION")
+      - name: R package cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ env.R_LIBS_USER }}/*
+          key: r-${{ matrix.config.rtools }}-R-LIBS-${{ hashFiles('r/DESCRIPTION') }}
+          restore-keys: r-${{ matrix.config.rtools }}-R-LIBS-
       - name: Install R package dependencies
         shell: Rscript {0}
         run: |
           # options(pkgType="win.binary") # processx doesn't have a binary for UCRT yet
           install.packages(c("remotes", "rcmdcheck"))
           remotes::install_deps("r", dependencies = TRUE)
+      - name: Restore DESCRIPTION for 3.6
+        if: ${{ matrix.config.rtools == 35 }}
+        run: |
+          rm r/DESCRIPTION
+          mv r/DESCRIPTION.bak r/DESCRIPTION
       - name: Check
         shell: Rscript {0}
         run: |
@@ -357,6 +382,15 @@ jobs:
             check_dir = 'check',
             timeout = 3600
           )
+      - name: Run lintr
+        env: 
+          NOT_CRAN: "true"
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: Rscript {0}
+        working-directory: r
+        run: |
+          remotes::install_github("jonkeane/lintr@arrow-branch")
+          lintr::expect_lint_free()
       - name: Dump install logs
         shell: cmd
         run: cat r/check/arrow.Rcheck/00install.out

.github/workflows/ruby.yml

@@ -90,6 +90,7 @@ jobs:
           ulimit -c unlimited
           archery docker run \
             -e ARROW_FLIGHT=ON \
+            -e ARROW_GCS=ON \
             -e Protobuf_SOURCE=BUNDLED \
             -e gRPC_SOURCE=BUNDLED \
             ubuntu-ruby
@@ -110,6 +111,7 @@ jobs:
       ARROW_BUILD_TESTS: OFF
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_GLIB_GTK_DOC: true
       ARROW_GLIB_WERROR: true
       ARROW_HOME: /usr/local
@@ -188,6 +190,7 @@ jobs:
       ARROW_BUILD_TYPE: release
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}
       ARROW_JEMALLOC: OFF

c_glib/arrow-flight-glib/client.cpp

@@ -251,12 +251,11 @@ gaflight_client_new(GAFlightLocation *location,
   arrow::Status status;
   if (options) {
     const auto flight_options = gaflight_client_options_get_raw(options);
-    status = arrow::flight::FlightClient::Connect(*flight_location,
-                                                  *flight_options,
-                                                  &flight_client);
+    auto result = arrow::flight::FlightClient::Connect(*flight_location, *flight_options);
+    status = std::move(result).Value(&flight_client);
   } else {
-    status = arrow::flight::FlightClient::Connect(*flight_location,
-                                                  &flight_client);
+    auto result = arrow::flight::FlightClient::Connect(*flight_location);
+    status = std::move(result).Value(&flight_client);
   }
   if (garrow::check(error, status, "[flight-client][new]")) {
     return gaflight_client_new_raw(flight_client.release());
@@ -315,9 +314,8 @@ gaflight_client_list_flights(GAFlightClient *client,
     flight_options = gaflight_call_options_get_raw(options);
   }
   std::unique_ptr<arrow::flight::FlightListing> flight_listing;
-  auto status = flight_client->ListFlights(*flight_options,
-                                           *flight_criteria,
-                                           &flight_listing);
+  auto result = flight_client->ListFlights(*flight_options, *flight_criteria);
+  auto status = std::move(result).Value(&flight_listing);
   if (!garrow::check(error,
                      status,
                      "[flight-client][list-flights]")) {
@@ -326,7 +324,7 @@ gaflight_client_list_flights(GAFlightClient *client,
   GList *listing = NULL;
   std::unique_ptr<arrow::flight::FlightInfo> flight_info;
   while (true) {
-    status = flight_listing->Next(&flight_info);
+    status = flight_listing->Next().Value(&flight_info);
     if (!garrow::check(error,
                        status,
                        "[flight-client][list-flights]")) {
@@ -369,9 +367,8 @@ gaflight_client_do_get(GAFlightClient *client,
     flight_options = gaflight_call_options_get_raw(options);
   }
   std::unique_ptr<arrow::flight::FlightStreamReader> flight_reader;
-  auto status = flight_client->DoGet(*flight_options,
-                                     *flight_ticket,
-                                     &flight_reader);
+  auto result = flight_client->DoGet(*flight_options, *flight_ticket);
+  auto status = std::move(result).Value(&flight_reader);
   if (garrow::check(error,
                     status,
                     "[flight-client][do-get]")) {

c_glib/arrow-flight-glib/common.cpp

@@ -245,7 +245,7 @@ gaflight_location_new(const gchar *uri,
   auto location = GAFLIGHT_LOCATION(g_object_new(GAFLIGHT_TYPE_LOCATION, NULL));
   auto flight_location = gaflight_location_get_raw(location);
   if (garrow::check(error,
-                    arrow::flight::Location::Parse(uri, flight_location),
+                    arrow::flight::Location::Parse(uri).Value(flight_location),
                     "[flight-location][new]")) {
     return location;
   } else {
@@ -1018,10 +1018,10 @@ gaflight_info_get_schema(GAFlightInfo *info,
   std::shared_ptr<arrow::Schema> arrow_schema;
   if (options) {
     auto arrow_memo = garrow_read_options_get_dictionary_memo_raw(options);
-    status = flight_info->GetSchema(arrow_memo, &arrow_schema);
+    status = flight_info->GetSchema(arrow_memo).Value(&arrow_schema);
   } else {
     arrow::ipc::DictionaryMemo arrow_memo;
-    status = flight_info->GetSchema(&arrow_memo, &arrow_schema);
+    status = flight_info->GetSchema(&arrow_memo).Value(&arrow_schema);
   }
   if (garrow::check(error, status, "[flight-info][get-schema]")) {
     return garrow_schema_new_raw(&arrow_schema);
@@ -1287,7 +1287,7 @@ gaflight_record_batch_reader_read_next(GAFlightRecordBatchReader *reader,
 {
   auto flight_reader = gaflight_record_batch_reader_get_raw(reader);
   arrow::flight::FlightStreamChunk flight_chunk;
-  auto status = flight_reader->Next(&flight_chunk);
+  auto status = flight_reader->Next().Value(&flight_chunk);
   if (garrow::check(error, status, "[flight-record-batch-reader][read-next]")) {
     if (flight_chunk.data) {
       return gaflight_stream_chunk_new_raw(&flight_chunk);
@@ -1314,7 +1314,7 @@ gaflight_record_batch_reader_read_all(GAFlightRecordBatchReader *reader,
 {
   auto flight_reader = gaflight_record_batch_reader_get_raw(reader);
   std::shared_ptr<arrow::Table> arrow_table;
-  auto status = flight_reader->ReadAll(&arrow_table);
+  auto status = flight_reader->ToTable().Value(&arrow_table);
   if (garrow::check(error, status, "[flight-record-batch-reader][read-all]")) {
     return garrow_table_new_raw(&arrow_table);
   } else {

c_glib/arrow-flight-glib/server.cpp

@@ -450,15 +450,14 @@ namespace gaflight {
       return stream->schema();
     }
 
-    arrow::Status GetSchemaPayload(
-      arrow::flight::FlightPayload *payload) override {
+    arrow::Result<arrow::flight::FlightPayload> GetSchemaPayload() override {
       auto stream = gaflight_data_stream_get_raw(gastream_);
-      return stream->GetSchemaPayload(payload);
+      return stream->GetSchemaPayload();
     }
 
-    arrow::Status Next(arrow::flight::FlightPayload *payload) override {
+    arrow::Result<arrow::flight::FlightPayload> Next() override {
       auto stream = gaflight_data_stream_get_raw(gastream_);
-      return stream->Next(payload);
+      return stream->Next();
     }
 
   private:

ci/conda_env_python.txt

@@ -26,7 +26,6 @@ numpy>=1.16.6
 pytest
 pytest-faulthandler
 pytest-lazy-fixture
-pytz
 s3fs>=2021.8.0
 setuptools
 setuptools_scm

ci/docker/debian-10-go.dockerfile

@@ -19,7 +19,7 @@ ARG arch=amd64
 ARG go=1.15
 FROM ${arch}/golang:${go}-buster
 
-RUN GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck@latest
+RUN GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
 
 # TODO(kszucs):
 # 1. add the files required to install the dependencies to .dockerignore

ci/docker/debian-11-go.dockerfile

@@ -19,7 +19,7 @@ ARG arch=amd64
 ARG go=1.16
 FROM ${arch}/golang:${go}-bullseye
 
-RUN GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck@latest
+RUN GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
 
 # TODO(kszucs):
 # 1. add the files required to install the dependencies to .dockerignore

ci/docker/ubuntu-18.04-verify-rc.dockerfile

@@ -19,42 +19,8 @@ ARG arch=amd64
 FROM ${arch}/ubuntu:18.04
 
 ENV DEBIAN_FRONTEND=noninteractive
-
-ARG llvm=12
-RUN apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-       apt-transport-https \
-       ca-certificates \
-       gnupg \
-       wget && \
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \
-       /etc/apt/sources.list.d/llvm.list && \
-    apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-        build-essential \
-        clang \
-        cmake \
-        curl \
-        git \
-        libcurl4-openssl-dev \
-        libgirepository1.0-dev \
-        libglib2.0-dev \
-        libsqlite3-dev \
-        libssl-dev \
-        llvm-${llvm}-dev \
-        maven \
-        ninja-build \
-        openjdk-11-jdk \
-        pkg-config \
-        python3-pip \
-        python3.8-dev \
-        python3.8-venv \
-        ruby-dev \
-        wget \
-        tzdata && \
+COPY dev/release/setup-ubuntu.sh /
+RUN /setup-ubuntu.sh && \
+    rm /setup-ubuntu.sh && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
-
-RUN python3.8 -m pip install -U pip && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1

ci/docker/ubuntu-20.04-verify-rc.dockerfile

@@ -19,28 +19,8 @@ ARG arch=amd64
 FROM ${arch}/ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-        build-essential \
-        clang \
-        cmake \
-        curl \
-        git \
-        libcurl4-openssl-dev \
-        libgirepository1.0-dev \
-        libglib2.0-dev \
-        libsqlite3-dev \
-        libssl-dev \
-        llvm-dev \
-        maven \
-        ninja-build \
-        nlohmann-json3-dev \
-        openjdk-11-jdk \
-        pkg-config \
-        python3-dev \
-        python3-pip \
-        python3-venv \
-        ruby-dev \
-        wget && \
+COPY dev/release/setup-ubuntu.sh /
+RUN /setup-ubuntu.sh && \
+    rm /setup-ubuntu.sh && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*