Merge branch 'main' into module-cleanup

delta-io · Sep 23, 2023 · 22b500d · 22b500d
2 parents c9bd37c + 72f8531
commit 22b500d
Show file tree

Hide file tree

Showing 158 changed files with 10,063 additions and 3,085 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -67,8 +67,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       # Disable full debug symbol generation to speed up CI build and keep memory down
-      # "1" means line tables only, which is useful for panic tracebacks.
-      RUSTFLAGS: -C debuginfo=1
+      RUSTFLAGS: -C debuginfo=line-tables-only
+      # Disable incremental builds by cargo for CI which should save disk space
+      # and hopefully avoid final link "No space left on device"
+      CARGO_INCREMENTAL: 0
 
     steps:
       - uses: actions/checkout@v3
@@ -89,9 +91,10 @@ jobs:
     name: Integration Tests
     runs-on: ubuntu-latest
     env:
+      CARGO_INCREMENTAL: 0
       # Disable full debug symbol generation to speed up CI build and keep memory down
-      # "1" means line tables only, which is useful for panic tracebacks.
-      RUSTFLAGS: "-C debuginfo=1"
+      # <https://doc.rust-lang.org/cargo/reference/profiles.html>
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
       RUST_BACKTRACE: "1"
@@ -145,7 +148,8 @@ jobs:
   parquet2_test:
     runs-on: ubuntu-latest
     env:
-      RUSTFLAGS: "-C debuginfo=0"
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      CARGO_INCREMENTAL: 0
 
     steps:
       - uses: actions/checkout@v3

diff --git a/.github/workflows/dev_pr/labeler.yml b/.github/workflows/dev_pr/labeler.yml
@@ -30,4 +30,3 @@ proofs:
 
 tlaplus:
   - tlaplus/**/*
-
diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml
@@ -39,7 +39,8 @@ jobs:
     name: Python Build (Python 3.7 PyArrow 8.0.0)
     runs-on: ubuntu-latest
     env:
-      RUSTFLAGS: "-C debuginfo=0"
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      CARGO_INCREMENTAL: 0
 
     # use the same environment we have for python release
     container: quay.io/pypa/manylinux2014_x86_64:2022-09-24-4f086d0
@@ -65,14 +66,16 @@ jobs:
 
       - name: Build and install deltalake
         run: |
+          # Needed for openssl build
+          yum install -y perl-IPC-Cmd
           pip install virtualenv
           virtualenv venv
           source venv/bin/activate
           make setup
           # Install minimum PyArrow version
           pip install -e .[pandas,devel] pyarrow==8.0.0
         env:
-          RUSTFLAGS: "-C debuginfo=0"
+          RUSTFLAGS: "-C debuginfo=line-tables-only"
 
       - name: Run tests
         run: |
@@ -88,6 +91,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       RUSTFLAGS: "-C debuginfo=1"
+      CARGO_INCREMENTAL: 0
 
     steps:
       - uses: actions/checkout@v3
@@ -121,13 +125,13 @@ jobs:
       - name: Run tests
         run: |
           source venv/bin/activate
-          python -m pytest -m '((s3 or azure) and integration) or not integration'
+          python -m pytest -m '((s3 or azure) and integration) or not integration and not benchmark'
 
       - name: Test without pandas
         run: |
           source venv/bin/activate
           pip uninstall --yes pandas
-          python -m pytest -m "not pandas and not integration"
+          python -m pytest -m "not pandas and not integration and not benchmark"
           pip install pandas
 
       - name: Build Sphinx documentation
@@ -139,7 +143,8 @@ jobs:
     name: Python Benchmark
     runs-on: ubuntu-latest
     env:
-      RUSTFLAGS: "-C debuginfo=0"
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      CARGO_INCREMENTAL: 0
 
     steps:
       - uses: actions/checkout@v2
@@ -188,7 +193,8 @@ jobs:
     name: PySpark Integration Tests
     runs-on: ubuntu-latest
     env:
-      RUSTFLAGS: "-C debuginfo=0"
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      CARGO_INCREMENTAL: 0
 
     steps:
       - uses: actions/checkout@v3
@@ -227,7 +233,8 @@ jobs:
     name: Running with Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
     env:
-      RUSTFLAGS: "-C debuginfo=0"
+      RUSTFLAGS: "-C debuginfo=line-tables-only"
+      CARGO_INCREMENTAL: 0
 
     strategy:
       matrix:

diff --git a/.github/workflows/python_release.yml b/.github/workflows/python_release.yml
@@ -39,6 +39,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      # We use extra recent Cargo.toml syntax, so we need at least Rust 1.71.0
+      - name: Install newer rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: stable
+          override: true
+
       - name: Publish to pypi (without sdist)
         uses: messense/maturin-action@v1
         env:
@@ -95,6 +103,8 @@ jobs:
           target: x86_64-unknown-linux-gnu
           command: publish
           args: --skip-existing -m python/Cargo.toml ${{ env.FEATURES_FLAG }}
+          # for openssl build
+          before-script-linux: yum install -y perl-IPC-Cmd
 
       - name: Publish manylinux to pypi aarch64 (without sdist)
         uses: messense/maturin-action@v1
@@ -104,6 +114,8 @@ jobs:
           target: aarch64-unknown-linux-gnu
           command: publish
           args: --skip-existing -m python/Cargo.toml --no-sdist ${{ env.FEATURES_FLAG }}
+          # for openssl build
+          before-script-linux: yum install -y perl-IPC-Cmd
 
   release-docs:
     needs:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,68 @@
 # Changelog
 
+## [rust-v0.15.0](https://github.com/delta-io/delta-rs/tree/rust-v0.15.0) (2023-09-06)
+
+[Full Changelog](https://github.com/delta-io/delta-rs/compare/rust-v0.14.0...rust-v0.15.0)
+
+**Implemented enhancements:**
+
+- Configurable number of retries for transaction commit loop [\#1595](https://github.com/delta-io/delta-rs/issues/1595)
+
+**Fixed bugs:**
+
+- Unable to read table using VM Managed Identity on Azure [\#1462](https://github.com/delta-io/delta-rs/issues/1462)
+- Unable to query by partition column  [\#1445](https://github.com/delta-io/delta-rs/issues/1445)
+
+**Merged pull requests:**
+
+- fix: update python test [\#1608](https://github.com/delta-io/delta-rs/pull/1608) ([wjones127](https://github.com/wjones127))
+- chore: update datafusion to 30, arrow to 45 [\#1606](https://github.com/delta-io/delta-rs/pull/1606) ([scsmithr](https://github.com/scsmithr))
+- fix: just make pyarrow 12 the max [\#1603](https://github.com/delta-io/delta-rs/pull/1603) ([wjones127](https://github.com/wjones127))
+- fix: support partial statistics in JSON [\#1599](https://github.com/delta-io/delta-rs/pull/1599) ([CurtHagenlocher](https://github.com/CurtHagenlocher))
+- feat: allow configurable number of `commit` attempts [\#1596](https://github.com/delta-io/delta-rs/pull/1596) ([cmackenzie1](https://github.com/cmackenzie1))
+- fix: querying on date partitions \(fixes \#1445\) [\#1594](https://github.com/delta-io/delta-rs/pull/1594) ([watfordkcf](https://github.com/watfordkcf))
+- refactor: clean up arrow schema defs [\#1590](https://github.com/delta-io/delta-rs/pull/1590) ([polynomialherder](https://github.com/polynomialherder))
+- feat: add metadata for operations::write::WriteBuilder [\#1584](https://github.com/delta-io/delta-rs/pull/1584) ([abhimanyusinghgaur](https://github.com/abhimanyusinghgaur))
+- feat: add metadata for deletion vectors [\#1583](https://github.com/delta-io/delta-rs/pull/1583) ([aersam](https://github.com/aersam))
+- fix: remove alpha classifier [\#1578](https://github.com/delta-io/delta-rs/pull/1578) ([marcelotrevisani](https://github.com/marcelotrevisani))
+- refactor: use pa.table.cast in delta\_arrow\_schema\_from\_pandas [\#1573](https://github.com/delta-io/delta-rs/pull/1573) ([ion-elgreco](https://github.com/ion-elgreco))
+
+## [rust-v0.14.0](https://github.com/delta-io/delta-rs/tree/rust-v0.14.0) (2023-08-01)
+
+[Full Changelog](https://github.com/delta-io/delta-rs/compare/rust-v0.13.0...rust-v0.14.0)
+
+**Implemented enhancements:**
+
+- Define common dependencies in Cargo Workspace [\#1572](https://github.com/delta-io/delta-rs/issues/1572)
+- Make `delta_datafusion::find_files` public [\#1559](https://github.com/delta-io/delta-rs/issues/1559)
+
+**Fixed bugs:**
+
+- Excessive integration test sizes causing builds to fail [\#1550](https://github.com/delta-io/delta-rs/issues/1550)
+- Slack invite link is not working [\#1530](https://github.com/delta-io/delta-rs/issues/1530)
+
+**Merged pull requests:**
+
+- fix: correct whitespace in delta protocol reader minimum version error message [\#1576](https://github.com/delta-io/delta-rs/pull/1576) ([polynomialherder](https://github.com/polynomialherder))
+- chore: move deps to `[workspace.dependencies]` [\#1575](https://github.com/delta-io/delta-rs/pull/1575) ([cmackenzie1](https://github.com/cmackenzie1))
+- chore: update `datafusion` to `28` and arrow to `43` [\#1571](https://github.com/delta-io/delta-rs/pull/1571) ([cmackenzie1](https://github.com/cmackenzie1))
+- ci: don't run benchmark in debug mode [\#1566](https://github.com/delta-io/delta-rs/pull/1566) ([wjones127](https://github.com/wjones127))
+- ci: install newer rust for macos python release [\#1565](https://github.com/delta-io/delta-rs/pull/1565) ([wjones127](https://github.com/wjones127))
+- feat: make find\_files public [\#1560](https://github.com/delta-io/delta-rs/pull/1560) ([yjshen](https://github.com/yjshen))
+- feat!: bulk delete for vacuum [\#1556](https://github.com/delta-io/delta-rs/pull/1556) ([Blajda](https://github.com/Blajda))
+- chore: address some integration test bloat of disk usage for development [\#1552](https://github.com/delta-io/delta-rs/pull/1552) ([rtyler](https://github.com/rtyler))
+- docs: port docs to mkdocs [\#1548](https://github.com/delta-io/delta-rs/pull/1548) ([MrPowers](https://github.com/MrPowers))
+- chore: disable incremental builds in CI for saving space [\#1545](https://github.com/delta-io/delta-rs/pull/1545) ([rtyler](https://github.com/rtyler))
+- fix: revert premature merge of an attempted fix for binary column statistics [\#1544](https://github.com/delta-io/delta-rs/pull/1544) ([rtyler](https://github.com/rtyler))
+- chore: increment python version [\#1542](https://github.com/delta-io/delta-rs/pull/1542) ([wjones127](https://github.com/wjones127))
+- feat: add restore command in python binding [\#1529](https://github.com/delta-io/delta-rs/pull/1529) ([loleek](https://github.com/loleek))
+
+## [rust-v0.13.1](https://github.com/delta-io/delta-rs/tree/rust-v0.13.1) (2023-07-18)
+
+**Fixed bugs:**
+
+* Revert premature merge of an attempted fix for binary column statistics [\#1544](https://github.com/delta-io/delta-rs/pull/1544)
+
 ## [rust-v0.13.0](https://github.com/delta-io/delta-rs/tree/rust-v0.13.0) (2023-07-15)
 
 [Full Changelog](https://github.com/delta-io/delta-rs/compare/rust-v0.12.0...rust-v0.13.0)

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,10 +1,52 @@
 [workspace]
-members = [
-    "rust",
-    "python",
-]
+members = ["rust", "python"]
 exclude = ["proofs", "delta-inspect"]
+resolver = "2"
 
 [profile.release-with-debug]
 inherits = "release"
 debug = true
+
+# Reducing the debuginfo for the test profile in order to trim the disk and RAM
+# usage during development
+# <https://github.com/delta-io/delta-rs/issues/1550?
+[profile.test]
+debug = "line-tables-only"
+
+[workspace.dependencies]
+# arrow
+arrow = { version = "46" }
+arrow-array = { version = "46" }
+arrow-buffer = { version = "46" }
+arrow-cast = { version = "46" }
+arrow-ord = { version = "46" }
+arrow-row = { version = "46" }
+arrow-schema = { version = "46" }
+arrow-select = { version = "46" }
+parquet = { version = "46" }
+
+# datafusion
+datafusion = { version = "31" }
+datafusion-expr = { version = "31" }
+datafusion-common = { version = "31" }
+datafusion-proto = { version = "31" }
+datafusion-sql = { version = "31" }
+datafusion-physical-expr = { version = "31" }
+
+# serde
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+# "stdlib"
+bytes = { version = "1" }
+chrono = { version = "0.4.31", default-features = false, features = ["clock"] }
+regex = { version = "1" }
+thiserror = { version = "1" }
+url = { version = "2" }
+uuid = { version = "1" }
+
+# runtime / async
+async-trait = { version = "0.1" }
+futures = { version = "0.3" }
+tokio = { version = "1" }
+num_cpus = { version = "1" }
diff --git a/README.adoc b/README.adoc