feat: use url parsing from object store

.github/workflows/build.yml

@@ -102,7 +102,7 @@ jobs:
       AWS_ACCESS_KEY_ID: deltalake
       AWS_SECRET_ACCESS_KEY: weloverust
       AWS_ENDPOINT_URL: http://localhost:4566
-      AWS_STORAGE_ALLOW_HTTP: "1"
+      AWS_ALLOW_HTTP: "1"
       AZURE_USE_EMULATOR: "1"
       AZURE_STORAGE_ALLOW_HTTP: "1"
       AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"

python/tests/conftest.py

@@ -103,7 +103,7 @@ def s3_localstack_creds():
 
 @pytest.fixture()
 def s3_localstack(monkeypatch, s3_localstack_creds):
-    monkeypatch.setenv("AWS_STORAGE_ALLOW_HTTP", "TRUE")
+    monkeypatch.setenv("AWS_ALLOW_HTTP", "TRUE")
     for key, value in s3_localstack_creds.items():
         monkeypatch.setenv(key, value)
 

python/tests/test_fs.py

@@ -139,7 +139,7 @@ def test_roundtrip_s3_direct(s3_localstack_creds, sample_data: pa.Table):
 
     # Can pass storage_options in directly
     storage_opts = {
-        "AWS_STORAGE_ALLOW_HTTP": "true",
+        "AWS_ALLOW_HTTP": "true",
         "AWS_S3_ALLOW_UNSAFE_RENAME": "true",
     }
     storage_opts.update(s3_localstack_creds)

rust/Cargo.toml

@@ -108,7 +108,7 @@ dashmap = { version = "5", optional = true }
 sqlparser = { version = "0.37", optional = true }
 
 # NOTE dependencies only for integration tests
-fs_extra = { version = "1.2.0", optional = true }
+fs_extra = { version = "1.3.0", optional = true }
 tempdir = { version = "0", optional = true }
 
 dynamodb_lock = { version = "0", default-features = false, optional = true }

rust/examples/recordbatch-writer.rs

@@ -45,7 +45,7 @@ async fn main() -> Result<(), DeltaTableError> {
             info!("It doesn't look like our delta table has been created");
             create_initialized_table(&table_path).await
         }
-        Err(err) => Err(err).unwrap(),
+        Err(err) => panic!("{:?}", err),
     };
 
     let writer_properties = WriterProperties::builder()

rust/src/builder.rs

@@ -341,7 +341,7 @@ pub mod s3_storage_options {
     /// See also <https://docs.rs/rusoto_sts/0.47.0/rusoto_sts/struct.WebIdentityProvider.html#method.from_k8s_env>.
     pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME";
     /// Allow http connections - mainly useful for integration tests
-    pub const AWS_STORAGE_ALLOW_HTTP: &str = "AWS_STORAGE_ALLOW_HTTP";
+    pub const AWS_ALLOW_HTTP: &str = "AWS_ALLOW_HTTP";
 
     /// If set to "true", allows creating commits without concurrent writer protection.
     /// Only safe if there is one writer to a given table.

rust/src/data_catalog/storage/mod.rs

@@ -46,9 +46,9 @@ impl ListingSchemaProvider {
         storage_options: Option<HashMap<String, String>>,
     ) -> DeltaResult<Self> {
         let uri = ensure_table_uri(root_uri)?;
-        let storage_options = storage_options.unwrap_or_default().into();
+        let mut storage_options = storage_options.unwrap_or_default().into();
         // We already parsed the url, so unwrapping is safe.
-        let store = configure_store(&uri, &storage_options)?;
+        let store = configure_store(&uri, &mut storage_options)?;
         Ok(Self {
             authority: uri.to_string(),
             store,