Add ONTAP S3 Store with existence cache

nativelink-config/src/stores.rs

@@ -78,6 +78,59 @@ pub enum StoreSpec {
     ///
     experimental_s3_store(S3Spec),
 
+    /// `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend
+    /// to store files. This store is specifically configured for ONTAP's S3 requirements
+    /// including custom TLS configuration, credentials management, and proper vserver
+    /// configuration.
+    ///
+    /// This store uses AWS environment variables for credentials:
+    /// - `AWS_ACCESS_KEY_ID`
+    /// - `AWS_SECRET_ACCESS_KEY`
+    /// - `AWS_DEFAULT_REGION`
+    ///
+    /// Example JSON Config:
+    /// ```json
+    /// "ontap_s3_store": {
+    ///   "endpoint": "https://ontap-s3-endpoint:443",
+    ///   "vserver_name": "your-vserver",
+    ///   "bucket": "your-bucket",
+    ///   "root_certificates": "/path/to/certs.pem",  // Optional
+    ///   "key_prefix": "test-prefix/",               // Optional
+    ///   "retry": {
+    ///     "max_retries": 6,
+    ///     "delay": 0.3,
+    ///     "jitter": 0.5
+    ///   },
+    ///   "multipart_max_concurrent_uploads": 10
+    /// }
+    /// ```
+    ontap_s3_store(OntapS3Spec),
+
+    /// ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store
+    /// to optimize repeated existence checks. It maintains an in-memory cache of object
+    /// digests and periodically syncs this cache to disk for persistence.
+    ///
+    /// The cache helps reduce latency for repeated calls to check object existence,
+    /// while still ensuring eventual consistency with the underlying ONTAP S3 store.
+    ///
+    /// Example JSON Config:
+    /// ```json
+    /// "ontap_s3_existence_cache": {
+    ///   "index_path": "/path/to/cache/index.json",
+    ///   "sync_interval_seconds": 300,
+    ///   "backend": {
+    ///     "ontap_s3_store": {
+    ///       "endpoint": "https://ontap-s3-endpoint:443",
+    ///       "vserver_name": "your-vserver",
+    ///       "bucket": "your-bucket",
+    ///       "key_prefix": "test-prefix/"
+    ///     }
+    ///   }
+    /// }
+    /// ```
+    ///
+    ontap_s3_existence_cache(Box<OntapS3ExistenceCacheSpec>),
+
     /// Verify store is used to apply verifications to an underlying
     /// store implementation. It is strongly encouraged to validate
     /// as much data as you can before accepting data from a client,
@@ -512,6 +565,42 @@ pub struct FilesystemSpec {
     pub block_size: u64,
 }
 
+// NetApp ONTAP S3 Spec
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct OntapS3Spec {
+    #[serde(deserialize_with = "convert_string_with_shellexpand")]
+    pub endpoint: String,
+    #[serde(deserialize_with = "convert_string_with_shellexpand")]
+    pub vserver_name: String,
+    #[serde(deserialize_with = "convert_string_with_shellexpand")]
+    pub bucket: String,
+    #[serde(default)]
+    pub root_certificates: Option<String>,
+    #[serde(default)]
+    pub key_prefix: Option<String>,
+    #[serde(default)]
+    pub retry: Retry,
+    #[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
+    pub consider_expired_after_s: u32,
+    pub max_retry_buffer_per_request: Option<usize>,
+    pub multipart_max_concurrent_uploads: Option<usize>,
+    #[serde(default)]
+    pub insecure_allow_http: bool,
+    #[serde(default)]
+    pub disable_http2: bool,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct OntapS3ExistenceCacheSpec {
+    #[serde(deserialize_with = "convert_string_with_shellexpand")]
+    pub index_path: String,
+    #[serde(deserialize_with = "convert_numeric_with_shellexpand")]
+    pub sync_interval_seconds: u32,
+    pub backend: Box<StoreSpec>,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(deny_unknown_fields)]
 pub struct FastSlowSpec {

nativelink-store/BUILD.bazel

@@ -22,6 +22,8 @@ rust_library(
         "src/lib.rs",
         "src/memory_store.rs",
         "src/noop_store.rs",
+        "src/ontap_s3_existence_cache_store.rs",
+        "src/ontap_s3_store.rs",
         "src/redis_store.rs",
         "src/redis_utils/ft_aggregate.rs",
         "src/redis_utils/mod.rs",
@@ -53,18 +55,23 @@ rust_library(
         "@crates//:bytes",
         "@crates//:bytes-utils",
         "@crates//:const_format",
+        "@crates//:filetime",
         "@crates//:fred",
         "@crates//:futures",
         "@crates//:hex",
         "@crates//:http-body",
         "@crates//:hyper-0.14.32",
         "@crates//:hyper-rustls",
+        "@crates//:hyper-util",
         "@crates//:lz4_flex",
         "@crates//:parking_lot",
         "@crates//:patricia_tree",
         "@crates//:prost",
         "@crates//:rand",
+        "@crates//:rustls",
+        "@crates//:rustls-pemfile",
         "@crates//:serde",
+        "@crates//:serde_json",
         "@crates//:tokio",
         "@crates//:tokio-stream",
         "@crates//:tokio-util",
@@ -86,6 +93,8 @@ rust_test_suite(
         "tests/fast_slow_store_test.rs",
         "tests/filesystem_store_test.rs",
         "tests/memory_store_test.rs",
+        "tests/ontap_s3_existence_cache_store_test.rs",
+        "tests/ontap_s3_store_test.rs",
         "tests/redis_store_test.rs",
         "tests/ref_store_test.rs",
         "tests/s3_store_test.rs",
@@ -126,6 +135,7 @@ rust_test_suite(
         "@crates//:serde_json",
         "@crates//:serial_test",
         "@crates//:sha2",
+        "@crates//:tempfile",
         "@crates//:tokio",
         "@crates//:tokio-stream",
         "@crates//:tracing",

nativelink-store/Cargo.toml

@@ -54,7 +54,10 @@ lz4_flex = { version = "0.11.3", default-features = false }
 parking_lot = "0.12.3"
 prost = { version = "0.13.5", default-features = false }
 rand = { version = "0.9.0", default-features = false, features = ["thread_rng"] }
+rustls = { version = "0.21.12", default-features = false, features = ["dangerous_configuration",] }
+rustls-pemfile = { version = "2.1.2", default-features = false }
 serde = { version = "1.0.218", default-features = false }
+serde_json = { version = "1.0.135", default-features = false }
 tokio = { version = "1.43.0", features = ["fs", "rt-multi-thread", "signal", "io-util"], default-features = false }
 tokio-stream = { version = "0.1.17", features = ["fs"], default-features = false }
 tokio-util = { version = "0.7.13" }
@@ -80,5 +83,6 @@ aws-sdk-s3 = { version = "=1.78.0", features = [
 aws-smithy-runtime-api = "=1.7.3"
 rand = { version = "0.9.0", default-features = false, features = ["thread_rng", "small_rng"] }
 serde_json = "1.0.139"
+tempfile = "3.8.1"
 fred = { version = "10.0.4", default-features = false, features = ["mocks"] }
 tracing-subscriber = { version = "0.3.19", default-features = false }

nativelink-store/src/default_store_factory.rs

@@ -32,6 +32,8 @@ use crate::filesystem_store::FilesystemStore;
 use crate::grpc_store::GrpcStore;
 use crate::memory_store::MemoryStore;
 use crate::noop_store::NoopStore;
+use crate::ontap_s3_existence_cache_store::OntapS3ExistenceCache;
+use crate::ontap_s3_store::OntapS3Store;
 use crate::redis_store::RedisStore;
 use crate::ref_store::RefStore;
 use crate::s3_store::S3Store;
@@ -51,6 +53,10 @@ pub fn store_factory<'a>(
         let store: Arc<dyn StoreDriver> = match backend {
             StoreSpec::memory(spec) => MemoryStore::new(spec),
             StoreSpec::experimental_s3_store(spec) => S3Store::new(spec, SystemTime::now).await?,
+            StoreSpec::ontap_s3_store(spec) => OntapS3Store::new(spec, SystemTime::now).await?,
+            StoreSpec::ontap_s3_existence_cache(spec) => {
+                OntapS3ExistenceCache::new(spec, SystemTime::now).await?
+            }
             StoreSpec::redis_store(spec) => RedisStore::new(spec.clone())?,
             StoreSpec::verify(spec) => VerifyStore::new(
                 spec,

nativelink-store/src/lib.rs

@@ -24,6 +24,8 @@ pub mod filesystem_store;
 pub mod grpc_store;
 pub mod memory_store;
 pub mod noop_store;
+pub mod ontap_s3_existence_cache_store;
+pub mod ontap_s3_store;
 pub mod redis_store;
 mod redis_utils;
 pub mod ref_store;