Merge pull request #2579 from subspace/farmer-caching-tweaks

Farmer caching tweaks

Author: nazar-pc

crates/subspace-farmer-components/src/file_ext.rs

@@ -30,7 +30,14 @@ impl OpenOptionsExt for OpenOptions {
     #[cfg(windows)]
     fn advise_random_access(&mut self) -> &mut Self {
         use std::os::windows::fs::OpenOptionsExt;
-        self.custom_flags(winapi::um::winbase::FILE_FLAG_RANDOM_ACCESS)
+        // `FILE_FLAG_WRITE_THROUGH` below is a bit of a hack, especially in `advise_random_access`,
+        // but it helps with memory usage and feels like should be default. Since `.custom_flags()`
+        // overrides previous value, we need to set bitwise OR of two flags rather that two flags
+        // separately.
+        self.custom_flags(
+            winapi::um::winbase::FILE_FLAG_RANDOM_ACCESS
+                | winapi::um::winbase::FILE_FLAG_WRITE_THROUGH,
+        )
     }
 
     #[cfg(target_os = "linux")]

crates/subspace-farmer/src/farmer_cache.rs

@@ -886,10 +886,18 @@ impl FarmerCache {
 
         let should_store_fut = tokio::task::spawn_blocking({
             let plot_caches = Arc::clone(&self.plot_caches);
+            let piece_caches = Arc::clone(&self.piece_caches);
             let next_plot_cache = Arc::clone(&self.next_plot_cache);
             let piece = piece.clone();
 
             move || {
+                for cache in piece_caches.read().iter() {
+                    if cache.stored_pieces.contains_key(&key) {
+                        // Already stored in normal piece cache, no need to store it again
+                        return;
+                    }
+                }
+
                 let plot_caches = plot_caches.read();
                 let plot_caches_len = plot_caches.len();
 

crates/subspace-farmer/src/single_disk_farm/plot_cache.rs

@@ -16,6 +16,13 @@ use subspace_networking::utils::multihash::ToMultihash;
 use thiserror::Error;
 use tracing::{debug, info, warn};
 
+/// Max plot space for which to use caching, for larger gaps between the plotted part and the end of
+/// the file it will result in very long period of writing zeroes on Windows, see
+/// https://stackoverflow.com/q/78058306/3806795
+///
+/// Currently set to 2TiB.
+const MAX_WINDOWS_PLOT_SPACE_FOR_CACHE: u64 = 2 * 1024 * 1024 * 1024 * 1024;
+
 /// Disk plot cache open error
 #[derive(Debug, Error)]
 pub enum DiskPlotCacheError {
@@ -68,33 +75,36 @@ impl DiskPlotCache {
             // Clippy complains about `RecordKey`, but it is not changing here, so it is fine
             #[allow(clippy::mutable_key_type)]
             let mut map = HashMap::new();
+            let mut next_offset = None;
 
             let file_size = sector_size * u64::from(target_sector_count);
             let plotted_size = sector_size * sectors_metadata.len() as u64;
 
-            // Step over all free potential offsets for pieces that could have been cached
-            let from_offset = (plotted_size / Self::element_size() as u64) as u32;
-            let to_offset = (file_size / Self::element_size() as u64) as u32;
-            let mut next_offset = None;
-            // TODO: Parallelize or read in larger batches
-            for offset in (from_offset..to_offset).rev() {
-                match Self::read_piece_internal(file, offset, &mut element) {
-                    Ok(maybe_piece_index) => match maybe_piece_index {
-                        Some(piece_index) => {
-                            map.insert(RecordKey::from(piece_index.to_multihash()), offset);
-                        }
-                        None => {
+            // Avoid writing over large gaps on Windows that is very lengthy process
+            if !cfg!(windows) || (file_size - plotted_size) <= MAX_WINDOWS_PLOT_SPACE_FOR_CACHE {
+                // Step over all free potential offsets for pieces that could have been cached
+                let from_offset = (plotted_size / Self::element_size() as u64) as u32;
+                let to_offset = (file_size / Self::element_size() as u64) as u32;
+                // TODO: Parallelize or read in larger batches
+                for offset in (from_offset..to_offset).rev() {
+                    match Self::read_piece_internal(file, offset, &mut element) {
+                        Ok(maybe_piece_index) => match maybe_piece_index {
+                            Some(piece_index) => {
+                                map.insert(RecordKey::from(piece_index.to_multihash()), offset);
+                            }
+                            None => {
+                                next_offset.replace(offset);
+                                break;
+                            }
+                        },
+                        Err(DiskPlotCacheError::ChecksumMismatch) => {
                             next_offset.replace(offset);
                             break;
                         }
-                    },
-                    Err(DiskPlotCacheError::ChecksumMismatch) => {
-                        next_offset.replace(offset);
-                        break;
-                    }
-                    Err(error) => {
-                        warn!(%error, %offset, "Failed to read plot cache element");
-                        break;
+                        Err(error) => {
+                            warn!(%error, %offset, "Failed to read plot cache element");
+                            break;
+                        }
                     }
                 }
             }

crates/subspace-farmer/src/utils.rs

@@ -20,9 +20,7 @@ use std::{io, thread};
 use thread_priority::{set_current_thread_priority, ThreadPriority};
 use tokio::runtime::Handle;
 use tokio::task;
-use tracing::debug;
-#[cfg(feature = "numa")]
-use tracing::warn;
+use tracing::{debug, warn};
 
 /// It doesn't make a lot of sense to have a huge number of farming threads, 32 is plenty
 const MAX_DEFAULT_FARMING_THREADS: usize = 32;