Skip to content

Commit 446ecca

Browse files
nazar-pcnazar-pc
authored
Merge pull request #2581 from subspace/unbuffered-farming-io-windows
Use a wrapper with unbuffered I/O for farming on Windows
2 parents 1486b54 + 424b512 commit 446ecca

File tree

9 files changed

+401
-90
lines changed

9 files changed

+401
-90
lines changed

‎crates/subspace-farmer-components/src/file_ext.rs

+27
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@ pub trait OpenOptionsExt {
99
/// undesirable, only has impact on Windows, for other operating systems see [`FileExt`]
1010
fn advise_random_access(&mut self) -> &mut Self;
1111

12+
/// Advise Windows to not use buffering for this file and that file access will be random.
13+
///
14+
/// NOTE: There are major alignment requirements described here:
15+
/// https://learn.microsoft.com/en-us/windows/win32/fileio/file-buffering#alignment-and-file-access-requirements
16+
#[cfg(windows)]
17+
fn advise_unbuffered(&mut self) -> &mut Self;
18+
1219
/// Advise OS/file system that file will use sequential access and read-ahead behavior is
1320
/// desirable, only has impact on Windows, for other operating systems see [`FileExt`]
1421
fn advise_sequential_access(&mut self) -> &mut Self;
@@ -40,6 +47,15 @@ impl OpenOptionsExt for OpenOptions {
4047
)
4148
}
4249

50+
#[cfg(windows)]
51+
fn advise_unbuffered(&mut self) -> &mut Self {
52+
use std::os::windows::fs::OpenOptionsExt;
53+
self.custom_flags(
54+
winapi::um::winbase::FILE_FLAG_WRITE_THROUGH
55+
| winapi::um::winbase::FILE_FLAG_NO_BUFFERING,
56+
)
57+
}
58+
4359
#[cfg(target_os = "linux")]
4460
fn advise_sequential_access(&mut self) -> &mut Self {
4561
// Not supported
@@ -62,6 +78,9 @@ impl OpenOptionsExt for OpenOptions {
6278
/// Extension convenience trait that allows pre-allocating files, suggesting random access pattern
6379
/// and doing cross-platform exact reads/writes
6480
pub trait FileExt {
81+
/// Get allocated file size
82+
fn allocated_size(&self) -> Result<u64>;
83+
6584
/// Make sure file has specified number of bytes allocated for it
6685
fn preallocate(&self, len: u64) -> Result<()>;
6786

@@ -81,7 +100,15 @@ pub trait FileExt {
81100
}
82101

83102
impl FileExt for File {
103+
fn allocated_size(&self) -> Result<u64> {
104+
fs4::FileExt::allocated_size(self)
105+
}
106+
84107
fn preallocate(&self, len: u64) -> Result<()> {
108+
// TODO: Hack due to bugs on Windows: https://github.com/al8n/fs4-rs/issues/13
109+
if fs4::FileExt::allocated_size(self)? == len {
110+
return Ok(());
111+
}
85112
fs4::FileExt::allocate(self, len)
86113
}
87114

‎crates/subspace-farmer-components/src/reading.rs

+26-4
Original file line numberDiff line numberDiff line change
@@ -148,16 +148,39 @@ where
148148
)
149149
.collect::<Vec<_>>();
150150

151+
let sector_contents_map_size = SectorContentsMap::encoded_size(pieces_in_sector) as u64;
151152
match sector {
152153
ReadAt::Sync(sector) => {
154+
// TODO: Random reads are slow on Windows due to a variety of bugs with its disk
155+
// subsystem:
156+
// * https://learn.microsoft.com/en-us/answers/questions/1601862/windows-is-leaking-memory-when-reading-random-chun
157+
// * https://learn.microsoft.com/en-us/answers/questions/1608540/getfileinformationbyhandle-followed-by-read-with-f
158+
// * and likely some more that are less obvious
159+
// As a workaround, read the whole sector at once instead of individual record chunks,
160+
// which while results in higher data transfer from SSD, reads data in larger blocks
161+
// and ends up being faster in many cases even though it uses more RAM while doing so.
162+
// It is also likely possible to read large parts of the sector instead of the whole
163+
// sector if code below is not parallelized, but logic will become even more
164+
// convoluted, so for now it is not done.
165+
#[cfg(windows)]
166+
let sector_bytes = {
167+
let mut sector_bytes = vec![0u8; crate::sector::sector_size(pieces_in_sector)];
168+
sector.read_at(&mut sector_bytes, 0)?;
169+
sector_bytes
170+
};
153171
read_chunks_inputs.into_par_iter().flatten().try_for_each(
154172
|(maybe_record_chunk, chunk_location, encoded_chunk_used, s_bucket)| {
155173
let mut record_chunk = [0; Scalar::FULL_BYTES];
174+
#[cfg(windows)]
175+
record_chunk.copy_from_slice(
176+
&sector_bytes[sector_contents_map_size as usize
177+
+ chunk_location as usize * Scalar::FULL_BYTES..][..Scalar::FULL_BYTES],
178+
);
179+
#[cfg(not(windows))]
156180
sector
157181
.read_at(
158182
&mut record_chunk,
159-
SectorContentsMap::encoded_size(pieces_in_sector) as u64
160-
+ chunk_location * Scalar::FULL_BYTES as u64,
183+
sector_contents_map_size + chunk_location * Scalar::FULL_BYTES as u64,
161184
)
162185
.map_err(|error| ReadingError::FailedToReadChunk {
163186
chunk_location,
@@ -198,8 +221,7 @@ where
198221
&sector
199222
.read_at(
200223
vec![0; Scalar::FULL_BYTES],
201-
SectorContentsMap::encoded_size(pieces_in_sector) as u64
202-
+ chunk_location * Scalar::FULL_BYTES as u64,
224+
sector_contents_map_size + chunk_location * Scalar::FULL_BYTES as u64,
203225
)
204226
.await
205227
.map_err(|error| ReadingError::FailedToReadChunk {

‎crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs

+93-7
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg};
1010
use subspace_core_primitives::{Record, SolutionRange};
1111
use subspace_erasure_coding::ErasureCoding;
1212
use subspace_farmer::single_disk_farm::farming::rayon_files::RayonFiles;
13+
use subspace_farmer::single_disk_farm::farming::unbuffered_io_file_windows::UnbufferedIoFileWindows;
1314
use subspace_farmer::single_disk_farm::farming::{PlotAudit, PlotAuditOptions};
1415
use subspace_farmer::single_disk_farm::{SingleDiskFarm, SingleDiskFarmSummary};
1516
use subspace_farmer_components::sector::sector_size;
@@ -161,12 +162,48 @@ fn audit(
161162
)
162163
});
163164
}
165+
if cfg!(windows) {
166+
let plot = RayonFiles::open_with(
167+
&disk_farm.join(SingleDiskFarm::PLOT_FILE),
168+
UnbufferedIoFileWindows::open,
169+
)
170+
.map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?;
171+
let plot_audit = PlotAudit::new(&plot);
172+
173+
group.bench_function("plot/rayon/unbuffered", |b| {
174+
b.iter_batched(
175+
rand::random,
176+
|global_challenge| {
177+
let options = PlotAuditOptions::<PosTable> {
178+
public_key: single_disk_farm_info.public_key(),
179+
reward_address: single_disk_farm_info.public_key(),
180+
slot_info: SlotInfo {
181+
slot_number: 0,
182+
global_challenge,
183+
// No solution will be found, pure audit
184+
solution_range: SolutionRange::MIN,
185+
// No solution will be found, pure audit
186+
voting_solution_range: SolutionRange::MIN,
187+
},
188+
sectors_metadata: &sectors_metadata,
189+
kzg: &kzg,
190+
erasure_coding: &erasure_coding,
191+
maybe_sector_being_modified: None,
192+
table_generator: &table_generator,
193+
};
194+
195+
black_box(plot_audit.audit(black_box(options)))
196+
},
197+
BatchSize::SmallInput,
198+
)
199+
});
200+
}
164201
{
165202
let plot = RayonFiles::open(&disk_farm.join(SingleDiskFarm::PLOT_FILE))
166203
.map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?;
167204
let plot_audit = PlotAudit::new(&plot);
168205

169-
group.bench_function("plot/rayon", |b| {
206+
group.bench_function("plot/rayon/regular", |b| {
170207
b.iter_batched(
171208
rand::random,
172209
|global_challenge| {
@@ -252,7 +289,7 @@ fn prove(
252289
.open(disk_farm.join(SingleDiskFarm::PLOT_FILE))
253290
.map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?;
254291
let plot_audit = PlotAudit::new(&plot);
255-
let options = PlotAuditOptions::<PosTable> {
292+
let mut options = PlotAuditOptions::<PosTable> {
256293
public_key: single_disk_farm_info.public_key(),
257294
reward_address: single_disk_farm_info.public_key(),
258295
slot_info: SlotInfo {
@@ -267,7 +304,7 @@ fn prove(
267304
kzg: &kzg,
268305
erasure_coding: &erasure_coding,
269306
maybe_sector_being_modified: None,
270-
table_generator: &table_generator,
307+
table_generator: &Mutex::new(PosTable::generator()),
271308
};
272309

273310
let mut audit_results = plot_audit.audit(options).unwrap();
@@ -279,12 +316,60 @@ fn prove(
279316
return result;
280317
}
281318

319+
options.slot_info.global_challenge = rand::random();
320+
audit_results = plot_audit.audit(options).unwrap();
321+
322+
audit_results.pop().unwrap()
323+
},
324+
|(_sector_index, mut provable_solutions)| {
325+
while black_box(provable_solutions.next()).is_none() {
326+
// Try to create one solution and exit
327+
}
328+
},
329+
BatchSize::SmallInput,
330+
)
331+
});
332+
}
333+
if cfg!(windows) {
334+
let plot = RayonFiles::open_with(
335+
&disk_farm.join(SingleDiskFarm::PLOT_FILE),
336+
UnbufferedIoFileWindows::open,
337+
)
338+
.map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?;
339+
let plot_audit = PlotAudit::new(&plot);
340+
let mut options = PlotAuditOptions::<PosTable> {
341+
public_key: single_disk_farm_info.public_key(),
342+
reward_address: single_disk_farm_info.public_key(),
343+
slot_info: SlotInfo {
344+
slot_number: 0,
345+
global_challenge: rand::random(),
346+
// Solution is guaranteed to be found
347+
solution_range: SolutionRange::MAX,
348+
// Solution is guaranteed to be found
349+
voting_solution_range: SolutionRange::MAX,
350+
},
351+
sectors_metadata: &sectors_metadata,
352+
kzg: &kzg,
353+
erasure_coding: &erasure_coding,
354+
maybe_sector_being_modified: None,
355+
table_generator: &table_generator,
356+
};
357+
let mut audit_results = plot_audit.audit(options).unwrap();
358+
359+
group.bench_function("plot/rayon/unbuffered", |b| {
360+
b.iter_batched(
361+
|| {
362+
if let Some(result) = audit_results.pop() {
363+
return result;
364+
}
365+
366+
options.slot_info.global_challenge = rand::random();
282367
audit_results = plot_audit.audit(options).unwrap();
283368

284369
audit_results.pop().unwrap()
285370
},
286371
|(_sector_index, mut provable_solutions)| {
287-
while (provable_solutions.next()).is_none() {
372+
while black_box(provable_solutions.next()).is_none() {
288373
// Try to create one solution and exit
289374
}
290375
},
@@ -296,7 +381,7 @@ fn prove(
296381
let plot = RayonFiles::open(&disk_farm.join(SingleDiskFarm::PLOT_FILE))
297382
.map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?;
298383
let plot_audit = PlotAudit::new(&plot);
299-
let options = PlotAuditOptions::<PosTable> {
384+
let mut options = PlotAuditOptions::<PosTable> {
300385
public_key: single_disk_farm_info.public_key(),
301386
reward_address: single_disk_farm_info.public_key(),
302387
slot_info: SlotInfo {
@@ -315,19 +400,20 @@ fn prove(
315400
};
316401
let mut audit_results = plot_audit.audit(options).unwrap();
317402

318-
group.bench_function("plot/rayon", |b| {
403+
group.bench_function("plot/rayon/regular", |b| {
319404
b.iter_batched(
320405
|| {
321406
if let Some(result) = audit_results.pop() {
322407
return result;
323408
}
324409

410+
options.slot_info.global_challenge = rand::random();
325411
audit_results = plot_audit.audit(options).unwrap();
326412

327413
audit_results.pop().unwrap()
328414
},
329415
|(_sector_index, mut provable_solutions)| {
330-
while (provable_solutions.next()).is_none() {
416+
while black_box(provable_solutions.next()).is_none() {
331417
// Try to create one solution and exit
332418
}
333419
},

‎crates/subspace-farmer/src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
iter_collect_into,
1010
let_chains,
1111
never_type,
12+
slice_flatten,
1213
trait_alias,
1314
try_blocks,
1415
type_alias_impl_trait,

0 commit comments

Comments
 (0)