Skip to content

Commit 8f4ade4

Browse files
Leo-BesanconLeo-Besancon
committed
Add metrics for massa-db-worker change history sizes
1 parent 69c1725 commit 8f4ade4

File tree

22 files changed

+132
-0
lines changed

22 files changed

+132
-0
lines changed

‎massa-async-pool/src/pool.rs

+6
Original file line numberDiff line numberDiff line change
@@ -1100,6 +1100,7 @@ mod tests {
11001100
max_versioning_elements_size: 100,
11011101
thread_count: THREAD_COUNT,
11021102
max_ledger_backups: 100,
1103+
enable_metrics: true,
11031104
};
11041105
let db: ShareableMassaDBController = Arc::new(RwLock::new(
11051106
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
@@ -1142,6 +1143,7 @@ mod tests {
11421143
max_versioning_elements_size: 100,
11431144
thread_count: THREAD_COUNT,
11441145
max_ledger_backups: 100,
1146+
enable_metrics: true,
11451147
};
11461148
let db: ShareableMassaDBController = Arc::new(RwLock::new(
11471149
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
@@ -1204,6 +1206,7 @@ mod tests {
12041206
max_versioning_elements_size: 100,
12051207
thread_count: THREAD_COUNT,
12061208
max_ledger_backups: 100,
1209+
enable_metrics: true,
12071210
};
12081211
let db: ShareableMassaDBController = Arc::new(RwLock::new(
12091212
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
@@ -1261,6 +1264,7 @@ mod tests {
12611264
max_versioning_elements_size: 100,
12621265
thread_count: THREAD_COUNT,
12631266
max_ledger_backups: 100,
1267+
enable_metrics: true,
12641268
};
12651269
let db: ShareableMassaDBController = Arc::new(RwLock::new(
12661270
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
@@ -1318,6 +1322,7 @@ mod tests {
13181322
max_versioning_elements_size: 100,
13191323
thread_count: THREAD_COUNT,
13201324
max_ledger_backups: 100,
1325+
enable_metrics: true,
13211326
};
13221327
let db: ShareableMassaDBController = Arc::new(RwLock::new(
13231328
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>,
@@ -1368,6 +1373,7 @@ mod tests {
13681373
max_versioning_elements_size: 100,
13691374
thread_count: THREAD_COUNT,
13701375
max_ledger_backups: 100,
1376+
enable_metrics: true,
13711377
};
13721378
let db: ShareableMassaDBController = Arc::new(RwLock::new(Box::new(MassaDB::new(
13731379
db_config.clone(),

‎massa-bootstrap/src/tests/binders.rs

+3
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@ fn test_staying_connected_without_message_trigger_read_timeout() {
451451
max_versioning_elements_size: 100_000_000,
452452
thread_count: THREAD_COUNT,
453453
max_ledger_backups: 10,
454+
enable_metrics: true,
454455
}))
455456
as Box<(dyn MassaDBController + 'static)>));
456457
let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap();
@@ -549,6 +550,7 @@ fn test_staying_connected_pass_handshake_but_deadline_after() {
549550
max_versioning_elements_size: 100_000_000,
550551
thread_count: THREAD_COUNT,
551552
max_ledger_backups: 10,
553+
enable_metrics: true,
552554
}))
553555
as Box<(dyn MassaDBController + 'static)>));
554556
let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap();
@@ -647,6 +649,7 @@ fn test_staying_connected_pass_handshake_but_deadline_during_data_exchange() {
647649
max_versioning_elements_size: 100_000_000,
648650
thread_count: THREAD_COUNT,
649651
max_ledger_backups: 10,
652+
enable_metrics: true,
650653
}))
651654
as Box<(dyn MassaDBController + 'static)>));
652655
let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap();

‎massa-bootstrap/src/tests/universe_client.rs

+1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ impl TestUniverse for BootstrapClientTestUniverse {
6060
max_final_state_elements_size: MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE as usize,
6161
thread_count: THREAD_COUNT,
6262
max_ledger_backups: 10,
63+
enable_metrics: true,
6364
}))
6465
as Box<(dyn MassaDBController + 'static)>));
6566
controllers

‎massa-bootstrap/src/tests/universe_server.rs

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ impl BootstrapServerForeignControllers {
5757
max_final_state_elements_size: MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE as usize,
5858
thread_count: THREAD_COUNT,
5959
max_ledger_backups: 10,
60+
enable_metrics: true,
6061
}))
6162
as Box<(dyn MassaDBController + 'static)>));
6263
Self {

‎massa-db-exports/src/controller.rs

+3
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ pub trait MassaDBController: Send + Sync + Debug {
8888
last_change_id: Option<Slot>,
8989
) -> Result<StreamBatch<Slot>, MassaDBError>;
9090

91+
/// Get the total size of the change history and the change versioning history respectively
92+
fn get_change_history_sizes(&self) -> (usize, usize);
93+
9194
/// Used in test to compare a prebuilt ledger with a ledger that has been built by the code
9295
#[cfg(feature = "test-exports")]
9396
fn get_entire_database(&self) -> Vec<BTreeMap<Vec<u8>, Vec<u8>>>;

‎massa-db-exports/src/settings.rs

+2
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,6 @@ pub struct MassaDBConfig {
1515
pub thread_count: u8,
1616
/// Maximum number of ledger backups to keep
1717
pub max_ledger_backups: u64,
18+
/// Enable metrics
19+
pub enable_metrics: bool,
1820
}

‎massa-db-worker/src/massa_db.rs

+58
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ pub struct RawMassaDB<
5454
pub change_id_deserializer: ChangeIDDeserializer,
5555
/// The current RocksDB batch of the database, in a Mutex to share it
5656
pub current_batch: Arc<Mutex<WriteBatch>>,
57+
/// If metrics are enabled, we keep track of the size of the changes associated to each change_id
58+
pub change_history_sizes: BTreeMap<ChangeID, (usize, usize)>,
5759
}
5860

5961
impl<ChangeID, ChangeIDSerializer, ChangeIDDeserializer> std::fmt::Debug
@@ -389,6 +391,12 @@ where
389391
}
390392
}
391393

394+
let changes_size = if self.config.enable_metrics {
395+
self.current_batch.lock().size_in_bytes()
396+
} else {
397+
0
398+
};
399+
392400
// in versioning_changes, we have the data that we do not want to include in hash
393401
// e.g everything that is not in 'Active' state (so hashes remain compatibles)
394402
for (key, value) in versioning_changes.iter() {
@@ -401,6 +409,30 @@ where
401409
}
402410
}
403411

412+
let changes_versioning_size = if self.config.enable_metrics {
413+
self.current_batch
414+
.lock()
415+
.size_in_bytes()
416+
.saturating_sub(changes_size)
417+
} else {
418+
0
419+
};
420+
421+
if self.config.enable_metrics {
422+
match self
423+
.change_history_sizes
424+
.entry(self.get_change_id().expect(CHANGE_ID_DESER_ERROR))
425+
{
426+
std::collections::btree_map::Entry::Vacant(entry) => {
427+
entry.insert((changes_size, changes_versioning_size));
428+
}
429+
std::collections::btree_map::Entry::Occupied(mut entry) => {
430+
entry.get_mut().0 += changes_size;
431+
entry.get_mut().1 += changes_versioning_size;
432+
}
433+
}
434+
}
435+
404436
if let Some(change_id) = change_id {
405437
self.set_change_id_to_batch(change_id);
406438
}
@@ -447,10 +479,12 @@ where
447479
if reset_history {
448480
self.change_history.clear();
449481
self.change_history_versioning.clear();
482+
self.change_history_sizes.clear();
450483
}
451484

452485
while self.change_history.len() > self.config.max_history_length {
453486
self.change_history.pop_first();
487+
self.change_history_sizes.pop_first();
454488
}
455489

456490
while self.change_history_versioning.len() > self.config.max_history_length {
@@ -614,6 +648,7 @@ impl RawMassaDB<Slot, SlotSerializer, SlotDeserializer> {
614648
config,
615649
change_history: BTreeMap::new(),
616650
change_history_versioning: BTreeMap::new(),
651+
change_history_sizes: BTreeMap::new(),
617652
change_id_serializer: SlotSerializer::new(),
618653
change_id_deserializer,
619654
current_batch,
@@ -851,6 +886,16 @@ impl MassaDBController for RawMassaDB<Slot, SlotSerializer, SlotDeserializer> {
851886
self.get_versioning_batch_to_stream(last_versioning_step, last_change_id)
852887
}
853888

889+
fn get_change_history_sizes(&self) -> (usize, usize) {
890+
let mut change_history_size = 0;
891+
let mut change_history_versioning_size = 0;
892+
for (state_size, versioning_size) in self.change_history_sizes.values() {
893+
change_history_size += state_size;
894+
change_history_versioning_size += versioning_size;
895+
}
896+
(change_history_size, change_history_versioning_size)
897+
}
898+
854899
#[cfg(feature = "test-exports")]
855900
fn get_entire_database(&self) -> Vec<BTreeMap<Vec<u8>, Vec<u8>>> {
856901
let handle_state = self.db.cf_handle(STATE_CF).expect(CF_ERROR);
@@ -933,6 +978,7 @@ mod test {
933978
max_versioning_elements_size: 100,
934979
thread_count: THREAD_COUNT,
935980
max_ledger_backups: 10,
981+
enable_metrics: true,
936982
};
937983
let mut db_opts = MassaDB::default_db_opts();
938984
// Additional checks (only for testing)
@@ -963,6 +1009,7 @@ mod test {
9631009
max_versioning_elements_size: 100,
9641010
thread_count: THREAD_COUNT,
9651011
max_ledger_backups: 10,
1012+
enable_metrics: true,
9661013
};
9671014
let mut db_opts = MassaDB::default_db_opts();
9681015
// Additional checks (only for testing)
@@ -1047,6 +1094,7 @@ mod test {
10471094
max_versioning_elements_size: 100,
10481095
thread_count: THREAD_COUNT,
10491096
max_ledger_backups: 10,
1097+
enable_metrics: true,
10501098
};
10511099
let mut db_opts = MassaDB::default_db_opts();
10521100
// Additional checks (only for testing)
@@ -1132,6 +1180,7 @@ mod test {
11321180
max_versioning_elements_size: 100,
11331181
thread_count: THREAD_COUNT,
11341182
max_ledger_backups: 10,
1183+
enable_metrics: true,
11351184
};
11361185
let mut db_opts = MassaDB::default_db_opts();
11371186
// Additional checks (only for testing)
@@ -1181,6 +1230,7 @@ mod test {
11811230
max_versioning_elements_size: 100,
11821231
thread_count: THREAD_COUNT,
11831232
max_ledger_backups: 10,
1233+
enable_metrics: true,
11841234
};
11851235
let mut db_backup_1_opts = MassaDB::default_db_opts();
11861236
db_backup_1_opts.create_if_missing(false);
@@ -1205,6 +1255,7 @@ mod test {
12051255
max_versioning_elements_size: 100,
12061256
thread_count: THREAD_COUNT,
12071257
max_ledger_backups: 10,
1258+
enable_metrics: true,
12081259
};
12091260
let mut db_backup_2_opts = MassaDB::default_db_opts();
12101261
db_backup_2_opts.create_if_missing(false);
@@ -1238,6 +1289,7 @@ mod test {
12381289
max_versioning_elements_size: 100,
12391290
thread_count: THREAD_COUNT,
12401291
max_ledger_backups: 10,
1292+
enable_metrics: true,
12411293
};
12421294
let mut db_opts = MassaDB::default_db_opts();
12431295
// Additional checks (only for testing)
@@ -1285,6 +1337,7 @@ mod test {
12851337
max_versioning_elements_size: 100,
12861338
thread_count: THREAD_COUNT,
12871339
max_ledger_backups: 10,
1340+
enable_metrics: true,
12881341
};
12891342
// let db_backup_2_opts = MassaDB::default_db_opts();
12901343

@@ -1334,6 +1387,7 @@ mod test {
13341387
max_versioning_elements_size: 100,
13351388
thread_count: THREAD_COUNT,
13361389
max_ledger_backups: 10,
1390+
enable_metrics: true,
13371391
};
13381392
let mut db_opts = MassaDB::default_db_opts();
13391393
// Additional checks (only for testing)
@@ -1427,6 +1481,7 @@ mod test {
14271481
max_versioning_elements_size: 100,
14281482
thread_count: THREAD_COUNT,
14291483
max_ledger_backups: 10,
1484+
enable_metrics: true,
14301485
};
14311486
let mut db_opts = MassaDB::default_db_opts();
14321487
// Additional checks (only for testing)
@@ -1515,6 +1570,7 @@ mod test {
15151570
max_versioning_elements_size: 10,
15161571
thread_count: THREAD_COUNT,
15171572
max_ledger_backups: 10,
1573+
enable_metrics: true,
15181574
};
15191575
let mut db_opts = MassaDB::default_db_opts();
15201576
// Additional checks (only for testing)
@@ -1600,6 +1656,7 @@ mod test {
16001656
max_versioning_elements_size: 20,
16011657
thread_count: THREAD_COUNT,
16021658
max_ledger_backups: 10,
1659+
enable_metrics: true,
16031660
};
16041661
let mut db_opts = MassaDB::default_db_opts();
16051662
// Additional checks (only for testing)
@@ -1712,6 +1769,7 @@ mod test {
17121769
max_versioning_elements_size: 20,
17131770
thread_count: THREAD_COUNT,
17141771
max_ledger_backups: 10,
1772+
enable_metrics: true,
17151773
};
17161774

17171775
let slot_1 = Slot::new(1, 0);

‎massa-deferred-calls/src/tests/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ fn call_registry_apply_changes() {
2323
max_versioning_elements_size: 100,
2424
thread_count: THREAD_COUNT,
2525
max_ledger_backups: 100,
26+
enable_metrics: true,
2627
};
2728
let call_id_serializer = DeferredCallIdSerializer::new();
2829
let db: ShareableMassaDBController = Arc::new(RwLock::new(
@@ -75,6 +76,7 @@ fn call_registry_get_slot_calls() {
7576
max_versioning_elements_size: 100,
7677
thread_count: THREAD_COUNT,
7778
max_ledger_backups: 100,
79+
enable_metrics: true,
7880
};
7981
let call_id_serializer = DeferredCallIdSerializer::new();
8082
let db: ShareableMassaDBController = Arc::new(RwLock::new(

‎massa-executed-ops/src/executed_denunciations.rs

+1
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@ mod test {
250250
max_versioning_elements_size: 100,
251251
max_ledger_backups: 10,
252252
thread_count: THREAD_COUNT,
253+
enable_metrics: true,
253254
};
254255
let db = Arc::new(RwLock::new(
255256
Box::new(MassaDB::new(db_config.clone())) as Box<(dyn MassaDBController + 'static)>

‎massa-executed-ops/src/executed_ops.rs

+3
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ mod test {
302302
max_versioning_elements_size: 100,
303303
thread_count: THREAD_COUNT,
304304
max_ledger_backups: 10,
305+
enable_metrics: true,
305306
};
306307
let db = Arc::new(RwLock::new(
307308
Box::new(MassaDB::new(db_config.clone())) as Box<(dyn MassaDBController + 'static)>
@@ -362,6 +363,7 @@ mod test {
362363
max_versioning_elements_size: 100,
363364
thread_count,
364365
max_ledger_backups: 10,
366+
enable_metrics: true,
365367
};
366368
let db_c_config = MassaDBConfig {
367369
path: tempdir_c.path().to_path_buf(),
@@ -370,6 +372,7 @@ mod test {
370372
max_versioning_elements_size: 100,
371373
thread_count,
372374
max_ledger_backups: 10,
375+
enable_metrics: true,
373376
};
374377

375378
let db_a = Arc::new(RwLock::new(

‎massa-execution-worker/src/interface_impl.rs

+1
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ impl InterfaceImpl {
123123
max_versioning_elements_size: 100_000,
124124
thread_count: THREAD_COUNT,
125125
max_ledger_backups: 10,
126+
enable_metrics: true,
126127
};
127128

128129
let db = Arc::new(RwLock::new(

‎massa-execution-worker/src/speculative_deferred_calls.rs

+1
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,7 @@ mod tests {
532532
max_versioning_elements_size: 100_000,
533533
thread_count: THREAD_COUNT,
534534
max_ledger_backups: 10,
535+
enable_metrics: true,
535536
};
536537

537538
let db = Arc::new(RwLock::new(

‎massa-execution-worker/src/tests/universe.rs

+1
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ impl ExecutionForeignControllers {
6969
max_versioning_elements_size: 100_000,
7070
thread_count: THREAD_COUNT,
7171
max_ledger_backups: 10,
72+
enable_metrics: true,
7273
};
7374

7475
let db = Arc::new(RwLock::new(

‎massa-final-state/src/final_state.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1074,6 +1074,7 @@ mod test {
10741074
max_versioning_elements_size: 100,
10751075
thread_count: THREAD_COUNT,
10761076
max_ledger_backups: 10,
1077+
enable_metrics: true,
10771078
};
10781079
let db = Arc::new(RwLock::new(
10791080
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>

‎massa-final-state/src/tests/scenarios.rs

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ fn create_final_state(temp_dir: &TempDir, reset_final_state: bool) -> Arc<RwLock
4141
max_versioning_elements_size: 100_000,
4242
thread_count,
4343
max_ledger_backups: 10,
44+
enable_metrics: true,
4445
};
4546
let db = Arc::new(RwLock::new(
4647
Box::new(MassaDB::new(db_config)) as Box<(dyn MassaDBController + 'static)>

‎massa-ledger-worker/src/ledger_db.rs

+1
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,7 @@ mod tests {
664664
max_versioning_elements_size: 100_000,
665665
max_ledger_backups: 10,
666666
thread_count: 32,
667+
enable_metrics: true,
667668
};
668669

669670
let db = Arc::new(RwLock::new(

0 commit comments

Comments
 (0)