1use crate::{
2 prefix_set::PrefixSet,
3 trie_cursor::{subnode::SubNodePosition, CursorSubNode, TrieCursor},
4 BranchNodeCompact, Nibbles,
5};
6use alloy_primitives::{map::HashSet, B256};
7use alloy_trie::{proof::AddedRemovedKeys, TrieMask};
8use reth_storage_errors::db::DatabaseError;
9use tracing::{instrument, trace};
10
11#[cfg(feature = "metrics")]
12use crate::metrics::WalkerMetrics;
13
14#[derive(Debug)]
18pub struct TrieWalker<C, K = AddedRemovedKeys> {
19 pub cursor: C,
21 pub stack: Vec<CursorSubNode>,
23 pub can_skip_current_node: bool,
27 pub changes: PrefixSet,
29 walk_all_changed_branch_children: bool,
32 removed_keys: Option<HashSet<Nibbles>>,
34 added_removed_keys: Option<K>,
38 #[cfg(feature = "metrics")]
39 metrics: WalkerMetrics,
41}
42
43impl<C: TrieCursor, K: AsRef<AddedRemovedKeys>> TrieWalker<C, K> {
44 pub fn state_trie_from_stack(cursor: C, stack: Vec<CursorSubNode>, changes: PrefixSet) -> Self {
46 Self::from_stack(
47 cursor,
48 stack,
49 changes,
50 #[cfg(feature = "metrics")]
51 crate::TrieType::State,
52 )
53 }
54
55 pub fn storage_trie_from_stack(
57 cursor: C,
58 stack: Vec<CursorSubNode>,
59 changes: PrefixSet,
60 ) -> Self {
61 Self::from_stack(
62 cursor,
63 stack,
64 changes,
65 #[cfg(feature = "metrics")]
66 crate::TrieType::Storage,
67 )
68 }
69
70 fn from_stack(
72 cursor: C,
73 stack: Vec<CursorSubNode>,
74 changes: PrefixSet,
75 #[cfg(feature = "metrics")] trie_type: crate::TrieType,
76 ) -> Self {
77 let mut this = Self {
78 cursor,
79 changes,
80 stack,
81 can_skip_current_node: false,
82 walk_all_changed_branch_children: false,
83 removed_keys: None,
84 added_removed_keys: None,
85 #[cfg(feature = "metrics")]
86 metrics: WalkerMetrics::new(trie_type),
87 };
88 this.update_skip_node();
89 this
90 }
91
92 pub fn with_deletions_retained(mut self, retained: bool) -> Self {
94 if retained {
95 self.removed_keys = Some(HashSet::default());
96 }
97 self
98 }
99
100 pub fn with_added_removed_keys<K2>(self, added_removed_keys: Option<K2>) -> TrieWalker<C, K2> {
103 TrieWalker {
104 cursor: self.cursor,
105 stack: self.stack,
106 can_skip_current_node: self.can_skip_current_node,
107 changes: self.changes,
108 walk_all_changed_branch_children: self.walk_all_changed_branch_children,
109 removed_keys: self.removed_keys,
110 added_removed_keys,
111 #[cfg(feature = "metrics")]
112 metrics: self.metrics,
113 }
114 }
115
116 pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self {
122 self.walk_all_changed_branch_children = enabled;
123 self
124 }
125
126 pub fn split(mut self) -> (Vec<CursorSubNode>, HashSet<Nibbles>) {
128 let keys = self.take_removed_keys();
129 (self.stack, keys)
130 }
131
132 pub fn take_removed_keys(&mut self) -> HashSet<Nibbles> {
134 self.removed_keys.take().unwrap_or_default()
135 }
136
137 pub fn print_stack(&self) {
139 println!("====================== STACK ======================");
140 for node in &self.stack {
141 println!("{node:?}");
142 }
143 println!("====================== END STACK ======================\n");
144 }
145
146 pub fn removed_keys_len(&self) -> usize {
148 self.removed_keys.as_ref().map_or(0, |u| u.len())
149 }
150
151 pub fn key(&self) -> Option<&Nibbles> {
153 self.stack.last().map(|n| n.full_key())
154 }
155
156 pub fn hash(&self) -> Option<B256> {
158 self.stack.last().and_then(|n| n.hash())
159 }
160
161 pub fn maybe_hash(&self) -> Option<B256> {
166 self.stack.last().and_then(|n| n.maybe_hash())
167 }
168
169 pub fn children_are_in_trie(&self) -> bool {
171 self.stack.last().is_some_and(|n| n.tree_flag())
172 }
173
174 #[instrument(level = "trace", skip(self), ret)]
176 pub fn next_unprocessed_key(&self) -> Option<(B256, Nibbles)> {
177 self.key()
178 .and_then(|key| if self.can_skip_current_node { key.increment() } else { Some(*key) })
179 .map(|key| (B256::right_padding_from(&key.pack()), key))
180 }
181
182 fn update_skip_node(&mut self) {
184 let old = self.can_skip_current_node;
185 self.can_skip_current_node = self.stack.last().is_some_and(|node| {
186 let key_is_only_nonremoved_child =
191 self.added_removed_keys.as_ref().is_some_and(|added_removed_keys| {
192 node.full_key_is_only_nonremoved_child(added_removed_keys.as_ref())
193 });
194
195 trace!(
196 target: "trie::walker",
197 ?key_is_only_nonremoved_child,
198 full_key=?node.full_key(),
199 "Checked for only non-removed child",
200 );
201
202 let branch_path_matches_prefix_set = self.walk_all_changed_branch_children &&
203 node.position().is_child() &&
204 self.changes.contains(&node.key);
205
206 !self.changes.contains(node.full_key()) &&
207 !branch_path_matches_prefix_set &&
208 node.hash_flag() &&
209 !key_is_only_nonremoved_child
210 });
211 trace!(
212 target: "trie::walker",
213 old,
214 new = self.can_skip_current_node,
215 last = ?self.stack.last(),
216 "updated skip node flag"
217 );
218 }
219
220 pub fn state_trie(cursor: C, changes: PrefixSet) -> Result<Self, DatabaseError> {
222 Self::new(
223 cursor,
224 changes,
225 #[cfg(feature = "metrics")]
226 crate::TrieType::State,
227 )
228 }
229
230 pub fn storage_trie(cursor: C, changes: PrefixSet) -> Result<Self, DatabaseError> {
232 Self::new(
233 cursor,
234 changes,
235 #[cfg(feature = "metrics")]
236 crate::TrieType::Storage,
237 )
238 }
239
240 fn new(
242 cursor: C,
243 changes: PrefixSet,
244 #[cfg(feature = "metrics")] trie_type: crate::TrieType,
245 ) -> Result<Self, DatabaseError> {
246 let mut this = Self {
248 cursor,
249 changes,
250 stack: vec![CursorSubNode::default()],
251 can_skip_current_node: false,
252 walk_all_changed_branch_children: false,
253 removed_keys: None,
254 added_removed_keys: Default::default(),
255 #[cfg(feature = "metrics")]
256 metrics: WalkerMetrics::new(trie_type),
257 };
258
259 if let Some((key, value)) = this.node(true)? {
261 this.stack[0] = CursorSubNode::new(key, Some(value));
262 }
263
264 this.update_skip_node();
266 Ok(this)
267 }
268
269 pub fn advance(&mut self) -> Result<(), DatabaseError> {
276 if let Some(last) = self.stack.last() {
277 if !self.can_skip_current_node && self.children_are_in_trie() {
278 trace!(
279 target: "trie::walker",
280 position = ?last.position(),
281 "cannot skip current node and children are in the trie"
282 );
283 match last.position() {
286 SubNodePosition::ParentBranch => self.move_to_next_sibling(true)?,
287 SubNodePosition::Child(_) => self.consume_node()?,
288 }
289 } else {
290 trace!(target: "trie::walker", "can skip current node");
291 self.move_to_next_sibling(false)?;
293 }
294
295 self.update_skip_node();
297 }
298
299 Ok(())
300 }
301
302 fn node(&mut self, exact: bool) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
304 let key = self.key().expect("key must exist");
305 let entry = if exact { self.cursor.seek_exact(*key)? } else { self.cursor.seek(*key)? };
306 #[cfg(feature = "metrics")]
307 self.metrics.inc_branch_nodes_seeked();
308
309 if let Some((_, node)) = &entry {
310 assert!(!node.state_mask.is_empty());
311 }
312
313 Ok(entry)
314 }
315
316 #[instrument(level = "trace", skip(self), ret)]
318 fn consume_node(&mut self) -> Result<(), DatabaseError> {
319 let Some((key, node)) = self.node(false)? else {
320 self.stack.clear();
322 return Ok(())
323 };
324
325 if !key.is_empty() && !self.stack.is_empty() {
329 self.stack[0].set_nibble(key.get_unchecked(0));
330 }
331
332 if let Some(subnode) = self.stack.last() &&
337 !key.starts_with(subnode.full_key())
338 {
339 #[cfg(feature = "metrics")]
340 self.metrics.inc_out_of_order_subnode(1);
341 self.move_to_next_sibling(false)?;
342 return Ok(())
343 }
344
345 self.remove_orphaned_descendants(key, node.tree_mask)?;
346
347 let subnode = CursorSubNode::new(key, Some(node));
349 let position = subnode.position();
350 self.stack.push(subnode);
351 self.update_skip_node();
352
353 if (!self.can_skip_current_node || position.is_child()) &&
356 let Some((keys, key)) = self.removed_keys.as_mut().zip(self.cursor.current()?)
357 {
358 keys.insert(key);
359 }
360
361 Ok(())
362 }
363
364 #[instrument(level = "trace", skip(self), ret)]
366 fn move_to_next_sibling(
367 &mut self,
368 allow_root_to_child_nibble: bool,
369 ) -> Result<(), DatabaseError> {
370 let Some(subnode) = self.stack.last_mut() else { return Ok(()) };
371
372 if subnode.position().is_last_child() ||
375 (subnode.position().is_parent() && !allow_root_to_child_nibble)
376 {
377 self.stack.pop();
378 self.move_to_next_sibling(false)?;
379 return Ok(())
380 }
381
382 subnode.inc_nibble();
383
384 if subnode.node.is_none() {
385 return self.consume_node()
386 }
387
388 loop {
390 let position = subnode.position();
391 if subnode.state_flag() {
392 trace!(target: "trie::walker", ?position, "found next sibling with state");
393 return Ok(())
394 }
395 if position.is_last_child() {
396 trace!(target: "trie::walker", ?position, "checked all siblings");
397 break
398 }
399 subnode.inc_nibble();
400 }
401
402 self.stack.pop();
404 self.move_to_next_sibling(false)?;
405
406 Ok(())
407 }
408
409 fn remove_orphaned_descendants(
419 &mut self,
420 key: Nibbles,
421 tree_mask: TrieMask,
422 ) -> Result<(), DatabaseError> {
423 let missing_children = !tree_mask;
424 if self.walk_all_changed_branch_children &&
425 !missing_children.is_empty() &&
426 let Some(removed_keys) = &mut self.removed_keys &&
427 self.changes.contains(&key)
428 {
429 for nibble in missing_children.iter() {
430 let mut prefix = key;
431 prefix.push(nibble);
432 let mut entry = self.cursor.seek(prefix)?;
433 #[cfg(feature = "metrics")]
434 self.metrics.inc_branch_nodes_seeked();
435 while let Some((path, _)) = entry &&
436 path.starts_with(&prefix)
437 {
438 removed_keys.insert(path);
439 entry = self.cursor.next()?;
440 }
441 }
442 self.cursor.seek_exact(key)?;
443 #[cfg(feature = "metrics")]
444 self.metrics.inc_branch_nodes_seeked();
445 }
446 Ok(())
447 }
448}
449
450#[cfg(test)]
451mod tests {
452 use super::*;
453 use crate::{
454 mock::KeyVisitType,
455 prefix_set::PrefixSetMut,
456 progress::StorageRootProgress,
457 test_utils::{storage_root_prehashed, TrieTestHarness},
458 trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory},
459 updates::StorageTrieUpdates,
460 StorageRoot,
461 };
462 use alloy_primitives::{map::B256Map, B256, U256};
463 use reth_storage_errors::db::DatabaseErrorInfo;
464 use std::collections::BTreeMap;
465
466 fn branch_node(state_mask: u16, tree_mask: u16, hash_mask: u16) -> BranchNodeCompact {
467 let hash_count = hash_mask.count_ones() as usize;
468 BranchNodeCompact::new(
469 TrieMask::new(state_mask),
470 TrieMask::new(tree_mask),
471 TrieMask::new(hash_mask),
472 vec![B256::ZERO; hash_count],
473 None,
474 )
475 }
476
477 fn root_branch_node(state_mask: u16, tree_mask: u16, hash_mask: u16) -> BranchNodeCompact {
478 let hash_count = hash_mask.count_ones() as usize;
479 BranchNodeCompact::new(
480 TrieMask::new(state_mask),
481 TrieMask::new(tree_mask),
482 TrieMask::new(hash_mask),
483 vec![B256::ZERO; hash_count],
484 Some(B256::ZERO),
485 )
486 }
487
488 fn walker_for_matching_branch_children_test(
489 walk_all_changed_branch_children: bool,
490 ) -> TrieWalker<crate::trie_cursor::mock::MockTrieCursor> {
491 let trie_nodes = BTreeMap::from([
492 (Nibbles::default(), root_branch_node(1 << 2, 1 << 2, 1 << 2)),
493 (
494 Nibbles::from_nibbles([0x2]),
495 branch_node((1 << 3) | (1 << 4), 0, (1 << 3) | (1 << 4)),
496 ),
497 ]);
498 let factory = MockTrieCursorFactory::new(trie_nodes, B256Map::default());
499
500 let mut prefix_set = PrefixSetMut::default();
501 prefix_set.insert(Nibbles::from_nibbles([0x2, 0x3, 0x1]));
502
503 TrieWalker::state_trie(factory.account_trie_cursor().unwrap(), prefix_set.freeze())
504 .unwrap()
505 .with_walk_all_changed_branch_children(walk_all_changed_branch_children)
506 }
507
508 #[test]
509 fn branch_siblings_remain_skippable_by_default() {
510 let mut walker = walker_for_matching_branch_children_test(false);
511
512 assert_eq!(walker.key().copied(), Some(Nibbles::default()));
513 assert!(!walker.can_skip_current_node);
514
515 walker.advance().unwrap();
516 assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2])));
517 assert!(!walker.can_skip_current_node);
518
519 walker.advance().unwrap();
520 assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x3])));
521 assert_eq!(walker.stack.last().unwrap().position(), SubNodePosition::Child(0x3));
522 assert!(!walker.can_skip_current_node);
523
524 walker.advance().unwrap();
525 assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x4])));
526 assert!(walker.can_skip_current_node);
527 }
528
529 #[test]
530 fn matching_branch_path_can_make_all_children_unskippable() {
531 let mut walker = walker_for_matching_branch_children_test(true);
532
533 walker.advance().unwrap();
534 walker.advance().unwrap();
535 walker.advance().unwrap();
536 assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x4])));
537 assert!(!walker.can_skip_current_node);
538 }
539
540 #[test]
541 fn changed_branch_children_remove_orphans() {
542 for regenerate_branch in [false, true] {
543 let mut storage = BTreeMap::from([
544 (B256::right_padding_from(&[0x3a]), U256::ONE),
545 (B256::right_padding_from(&[0x3a, 0x01]), U256::from(10)),
546 (B256::right_padding_from(&[0x3a, 0x10]), U256::from(7)),
547 (B256::right_padding_from(&[0x3b]), U256::from(2)),
548 (B256::right_padding_from(&[0x3c, 0x47]), U256::from(3)),
549 (B256::right_padding_from(&[0x80]), U256::from(4)),
550 (B256::right_padding_from(&[0x81]), U256::from(5)),
551 (B256::right_padding_from(&[0x80, 0x10]), U256::from(8)),
552 ]);
553 if regenerate_branch {
554 storage.insert(B256::right_padding_from(&[0x3c, 0x48]), U256::from(6));
555 storage.insert(B256::right_padding_from(&[0x3c, 0x47, 0x10]), U256::from(9));
556 }
557 let mut harness = TrieTestHarness::new(storage.clone());
558 let expected_nodes = harness
559 .storage_trie_updates()
560 .storage_nodes
561 .iter()
562 .map(|(key, node)| (*key, node.clone()))
563 .collect::<BTreeMap<_, _>>();
564 let mut stored_nodes = expected_nodes.clone();
565 let parent = Nibbles::from_nibbles([0x3]);
566 stored_nodes.get_mut(&parent).unwrap().tree_mask &=
567 !TrieMask::new((1 << 0xa) | (1 << 0xc));
568
569 let orphan_paths = [
572 Nibbles::from_nibbles([0x3, 0xc]),
573 Nibbles::from_nibbles([0x3, 0xc, 0x4]),
574 Nibbles::from_nibbles([0x3, 0xc, 0x4, 0xf]),
575 Nibbles::from_nibbles([0x3, 0xf, 0x1]),
576 Nibbles::from_nibbles([0x3, 0x1, 0x4]),
577 ];
578 assert!(expected_nodes.contains_key(&Nibbles::from_nibbles([0x3, 0xa])));
581 for path in orphan_paths {
582 let mut stale_node = branch_node(0b11, 0, 0b11);
583 stale_node.hashes = vec![B256::repeat_byte(0xee); 2].into();
584 stored_nodes.insert(path, stale_node);
585 }
586 harness.set_trie_nodes(stored_nodes.clone());
587
588 let prefix_set =
590 PrefixSetMut::from([Nibbles::unpack(B256::right_padding_from(&[0x3a]))]).freeze();
591 for (enabled, threshold) in [(false, u64::MAX), (true, u64::MAX), (true, 1)] {
592 let changes = if threshold == 1 {
595 PrefixSetMut::from([
596 Nibbles::unpack(B256::right_padding_from(&[0x3a])),
597 Nibbles::unpack(B256::right_padding_from(&[0x80])),
598 ])
599 .freeze()
600 } else {
601 prefix_set.clone()
602 };
603 let (root, updates) =
604 storage_root_with_progress(&harness, changes, enabled, threshold);
605
606 let mut resulting_nodes = stored_nodes.clone();
607 for (path, node) in updates.into_sorted().storage_nodes {
608 if let Some(node) = node {
609 resulting_nodes.insert(path, node);
610 } else {
611 resulting_nodes.remove(&path);
612 }
613 }
614 assert_eq!(root, storage_root_prehashed(storage.clone()));
615 if enabled {
616 assert_eq!(resulting_nodes, expected_nodes);
617 } else {
618 assert_ne!(resulting_nodes, expected_nodes);
620 }
621 }
622 }
623 }
624
625 fn storage_root_with_progress(
626 harness: &TrieTestHarness,
627 changes: PrefixSet,
628 enabled: bool,
629 threshold: u64,
630 ) -> (B256, StorageTrieUpdates) {
631 let mut previous_state = None;
632 let mut updates = StorageTrieUpdates::default();
633 let mut checkpoints = 0;
634 loop {
635 let progress = StorageRoot::new_hashed(
636 harness.trie_cursor_factory(),
637 harness.hashed_cursor_factory(),
638 harness.hashed_address(),
639 changes.clone(),
640 #[cfg(feature = "metrics")]
641 crate::metrics::TrieRootMetrics::new(crate::TrieType::Storage),
642 )
643 .with_walk_all_changed_branch_children(enabled)
644 .with_threshold(threshold)
645 .with_intermediate_state(previous_state)
646 .root_with_progress()
647 .unwrap();
648 match progress {
649 StorageRootProgress::Progress(state, _, new_updates) => {
650 updates.extend(new_updates);
651 previous_state = Some(*state);
652 checkpoints += 1;
653 }
654 StorageRootProgress::Complete(root, _, new_updates) => {
655 updates.extend(new_updates);
656 if threshold == 1 {
657 assert!(checkpoints > 1);
658 }
659 return (root, updates)
660 }
661 }
662 }
663 }
664
665 #[test]
666 fn orphan_cleanup_skips_live_subtrees() {
667 let parent = Nibbles::from_nibbles([0x3]);
668 let tree_mask = TrieMask::from_nibble(0xa);
669 let orphans = [
670 Nibbles::from_nibbles([0x3, 0xb]),
671 Nibbles::from_nibbles([0x3, 0xb, 0x5]),
672 Nibbles::from_nibbles([0x3, 0xb, 0x5, 0xf]),
673 ];
674 let mut nodes = BTreeMap::from([
675 (parent, branch_node((1 << 0xa) | (1 << 0xb), tree_mask.get(), 0)),
676 (Nibbles::from_nibbles([0x3, 0xa]), branch_node(0b11, 0, 0)),
677 (Nibbles::from_nibbles([0x4]), branch_node(0b11, 0, 0)),
678 ]);
679 for index in 0..128 {
680 nodes.insert(
681 Nibbles::from_nibbles([0x3, 0xa, index / 16, index % 16]),
682 branch_node(0b11, 0, 0),
683 );
684 }
685 nodes.extend(orphans.into_iter().map(|path| (path, branch_node(0b11, 0, 0))));
686 let factory = MockTrieCursorFactory::new(nodes, B256Map::default());
687 let changes = PrefixSetMut::from([Nibbles::from_nibbles([0x3, 0xa])]).freeze();
688 let mut walker =
689 TrieWalker::<_>::state_trie(factory.account_trie_cursor().unwrap(), changes)
690 .unwrap()
691 .with_walk_all_changed_branch_children(true)
692 .with_deletions_retained(true);
693 walker.cursor.seek_exact(parent).unwrap();
694 factory.visited_account_keys().clear();
695 walker.remove_orphaned_descendants(parent, tree_mask).unwrap();
696
697 assert_eq!(walker.take_removed_keys(), HashSet::from_iter(orphans));
698 assert_eq!(walker.cursor.current().unwrap(), Some(parent));
699 let next_count = factory
700 .visited_account_keys()
701 .iter()
702 .filter(|visit| matches!(visit.visit_type, KeyVisitType::Next))
703 .count();
704 assert!(next_count <= 4, "cleanup walked {next_count} entries for 3 orphan nodes");
705 }
706
707 #[test]
708 fn state_trie_initial_read_timeout_returns_error() {
709 let error =
710 TrieWalker::<_>::state_trie(TimedOutTrieCursor, PrefixSet::default()).unwrap_err();
711 assert_matches::assert_matches!(error, DatabaseError::Read(info) if info == timeout_error());
712 }
713
714 #[test]
715 fn storage_trie_initial_read_timeout_returns_error() {
716 let error =
717 TrieWalker::<_>::storage_trie(TimedOutTrieCursor, PrefixSet::default()).unwrap_err();
718 assert_matches::assert_matches!(error, DatabaseError::Read(info) if info == timeout_error());
719 }
720
721 fn timeout_error() -> DatabaseErrorInfo {
722 DatabaseErrorInfo { message: "read transaction has been timed out".into(), code: -96000 }
723 }
724
725 #[derive(Debug)]
726 struct TimedOutTrieCursor;
727
728 impl TrieCursor for TimedOutTrieCursor {
729 fn seek_exact(
730 &mut self,
731 key: Nibbles,
732 ) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
733 assert_eq!(key, Nibbles::default());
734 Err(DatabaseError::Read(timeout_error()))
735 }
736
737 fn seek(
738 &mut self,
739 _key: Nibbles,
740 ) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
741 unreachable!("initialization only seeks the root exactly")
742 }
743
744 fn next(&mut self) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
745 unreachable!("initialization only seeks the root exactly")
746 }
747
748 fn current(&mut self) -> Result<Option<Nibbles>, DatabaseError> {
749 unreachable!("initialization only seeks the root exactly")
750 }
751
752 fn reset(&mut self) {
753 unreachable!("initialization only seeks the root exactly")
754 }
755 }
756}