reth_cli_commands/download/
verify.rs

1use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress};
2use blake3::Hasher;
3use eyre::Result;
4use rayon::prelude::*;
5use reth_fs_util::{self as fs, FsPathError};
6use std::{
7    io::{ErrorKind, Read},
8    mem,
9    path::Path,
10};
11
12/// Read size per hash update. Large enough for `update_rayon` to split across threads.
13const HASH_BUFFER_SIZE: usize = 4 * 1024 * 1024;
14
15/// Verifies and cleans up extracted output files in one target directory.
16pub(crate) struct OutputVerifier<'a> {
17    /// Directory containing the output files declared by the manifest.
18    target_dir: &'a Path,
19    /// Custom location of static file outputs.
20    static_files_dir: Option<&'a Path>,
21}
22
23impl<'a> OutputVerifier<'a> {
24    /// Creates a verifier for one extraction target directory.
25    pub(crate) const fn new(target_dir: &'a Path, static_files_dir: Option<&'a Path>) -> Self {
26        Self { target_dir, static_files_dir }
27    }
28
29    /// Returns `true` only when every declared output file exists and matches size and BLAKE3.
30    /// Returns `false` if any file is missing, mismatched, or no outputs were declared, and an
31    /// error if a file exists but cannot be inspected or read.
32    pub(crate) fn verify(&self, output_files: &[OutputFileChecksum]) -> Result<bool> {
33        self.verify_with_progress(output_files, None)
34    }
35
36    /// Returns `true` only when every declared output file exists and matches size and BLAKE3,
37    /// updating the optional verification progress as file bytes are hashed.
38    ///
39    /// Sizes are checked before any file is hashed, then files are hashed in parallel.
40    pub(crate) fn verify_with_progress(
41        &self,
42        output_files: &[OutputFileChecksum],
43        progress: Option<&ArchiveVerificationProgress<'_>>,
44    ) -> Result<bool> {
45        if output_files.is_empty() {
46            return Ok(false);
47        }
48
49        let mut paths = Vec::with_capacity(output_files.len());
50        for expected in output_files {
51            let output_path = self.output_path(&expected.path);
52            match fs::metadata(&output_path) {
53                Ok(meta) if meta.len() == expected.size => paths.push(output_path),
54                Ok(_) => return Ok(false),
55                Err(FsPathError::Metadata { source, .. })
56                    if source.kind() == ErrorKind::NotFound =>
57                {
58                    return Ok(false)
59                }
60                Err(error) => return Err(error.into()),
61            }
62        }
63
64        output_files
65            .par_iter()
66            .zip(paths)
67            .map(|(expected, path)| {
68                Self::file_blake3_hex(&path, expected.size, progress)
69                    .map(|actual| actual.eq_ignore_ascii_case(&expected.blake3))
70            })
71            .find_any(|verified| !matches!(verified, Ok(true)))
72            .unwrap_or(Ok(true))
73    }
74
75    /// Removes any declared output files so a fresh archive attempt can restart cleanly.
76    pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) {
77        for output in output_files {
78            let _ = fs::remove_file(self.output_path(&output.path));
79        }
80    }
81
82    /// Resolves archive paths consistently for verification and retry cleanup.
83    pub(crate) fn output_path(&self, path: &str) -> std::path::PathBuf {
84        if let Some(static_files_dir) = self.static_files_dir &&
85            let Some(relative_path) = super::extract::static_file_relative_path(Path::new(path))
86        {
87            return static_files_dir.join(relative_path)
88        }
89        self.target_dir.join(path)
90    }
91
92    /// Computes the hex-encoded BLAKE3 checksum for one plain output file.
93    ///
94    /// The next buffer is read while the current one is hashed, so disk reads overlap hashing
95    /// instead of alternating with it.
96    fn file_blake3_hex(
97        path: &Path,
98        size: u64,
99        progress: Option<&ArchiveVerificationProgress<'_>>,
100    ) -> Result<String> {
101        let mut file = fs::open(path)?;
102        let mut hasher = Hasher::new();
103        let buf_len = size.min(HASH_BUFFER_SIZE as u64) as usize;
104        let mut current = vec![0_u8; buf_len];
105        let mut next = vec![0_u8; buf_len];
106
107        let mut filled = read_full(&mut file, &mut current)?;
108        while filled > 0 {
109            let (_, read) = rayon::join(
110                || {
111                    hasher.update_rayon(&current[..filled]);
112                },
113                || read_full(&mut file, &mut next),
114            );
115            if let Some(progress) = progress {
116                progress.record_verified(filled as u64);
117            }
118            filled = read?;
119            mem::swap(&mut current, &mut next);
120        }
121
122        Ok(hasher.finalize().to_hex().to_string())
123    }
124}
125
126/// Reads until `buf` is full or the reader is exhausted, returning the number of bytes read.
127///
128/// Full buffers keep every `update_rayon` call aligned, so BLAKE3 can hash it as parallel
129/// subtrees.
130fn read_full(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
131    let mut filled = 0;
132    while filled < buf.len() {
133        match reader.read(&mut buf[filled..]) {
134            Ok(0) => break,
135            Ok(n) => filled += n,
136            Err(error) if error.kind() == ErrorKind::Interrupted => {}
137            Err(error) => return Err(error),
138        }
139    }
140    Ok(filled)
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146
147    fn checksum(path: &str, data: &[u8]) -> OutputFileChecksum {
148        OutputFileChecksum {
149            path: path.into(),
150            size: data.len() as u64,
151            blake3: blake3::hash(data).to_hex().to_string(),
152        }
153    }
154
155    #[test]
156    fn verify_detects_each_kind_of_output_mismatch() {
157        let dir = tempfile::tempdir().unwrap();
158        let large: Vec<u8> = (0..2 * HASH_BUFFER_SIZE + 123).map(|i| (i % 251) as u8).collect();
159        let small = b"small".to_vec();
160        fs::write(dir.path().join("large"), &large).unwrap();
161        fs::write(dir.path().join("small"), &small).unwrap();
162        let outputs = vec![checksum("large", &large), checksum("small", &small)];
163        let verifier = OutputVerifier::new(dir.path(), None);
164
165        assert!(verifier.verify(&outputs).unwrap(), "matching multi-buffer outputs verify");
166
167        let mut uppercase = outputs.clone();
168        uppercase[0].blake3 = uppercase[0].blake3.to_ascii_uppercase();
169        assert!(verifier.verify(&uppercase).unwrap(), "checksums compare case-insensitively");
170
171        let mut flipped = large.clone();
172        flipped[HASH_BUFFER_SIZE + 7] ^= 1;
173        fs::write(dir.path().join("large"), &flipped).unwrap();
174        assert!(!verifier.verify(&outputs).unwrap(), "one flipped byte fails verification");
175
176        fs::write(dir.path().join("large"), &large[1..]).unwrap();
177        assert!(!verifier.verify(&outputs).unwrap(), "size mismatch fails verification");
178
179        fs::remove_file(dir.path().join("large")).unwrap();
180        assert!(!verifier.verify(&outputs).unwrap(), "missing file fails verification");
181
182        assert!(!verifier.verify(&[]).unwrap(), "archives without outputs never verify");
183    }
184
185    #[cfg(unix)]
186    #[test]
187    fn verify_returns_metadata_errors_other_than_not_found() {
188        let dir = tempfile::tempdir().unwrap();
189        fs::write(dir.path().join("file"), b"data").unwrap();
190        let verifier = OutputVerifier::new(dir.path(), None);
191
192        assert!(!verifier.verify(&[checksum("missing", b"data")]).unwrap());
193        assert!(
194            verifier.verify(&[checksum("file/child", b"data")]).is_err(),
195            "a path through a regular file is an error, not a missing output"
196        );
197    }
198
199    #[test]
200    fn custom_static_files_verification_and_cleanup() {
201        let datadir = tempfile::tempdir().unwrap();
202        let static_dir = tempfile::tempdir().unwrap();
203        let default_file = datadir.path().join("static_files/headers");
204        fs::create_dir_all(default_file.parent().unwrap()).unwrap();
205        fs::write(&default_file, b"headers").unwrap();
206        let outputs = [OutputFileChecksum {
207            path: "./static_files/headers".into(),
208            size: 7,
209            blake3: blake3::hash(b"headers").to_hex().to_string(),
210        }];
211        let verifier = OutputVerifier::new(datadir.path(), Some(static_dir.path()));
212        assert!(!verifier.verify(&outputs).unwrap());
213        let custom_file = static_dir.path().join("headers");
214        fs::write(&custom_file, b"headers").unwrap();
215        assert!(verifier.verify(&outputs).unwrap());
216        verifier.cleanup(&outputs);
217        assert!(!custom_file.exists());
218        assert_eq!(fs::read(default_file).unwrap(), b"headers");
219    }
220}
reth_cli_commands/download/verify.rs

reth_cli_commands/download/
verify.rs
