reth_cli_commands/download/
prune.rs

1//! Removes snapshot-managed data that the selected download plan does not list.
2
3use super::{
4    managed_datadir_path, planning::PlannedArchive, verify::OutputVerifier, MANAGED_DATADIR_PATHS,
5};
6use eyre::{Result, WrapErr};
7use reth_db::lockfile::StorageLock;
8use reth_fs_util as fs;
9use std::{
10    collections::HashSet,
11    path::{Path, PathBuf},
12};
13use tracing::info;
14
15/// Removes every file under the paths that `--force` clears (`db`, `rocksdb`, `static_files`,
16/// and `reth.toml`) that no planned archive lists as an output.
17///
18/// Listed files are kept so the startup integrity check can reuse the ones that still verify.
19/// Once the download completes, those paths hold exactly the snapshot's files, with no leftovers
20/// such as blocks or history indices from a chain the node extended past the snapshot block.
21///
22/// This relies on each archive's `output_files` listing every file the archive extracts. A file
23/// an archive extracts without listing it is removed here and not restored, because the archive
24/// is reused once its listed outputs verify.
25///
26/// A managed path that is itself a symlink is followed, because operators link whole data
27/// directories to other disks. Symlinks below it are removed rather than followed, so pruning
28/// never deletes files outside the data dir. Nested directories left empty are removed; the
29/// managed directories themselves are kept, since they may be mount points.
30///
31/// Fails if a running node holds the database lock, since pruning would otherwise delete its
32/// lock and files from under it.
33pub(crate) fn prune_unlisted_outputs(
34    planned: &[PlannedArchive],
35    target_dir: &Path,
36    static_files_dir: Option<&Path>,
37) -> Result<()> {
38    let db_dir = target_dir.join("db");
39    if db_dir.is_dir() {
40        drop(StorageLock::try_acquire(&db_dir).wrap_err("refusing to prune a data dir in use")?);
41    }
42
43    let verifier = OutputVerifier::new(target_dir, static_files_dir);
44    let listed = planned
45        .iter()
46        .flat_map(|planned| &planned.archive.output_files)
47        .map(|output| verifier.output_path(&output.path))
48        .collect::<HashSet<_>>();
49
50    let mut removed = 0;
51    for entry in MANAGED_DATADIR_PATHS {
52        let root = managed_datadir_path(entry, target_dir, static_files_dir);
53        if !root.try_exists()? {
54            continue
55        }
56
57        if fs::metadata(&root)?.is_dir() {
58            removed += prune_dir(&root, &listed)?;
59        } else if !listed.contains(&root) {
60            fs::remove_file(&root)?;
61            removed += 1;
62        }
63    }
64
65    info!(target: "reth::cli", removed, "Removed existing files not listed in the snapshot plan");
66    Ok(())
67}
68
69/// Removes unlisted entries below `dir` without following symlinks, and returns how many files
70/// or links it removed. Removing a symlink leaves its target intact.
71fn prune_dir(dir: &Path, listed: &HashSet<PathBuf>) -> Result<usize> {
72    let mut removed = 0;
73    for entry in fs::read_dir(dir)? {
74        let entry = entry?;
75        let path = entry.path();
76        // Unlike `fs::metadata`, this does not follow symlinks.
77        let file_type = entry
78            .file_type()
79            .wrap_err_with(|| format!("failed to read file type of {}", path.display()))?;
80        if file_type.is_dir() {
81            removed += prune_dir(&path, listed)?;
82            remove_dir_if_empty(&path)?;
83            continue
84        }
85        if listed.contains(&path) {
86            continue
87        }
88
89        // Windows removes a symlink to a directory with `remove_dir`, not `remove_file`.
90        #[cfg(windows)]
91        if std::os::windows::fs::FileTypeExt::is_symlink_dir(&file_type) {
92            std::fs::remove_dir(&path)
93                .wrap_err_with(|| format!("failed to remove {}", path.display()))?;
94            removed += 1;
95            continue
96        }
97        fs::remove_file(&path)?;
98        removed += 1;
99    }
100    Ok(removed)
101}
102
103/// Removes `dir` if pruning left it empty.
104fn remove_dir_if_empty(dir: &Path) -> Result<()> {
105    if fs::read_dir(dir)?.next().is_none() {
106        std::fs::remove_dir(dir)
107            .wrap_err_with(|| format!("failed to remove empty directory {}", dir.display()))?;
108    }
109    Ok(())
110}
111
112#[cfg(test)]
113mod tests {
114    use super::*;
115    use crate::download::manifest::{OutputFileChecksum, SnapshotArchive, SnapshotComponentType};
116    use tempfile::tempdir;
117
118    fn planned_with_outputs(paths: &[&str]) -> Vec<PlannedArchive> {
119        vec![PlannedArchive {
120            ty: SnapshotComponentType::State,
121            component: "State".to_string(),
122            archive: SnapshotArchive {
123                url: "https://example.com/state.tar.zst".to_string(),
124                file_name: "state.tar.zst".to_string(),
125                size: 1,
126                blake3: None,
127                output_files: paths
128                    .iter()
129                    .map(|path| OutputFileChecksum {
130                        path: (*path).to_string(),
131                        size: 1,
132                        blake3: String::new(),
133                    })
134                    .collect(),
135            },
136        }]
137    }
138
139    fn write_file(path: &Path) {
140        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
141        std::fs::write(path, b"data").unwrap();
142    }
143
144    #[test]
145    fn prune_removes_unlisted_files_and_keeps_listed_ones() {
146        let dir = tempdir().unwrap();
147        let target = dir.path();
148        let listed = ["db/mdbx.dat", "static_files/static_file_headers_0_499999"];
149        let leftovers = [
150            "db/stale.dat",
151            "rocksdb/000042.sst",
152            "static_files/static_file_headers_500000_999999",
153            "reth.toml",
154        ];
155        for path in listed.iter().chain(&leftovers).chain(&["known-peers.json"]) {
156            write_file(&target.join(path));
157        }
158
159        prune_unlisted_outputs(&planned_with_outputs(&listed), target, None).unwrap();
160
161        assert!(listed.iter().all(|path| target.join(path).exists()));
162        assert!(leftovers.iter().all(|path| !target.join(path).exists()));
163        assert!(target.join("known-peers.json").exists());
164    }
165
166    #[test]
167    fn prune_removes_nested_directories_it_empties_and_keeps_managed_ones() {
168        let dir = tempdir().unwrap();
169        write_file(&dir.path().join("rocksdb/nested/000042.sst"));
170
171        prune_unlisted_outputs(&planned_with_outputs(&["db/mdbx.dat"]), dir.path(), None).unwrap();
172
173        assert!(!dir.path().join("rocksdb/nested").exists());
174        assert!(dir.path().join("rocksdb").is_dir());
175    }
176
177    #[test]
178    fn prune_resolves_static_files_in_a_custom_directory() {
179        let dir = tempdir().unwrap();
180        let target = dir.path().join("datadir");
181        let static_files = dir.path().join("custom-static-files");
182        write_file(&static_files.join("static_file_headers_0_499999"));
183        write_file(&static_files.join("static_file_headers_500000_999999"));
184
185        prune_unlisted_outputs(
186            &planned_with_outputs(&["static_files/static_file_headers_0_499999"]),
187            &target,
188            Some(&static_files),
189        )
190        .unwrap();
191
192        assert!(static_files.join("static_file_headers_0_499999").exists());
193        assert!(!static_files.join("static_file_headers_500000_999999").exists());
194    }
195
196    #[test]
197    fn prune_skips_missing_managed_paths() {
198        let dir = tempdir().unwrap();
199
200        prune_unlisted_outputs(&planned_with_outputs(&["db/mdbx.dat"]), dir.path(), None).unwrap();
201
202        assert!(!dir.path().join("db").exists());
203    }
204
205    #[cfg(unix)]
206    #[test]
207    fn prune_removes_nested_symlinks_without_following_them() {
208        let dir = tempdir().unwrap();
209        let target = dir.path().join("datadir");
210        let outside = dir.path().join("outside");
211        write_file(&outside.join("unrelated.dat"));
212        write_file(&target.join("static_files/static_file_headers_0_499999"));
213        std::os::unix::fs::symlink(&outside, target.join("static_files/linked")).unwrap();
214
215        prune_unlisted_outputs(
216            &planned_with_outputs(&["static_files/static_file_headers_0_499999"]),
217            &target,
218            None,
219        )
220        .unwrap();
221
222        assert!(std::fs::symlink_metadata(target.join("static_files/linked")).is_err());
223        assert!(outside.join("unrelated.dat").exists());
224    }
225
226    #[cfg(unix)]
227    #[test]
228    fn prune_follows_a_symlinked_managed_directory() {
229        let dir = tempdir().unwrap();
230        let target = dir.path().join("datadir");
231        let other_disk = dir.path().join("other-disk");
232        write_file(&other_disk.join("static_file_headers_0_499999"));
233        write_file(&other_disk.join("static_file_headers_500000_999999"));
234        std::fs::create_dir_all(&target).unwrap();
235        std::os::unix::fs::symlink(&other_disk, target.join("static_files")).unwrap();
236
237        prune_unlisted_outputs(
238            &planned_with_outputs(&["static_files/static_file_headers_0_499999"]),
239            &target,
240            None,
241        )
242        .unwrap();
243
244        assert!(std::fs::symlink_metadata(target.join("static_files")).unwrap().is_symlink());
245        assert!(other_disk.join("static_file_headers_0_499999").exists());
246        assert!(!other_disk.join("static_file_headers_500000_999999").exists());
247    }
248}
reth_cli_commands/download/prune.rs

reth_cli_commands/download/
prune.rs
