[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10,"_11":12},"actionData","errors","root",{"_165":-5,"_168":169,"_170":171},"routes/shell",{"_42":43,"_164":-5,"_165":-5,"_166":167},"routes/repo",{"_34":-7,"_35":36,"_37":38,"_39":40,"_41":32,"_42":43,"_44":32,"_45":32,"_46":32,"_47":32,"_48":32,"_49":32,"_50":51,"_52":53,"_54":32,"_55":-7,"_56":-7,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":65,"_66":67,"_68":69,"_70":71},"routes/repo.blob",{"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":-5,"_26":27,"_28":29,"_29":30,"_31":32,"_33":14},"ref","main","path","crates/tidy_walk/src/lib.rs","oid","ca6f1b67e3a7b6ba184ab52659e788b4af9a443b","size",6944,"commit","12e96b5435e408a25422fcfd87d6fa5474b6e53a","mounts",[],"source","plain",{"_13":14,"_15":16},"kind","text","//! A parallel walk of the files under the paths a user gives. It reads each wanted file, runs the caller's work on\n//! it, and writes back what the work returns. The results come back sorted by path, with every error.\n//!\n//! ```text\n//! paths ─ plan ─→ I/O threads: one queue of Folder, File and Write tasks ─→ workers: work(state, &File)\n//! Folder: list, call `listing`, queue its folders and wanted files │\n//! File: read, then send to the workers │\n//! Write: write in place ◄────────────────────────────────────────────────┘\n//! ```\n//!\n//! Opens, reads and writes only happen on the I/O threads, so their number limits how many files are open at once.\n//! On macOS, opening files stops scaling at a few threads, system-wide, so there 6 I/O threads feed workers on\n//! every core. Elsewhere, opening scales, and the I/O threads run the work themselves (`workers: 0`).\n//!\n//! Every argument, and every file in a walked folder, is worked on at most once:\n//! - Arguments that are the same file or folder, by `(dev, ino)`, count once.\n//! - An argument inside a walked folder is handled under its own spelling, when the walk reaches it, or after the\n//! walk if a skipped folder or a link lies on its way.\n//! - Links found while walking are never followed.\n//! - Files with more than one hard link are set aside and worked on once each, after the walk.\n\nmod plan;\nmod pool;\n\nuse std::ffi::OsStr;\nuse std::path::{Path, PathBuf};\nuse std::sync::atomic::AtomicBool;\n\n/// How to walk.\npub struct Options<'a> {\n /// Whether to work on a file found in a folder, by its folder and name. Files given as paths are always worked\n /// on.\n pub wanted: &'a (dyn Fn(&Path, &OsStr) -> bool + Sync),\n /// Whether to leave out a folder found while walking, by its parent and name.\n pub skip: &'a (dyn Fn(&Path, &OsStr) -> bool + Sync),\n /// Threads that list folders, read files and write them. At least 1.\n pub io: usize,\n /// Threads that run the work. With 0, the I/O threads run it, right after reading each file.\n pub workers: usize,\n /// The most bytes in flight: files read and not yet worked on, and writes not yet done. One file is always let\n /// through.\n pub bytes: usize,\n}\n\nimpl Default for Options<'_> {\n /// Every file, without hidden folders and `node_modules`, with the thread counts that measured best: on macOS,\n /// 6 I/O threads and workers on every core, never more than 9% behind the best shape on corpora of tidy and\n /// mdtidy; elsewhere, I/O threads on every core that also run the work.\n fn default() -> Self {\n let cores = std::thread::available_parallelism().map_or(1, |count| count.get());\n let (io, workers) = match cfg!(target_os = \"macos\") {\n true => (cores.min(6), cores),\n false => (cores, 0),\n };\n Options { wanted: &everything, skip: &hidden_and_node_modules, io, workers, bytes: 256 << 20 }\n }\n}\n\n/// Wants every file.\npub fn everything(_: &Path, _: &OsStr) -> bool {\n true\n}\n\n/// Leaves out folders whose names start with `.`, and `node_modules`.\npub fn hidden_and_node_modules(_: &Path, name: &OsStr) -> bool {\n name.as_encoded_bytes().first() == Some(&b'.') || name == \"node_modules\"\n}\n\n/// A file to work on: its path, spelled from the path it was found under, and its bytes.\npub struct File {\n pub path: PathBuf,\n pub bytes: Vec,\n}\n\n/// What the work on a file returns.\npub struct Output {\n pub value: T,\n /// New contents for the file, which an I/O thread writes in place.\n pub write: Option>,\n}\n\n/// An entry of a folder.\n#[derive(Clone, Copy, Debug)]\npub struct Entry<'e> {\n pub name: &'e OsStr,\n pub kind: Kind,\n}\n\n#[derive(Clone, Copy, Debug, PartialEq, Eq)]\npub enum Kind {\n File,\n Folder,\n Link,\n Other,\n}\n\n#[derive(Debug)]\npub enum Error {\n Io(std::io::Error),\n /// The work, or one of the callbacks, panicked. The panic's message.\n Panic(String),\n}\n\nimpl std::fmt::Display for Error {\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n match self {\n Error::Io(error) => error.fmt(f),\n Error::Panic(message) => write!(f, \"panicked: {message}\"),\n }\n }\n}\n\nimpl std::error::Error for Error {}\n\n/// What a walk did.\npub struct Report {\n /// What the work returned, sorted by path. A result whose write failed is an error instead.\n pub results: Vec<(PathBuf, T)>,\n /// Paths that could not be found, listed, read or written, and those whose work panicked, sorted by path.\n pub errors: Vec<(PathBuf, Error)>,\n}\n\n/// Walks `paths`, and runs `work` on each file. `listing` sees the entries of each folder before its files and\n/// folders are queued. `state` makes the state of each thread that runs `work`, again after a panic.\npub fn run(\n paths: &[PathBuf],\n options: &Options,\n listing: impl Fn(&Path, &[Entry]) + Sync,\n state: impl Fn() -> S + Sync,\n work: impl Fn(&mut S, &File) -> Option> + Sync,\n) -> Report {\n let plan = plan::plan(paths);\n let mut report = Report {\n results: Vec::new(),\n errors: plan.errors.into_iter().map(|(path, error)| (path, Error::Io(error))).collect(),\n };\n let walk = pool::Walk {\n options,\n listing: &listing,\n state: &state,\n work: &work,\n arguments: &plan.arguments,\n nested: &plan.nested,\n claimed: plan.arguments.iter().map(|_| AtomicBool::new(false)).collect(),\n };\n let mut tasks: Vec = plan.top.iter().map(|&index| walk.argument(index)).collect();\n let mut held = Vec::new();\n while !tasks.is_empty() {\n let done = pool::run(&walk, tasks, true);\n report.results.extend(done.results);\n report.errors.extend(done.errors);\n held.extend(done.held);\n tasks = plan\n .nested\n .values()\n .filter(|&&index| walk.claim(index))\n .map(|&index| walk.argument(index))\n .collect();\n }\n let grouped = group(held);\n if !grouped.is_empty() {\n let tasks =\n grouped.into_iter().map(|(path, argument)| pool::Task::File { path, argument }).collect();\n let done = pool::run(&walk, tasks, false);\n report.results.extend(done.results);\n report.errors.extend(done.errors);\n }\n report.results.sort_by(|(left, _), (right, _)| left.cmp(right));\n report.errors.sort_by(|(left, _), (right, _)| left.cmp(right));\n report\n}\n\n/// One path for each file with more than one hard link: its first argument's spelling, or its smallest path.\nfn group(mut held: Vec) -> Vec<(PathBuf, Option)> {\n held.sort_by(|left, right| {\n (left.identity, left.argument.is_none(), left.argument, &left.path).cmp(&(\n right.identity,\n right.argument.is_none(),\n right.argument,\n &right.path,\n ))\n });\n held.dedup_by_key(|it| it.identity);\n held.into_iter().map(|it| (it.path, it.argument)).collect()\n}\n","editable",false,"branch","documentNavigation","repo","lukeed/tidy","repoId","b2cac5cc-818f-4958-803a-7d401ccd9a90","ownerId","user:5855893","organization","site","nexthub","private","admin","settings","maintainer","triage","starred","stars",0,"watchers",1,"watching","pinned","notify","views",[120,121,122,123,124,125,126,127,128,129,130,131,132],"navigation",{"_57":58,"_79":80,"_81":82,"_83":84,"_85":86,"_87":88,"_89":90},"sources",[],"viewer","meta",{"_72":73,"_74":75,"_76":73,"_44":32,"_77":14,"_78":32,"_50":51,"_52":53},"branches",[14],"tags",[],"cloneUrl","https://git.nexthub.sh/lukeed/tidy.git","description","","owner","lukeed","section","head","empty","pins",[102,103,104],"documents",{"_99":100,"_101":51},"code",{"_96":97,"_98":51},"automations",{"_93":94,"_95":-5},"delivery",{"_91":51,"_92":-5},"tag","W1_y18R9FUcyveBRlHeRXdVTtjo","releases","applications","workflows",2,"deployments","commits",350,"changes","counts",{},"total",{"_105":98,"_107":118,"_15":98,"_109":119,"_55":111,"_112":32,"_113":111,"_114":51},{"_105":115,"_107":116,"_15":115,"_109":117,"_55":111,"_112":32,"_113":111,"_114":51},{"_105":106,"_107":108,"_15":106,"_109":110,"_55":111,"_112":32,"_113":111,"_114":51},"id","issues","name","Issues","filters",{},true,"saved","pinnable","count","tasks","Tasks",{},"Changes",{},{"_105":98,"_107":118,"_15":98,"_109":163,"_55":111,"_112":32,"_113":111},{"_105":115,"_107":116,"_15":115,"_109":162,"_55":111,"_112":32,"_113":111},{"_105":106,"_107":108,"_15":106,"_109":161,"_55":111,"_112":32,"_113":111},{"_105":158,"_107":159,"_15":158,"_109":160,"_55":32,"_112":32,"_113":111},{"_105":155,"_107":156,"_15":155,"_109":157,"_55":32,"_112":32,"_113":111},{"_105":93,"_107":153,"_15":85,"_109":154,"_55":32,"_112":32,"_113":111},{"_105":95,"_107":150,"_15":151,"_109":152,"_55":32,"_112":32,"_113":111},{"_105":91,"_107":148,"_15":91,"_109":149,"_55":32,"_112":32,"_113":111},{"_105":144,"_107":145,"_15":146,"_109":147,"_55":32,"_112":32,"_113":111},{"_105":81,"_107":142,"_15":81,"_109":143,"_55":32,"_112":32,"_113":32},{"_105":139,"_107":140,"_15":139,"_109":141,"_55":32,"_112":32,"_113":32},{"_105":136,"_107":137,"_15":136,"_109":138,"_55":32,"_112":32,"_113":32},{"_105":133,"_107":134,"_15":133,"_109":135,"_55":32,"_112":32,"_113":32},"decisions","Decisions",{},"changelog","Changelog",{},"roadmap","Roadmap",{},"Documents",{},"activity","Activity","insights",{},"Releases",{},"Deployments","automations/deployments",{},"Workflows",{},"discussions","Discussions",{},"goals","Goals",{},{},{},{},"groups","user","agents",[],"now",1794686775597,"sidebar","expanded"]