[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10,"_11":12},"actionData","errors","root",{"_164":-5,"_167":168,"_169":170},"routes/shell",{"_41":42,"_163":-5,"_164":-5,"_165":166},"routes/repo",{"_34":-7,"_35":36,"_37":38,"_39":40,"_41":42,"_43":32,"_44":32,"_45":32,"_46":32,"_47":32,"_48":32,"_49":50,"_51":52,"_53":32,"_54":-7,"_55":-7,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70},"routes/repo.blob",{"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":-5,"_26":27,"_28":29,"_29":30,"_31":32,"_33":14},"ref","main","path","bench/walk/src/main.rs","oid","043d4a9279c8535d3aea8fddfbdeacbc2a599566","size",35301,"commit","12e96b5435e408a25422fcfd87d6fa5474b6e53a","mounts",[],"source","plain",{"_13":14,"_15":16},"kind","text","//! Times ways to walk a folder tree, alone or with reading each matching file. It chose the design of `tidy_walk`.\n//! `--real md` builds against mdtidy, which must sit next to this repo until it merges in.\n//!\n//! walk-bench ... [--threads ] [--read] [--ext md,markdown] [--work ] [--real tidy|md]\n//! [--readers ] [--cap ] [--write] [--gate ]\n//! walk-bench ops --op open|stat|read|rewrite|... [--threads ] [--procs ]\n//!\n//! Strategies: serial, bulk (serial, with getattrlistbulk), queue (a mutex queue of folders, each folder's files\n//! handled by the thread that listed it), steal (work stealing), steal-bulk, steal-at, ignore (the ignore crate's\n//! parallel walker), sem (tidy's queue of folder and file tasks, with at most --readers threads listing or reading),\n//! pipe (stealing readers send to --threads workers), pipe-queue (the same with a mutex queue of folders), and\n//! pipe-tasks (readers share folder and file tasks). Each skips hidden folders and node_modules, and doesn't follow\n//! links. `--real` runs tidy or mdtidy on each file, `--work` times. `ops` runs one call per listed path.\n\nuse crossbeam_deque::{Injector, Stealer, Worker};\nuse crossbeam_utils::Backoff;\nuse std::ffi::{OsStr, OsString};\nuse std::io::Read;\nuse std::os::unix::ffi::OsStrExt;\nuse std::path::{Path, PathBuf};\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::sync::{Condvar, Mutex};\n\n#[derive(Clone, Copy, PartialEq)]\nenum Kind {\n File,\n Dir,\n Other,\n}\n\nstruct Settings {\n /// Passes of simulated formatting over each file's bytes.\n work: usize,\n /// For `pipe`: how many threads walk and read.\n readers: usize,\n read: bool,\n /// Open each file without reading it.\n open: bool,\n extensions: Vec,\n /// Run a real formatter instead of the simulated work: tidy or mdtidy.\n real: Real,\n markdown: mdtidy::Options,\n /// For `pipe`: the channel's capacity in files.\n capacity: usize,\n /// For `pipe`: write each file's bytes back from the worker, through at most `gate` writers at once (0: no limit).\n write: bool,\n gate: usize,\n}\n\n/// A counting semaphore.\nstruct Permits {\n free: Mutex,\n ready: Condvar,\n}\n\nimpl Permits {\n fn with(&self, run: impl FnOnce() -> R) -> R {\n let mut free = self.free.lock().unwrap();\n while *free == 0 {\n free = self.ready.wait(free).unwrap();\n }\n *free -= 1;\n drop(free);\n let result = run();\n *self.free.lock().unwrap() += 1;\n self.ready.notify_one();\n result\n }\n}\n\n#[derive(Clone, Copy, PartialEq)]\nenum Real {\n None,\n Tidy,\n Md,\n}\n\n#[derive(Default)]\nstruct State {\n formatter: tidy::Formatter,\n}\n\n/// The work on one file: simulated, or a real formatter that reports whether the file changed.\nfn work(state: &mut State, path: &Path, bytes: &[u8], settings: &Settings) -> usize {\n match settings.real {\n Real::None => format(bytes, settings.work),\n Real::Tidy => {\n let Ok(text) = std::str::from_utf8(bytes) else { return 0 };\n let name = path.to_string_lossy();\n let Some(language) = tidy::Language::of_path(&name) else { return 0 };\n let options = tidy::Options::default();\n let mut changed = 0;\n for _ in 0..settings.work.max(1) {\n let result = state.formatter.format(text, language, &options, Some(&name), &mut tidy::embed::Verbatim);\n changed = result.map_or(0, |output| (output != text) as usize);\n }\n changed\n }\n Real::Md => {\n let Ok(text) = std::str::from_utf8(bytes) else { return 0 };\n let mut changed = 0;\n for _ in 0..settings.work.max(1) {\n changed = (mdtidy::format(text, &settings.markdown) != text) as usize;\n }\n changed\n }\n }\n}\n\n#[derive(Default)]\nstruct Tally {\n files: usize,\n bytes: usize,\n buffer: Vec,\n state: State,\n}\n\nimpl Tally {\n fn file(&mut self, path: &Path, settings: &Settings) {\n let matches = path.extension().is_some_and(|extension| {\n settings.extensions.is_empty() || settings.extensions.iter().any(|it| extension == it.as_str())\n });\n if !matches {\n return;\n }\n self.files += 1;\n if settings.open {\n drop(std::fs::File::open(path));\n }\n if settings.read {\n let Ok(mut file) = std::fs::File::open(path) else { return };\n self.buffer.clear();\n loop {\n if self.buffer.capacity() - self.buffer.len() < 16384 {\n self.buffer.reserve(65536);\n }\n let spare = self.buffer.spare_capacity_mut();\n let spare = unsafe { std::slice::from_raw_parts_mut(spare.as_mut_ptr().cast::(), spare.len()) };\n match file.read(spare) {\n Ok(0) | Err(_) => break,\n Ok(read) => unsafe { self.buffer.set_len(self.buffer.len() + read) },\n }\n }\n self.bytes += self.buffer.len();\n self.bytes += work(&mut self.state, path, &self.buffer, settings) & 1;\n }\n }\n}\n\n/// Simulated formatting: `passes` passes over the bytes.\nfn format(bytes: &[u8], passes: usize) -> usize {\n let mut sum = 0usize;\n for pass in 0..passes {\n for &byte in bytes {\n sum = sum.wrapping_mul(31).wrapping_add(byte as usize ^ pass);\n }\n }\n std::hint::black_box(sum)\n}\n\nfn skipped(name: &OsStr) -> bool {\n name.as_bytes().first() == Some(&b'.') || name == \"node_modules\"\n}\n\ntype List = fn(&Path, &mut Vec<(OsString, Kind)>);\n\n/// `read_dir` and the type from `d_type`.\nfn list_std(dir: &Path, out: &mut Vec<(OsString, Kind)>) {\n let Ok(entries) = std::fs::read_dir(dir) else { return };\n for entry in entries.flatten() {\n let Ok(kind) = entry.file_type() else { continue };\n let kind = if kind.is_dir() {\n Kind::Dir\n } else if kind.is_file() {\n Kind::File\n } else {\n Kind::Other\n };\n out.push((entry.file_name(), kind));\n }\n}\n\n#[cfg(not(target_os = \"macos\"))]\nfn list_bulk(dir: &Path, out: &mut Vec<(OsString, Kind)>) {\n list_std(dir, out)\n}\n\n/// `getattrlistbulk`: the names and types of a whole folder in a few calls.\n#[cfg(target_os = \"macos\")]\nfn list_bulk(dir: &Path, out: &mut Vec<(OsString, Kind)>) {\n let mut path = dir.as_os_str().as_bytes().to_vec();\n path.push(0);\n let fd = unsafe { libc::open(path.as_ptr().cast(), libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC) };\n if fd < 0 {\n return;\n }\n let mut list: libc::attrlist = unsafe { std::mem::zeroed() };\n list.bitmapcount = libc::ATTR_BIT_MAP_COUNT;\n list.commonattr = libc::ATTR_CMN_RETURNED_ATTRS | libc::ATTR_CMN_NAME | libc::ATTR_CMN_OBJTYPE;\n thread_local! {\n static BUFFER: std::cell::RefCell> = std::cell::RefCell::new(vec![0u8; 64 * 1024]);\n }\n BUFFER.with_borrow_mut(|buffer| loop {\n let count = unsafe {\n libc::getattrlistbulk(fd, (&raw mut list).cast(), buffer.as_mut_ptr().cast(), buffer.len(), 0)\n };\n if count <= 0 {\n break;\n }\n let mut at = 0usize;\n for _ in 0..count {\n let read_u32 = |offset: usize| u32::from_ne_bytes(buffer[offset..offset + 4].try_into().unwrap());\n let length = read_u32(at) as usize;\n let mut field = at + 4 + std::mem::size_of::();\n let offset = i32::from_ne_bytes(buffer[field..field + 4].try_into().unwrap());\n let name_length = read_u32(field + 4) as usize;\n let start = (field as isize + offset as isize) as usize;\n let name = OsStr::from_bytes(&buffer[start..start + name_length.saturating_sub(1)]).to_owned();\n field += 8;\n let kind = match read_u32(field) {\n 1 => Kind::File,\n 2 => Kind::Dir,\n _ => Kind::Other,\n };\n out.push((name, kind));\n at += length;\n }\n });\n unsafe { libc::close(fd) };\n}\n\nfn serial(roots: &[PathBuf], list: List, settings: &Settings) -> (usize, usize) {\n let mut tally = Tally::default();\n let mut stack: Vec = roots.to_vec();\n let mut entries = Vec::new();\n while let Some(dir) = stack.pop() {\n entries.clear();\n list(&dir, &mut entries);\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => stack.push(dir.join(name)),\n Kind::File => tally.file(&dir.join(name), settings),\n _ => {}\n }\n }\n }\n (tally.files, tally.bytes)\n}\n\nfn queue(roots: &[PathBuf], threads: usize, settings: &Settings) -> (usize, usize) {\n let state = Mutex::new((roots.to_vec(), 0usize));\n let ready = Condvar::new();\n let totals = Mutex::new((0, 0));\n std::thread::scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|| {\n let mut tally = Tally::default();\n let mut entries = Vec::new();\n loop {\n let dir = {\n let mut guard = state.lock().unwrap();\n loop {\n if let Some(dir) = guard.0.pop() {\n guard.1 += 1;\n break Some(dir);\n }\n if guard.1 == 0 {\n break None;\n }\n guard = ready.wait(guard).unwrap();\n }\n };\n let Some(dir) = dir else { break };\n entries.clear();\n list_std(&dir, &mut entries);\n let mut found = Vec::new();\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => found.push(dir.join(name)),\n Kind::File => tally.file(&dir.join(name), settings),\n _ => {}\n }\n }\n let mut guard = state.lock().unwrap();\n let wakes = !found.is_empty() || guard.1 == 1;\n guard.0.extend(found);\n guard.1 -= 1;\n drop(guard);\n if wakes {\n ready.notify_all();\n }\n }\n let mut totals = totals.lock().unwrap();\n totals.0 += tally.files;\n totals.1 += tally.bytes;\n });\n }\n });\n totals.into_inner().unwrap()\n}\n\nfn steal(roots: &[PathBuf], threads: usize, list: List, settings: &Settings) -> (usize, usize) {\n let injector = Injector::new();\n for root in roots {\n injector.push(root.clone());\n }\n let pending = AtomicUsize::new(roots.len());\n let workers: Vec> = (0..threads).map(|_| Worker::new_lifo()).collect();\n let stealers: Vec> = workers.iter().map(Worker::stealer).collect();\n let totals = Mutex::new((0, 0));\n std::thread::scope(|scope| {\n for (index, local) in workers.into_iter().enumerate() {\n let (injector, pending, stealers, totals) = (&injector, &pending, &stealers, &totals);\n scope.spawn(move || {\n let mut tally = Tally::default();\n let mut entries = Vec::new();\n let backoff = Backoff::new();\n loop {\n let task = local.pop().or_else(|| {\n std::iter::repeat_with(|| {\n injector.steal_batch_and_pop(&local).or_else(|| {\n stealers\n .iter()\n .enumerate()\n .filter(|(other, _)| *other != index)\n .map(|(_, stealer)| stealer.steal())\n .collect()\n })\n })\n .find(|steal| !steal.is_retry())\n .and_then(|steal| steal.success())\n });\n let Some(dir) = task else {\n if pending.load(Ordering::Acquire) == 0 {\n break;\n }\n backoff.snooze();\n continue;\n };\n backoff.reset();\n entries.clear();\n list(&dir, &mut entries);\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => {\n pending.fetch_add(1, Ordering::AcqRel);\n local.push(dir.join(name));\n }\n Kind::File => tally.file(&dir.join(name), settings),\n _ => {}\n }\n }\n pending.fetch_sub(1, Ordering::AcqRel);\n }\n let mut totals = totals.lock().unwrap();\n totals.0 += tally.files;\n totals.1 += tally.bytes;\n });\n }\n });\n totals.into_inner().unwrap()\n}\n\n/// Work stealing with folder descriptors: each folder is opened from its parent's descriptor, and each file from its\n/// folder's, so the kernel never resolves a whole path again.\nfn steal_at(roots: &[PathBuf], threads: usize, settings: &Settings) -> (usize, usize) {\n use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};\n use std::sync::Arc;\n struct Task {\n parent: Option>,\n path: PathBuf,\n }\n let injector = Injector::new();\n for root in roots {\n injector.push(Task { parent: None, path: root.clone() });\n }\n let pending = AtomicUsize::new(roots.len());\n let workers: Vec> = (0..threads).map(|_| Worker::new_lifo()).collect();\n let stealers: Vec> = workers.iter().map(Worker::stealer).collect();\n let totals = Mutex::new((0, 0));\n let open = |parent: Option<&OwnedFd>, path: &Path, flags: i32| -> Option {\n let name = match parent {\n Some(_) => path.file_name()?.as_bytes(),\n None => path.as_os_str().as_bytes(),\n };\n let mut c = name.to_vec();\n c.push(0);\n let fd = unsafe {\n libc::openat(parent.map_or(libc::AT_FDCWD, |it| it.as_raw_fd()), c.as_ptr().cast(), flags | libc::O_CLOEXEC)\n };\n (fd >= 0).then(|| unsafe { OwnedFd::from_raw_fd(fd) })\n };\n std::thread::scope(|scope| {\n for (index, local) in workers.into_iter().enumerate() {\n let (injector, pending, stealers, totals) = (&injector, &pending, &stealers, &totals);\n scope.spawn(move || {\n let mut tally = Tally::default();\n let backoff = Backoff::new();\n loop {\n let task = local.pop().or_else(|| {\n std::iter::repeat_with(|| {\n injector.steal_batch_and_pop(&local).or_else(|| {\n stealers\n .iter()\n .enumerate()\n .filter(|(other, _)| *other != index)\n .map(|(_, stealer)| stealer.steal())\n .collect()\n })\n })\n .find(|steal| !steal.is_retry())\n .and_then(|steal| steal.success())\n });\n let Some(task) = task else {\n if pending.load(Ordering::Acquire) == 0 {\n break;\n }\n backoff.snooze();\n continue;\n };\n backoff.reset();\n if let Some(dir) = open(task.parent.as_deref(), &task.path, libc::O_RDONLY | libc::O_DIRECTORY) {\n let dir = Arc::new(dir);\n let stream = unsafe { libc::fdopendir(libc::dup(dir.as_raw_fd())) };\n if !stream.is_null() {\n loop {\n let entry = unsafe { libc::readdir(stream) };\n if entry.is_null() {\n break;\n }\n let entry = unsafe { &*entry };\n let name = unsafe { std::ffi::CStr::from_ptr(entry.d_name.as_ptr()) }.to_bytes();\n if name == b\".\" || name == b\"..\" {\n continue;\n }\n let name = OsStr::from_bytes(name);\n match entry.d_type {\n libc::DT_DIR if !skipped(name) => {\n pending.fetch_add(1, Ordering::AcqRel);\n local.push(Task { parent: Some(dir.clone()), path: task.path.join(name) });\n }\n libc::DT_REG => {\n let path = task.path.join(name);\n let matches = path.extension().is_some_and(|extension| {\n settings.extensions.is_empty() || settings.extensions.iter().any(|it| extension == it.as_str())\n });\n if !matches {\n continue;\n }\n tally.files += 1;\n if settings.read || settings.open {\n let Some(file) = open(Some(&dir), &path, libc::O_RDONLY) else { continue };\n if settings.read {\n let mut file = std::fs::File::from(file);\n tally.buffer.clear();\n let _ = file.read_to_end(&mut tally.buffer);\n tally.bytes += tally.buffer.len();\n tally.bytes += work(&mut tally.state, &path, &tally.buffer, settings) & 1;\n }\n }\n }\n _ => {}\n }\n }\n unsafe { libc::closedir(stream) };\n }\n }\n pending.fetch_sub(1, Ordering::AcqRel);\n }\n let mut totals = totals.lock().unwrap();\n totals.0 += tally.files;\n totals.1 += tally.bytes;\n });\n }\n });\n totals.into_inner().unwrap()\n}\n\n/// `settings.readers` threads walk and read, and `threads` threads format what they read.\nfn pipe(roots: &[PathBuf], threads: usize, settings: &Settings, readers: Readers) -> (usize, usize) {\n let (send, receive) = crossbeam_channel::bounded::<(PathBuf, Vec)>(settings.capacity);\n let totals = Mutex::new((0, 0));\n let gate = Permits { free: Mutex::new(settings.gate), ready: Condvar::new() };\n std::thread::scope(|scope| {\n for _ in 0..threads {\n let (receive, totals, gate) = (receive.clone(), &totals, &gate);\n scope.spawn(move || {\n let (mut files, mut bytes) = (0, 0);\n let mut state = State::default();\n for (path, buffer) in receive {\n files += 1;\n bytes += buffer.len() + (work(&mut state, &path, &buffer, settings) & 1);\n if settings.write {\n let written = match settings.gate {\n 0 => std::fs::write(&path, &buffer),\n _ => gate.with(|| std::fs::write(&path, &buffer)),\n };\n written.unwrap();\n }\n }\n let mut totals = totals.lock().unwrap();\n totals.0 += files;\n totals.1 += bytes;\n });\n }\n drop(receive);\n match readers {\n Readers::Steal => drop(steal_paths(roots, settings.readers, settings, &Mutex::new(Vec::new()), &send)),\n Readers::Queue => queue_paths(roots, settings.readers, settings, &send),\n Readers::Tasks => task_paths(roots, settings.readers, settings, &send),\n }\n drop(send);\n });\n totals.into_inner().unwrap()\n}\n\n/// One pool of threads that list, read and format, sharing tidy's queue of folder and file tasks, with at most\n/// `settings.readers` threads inside a list or a read at once.\nfn sem(roots: &[PathBuf], threads: usize, settings: &Settings) -> (usize, usize) {\n enum Task {\n Folder(PathBuf),\n File(PathBuf),\n }\n let permits = Permits { free: Mutex::new(settings.readers), ready: Condvar::new() };\n let state = Mutex::new((roots.iter().cloned().map(Task::Folder).collect::>(), 0usize));\n let ready = Condvar::new();\n let totals = Mutex::new((0, 0));\n std::thread::scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|| {\n let (mut files, mut bytes) = (0, 0);\n let mut worker = State::default();\n let mut entries = Vec::new();\n loop {\n let task = {\n let mut guard = state.lock().unwrap();\n loop {\n if let Some(task) = guard.0.pop() {\n guard.1 += 1;\n break Some(task);\n }\n if guard.1 == 0 {\n break None;\n }\n guard = ready.wait(guard).unwrap();\n }\n };\n let Some(task) = task else { break };\n let mut found = Vec::new();\n match task {\n Task::Folder(dir) => {\n entries.clear();\n permits.with(|| list_std(&dir, &mut entries));\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => found.push(Task::Folder(dir.join(name))),\n Kind::File => {\n let path = dir.join(name);\n let matches = path.extension().is_some_and(|extension| {\n settings.extensions.is_empty() || settings.extensions.iter().any(|it| extension == it.as_str())\n });\n if matches {\n found.push(Task::File(path));\n }\n }\n _ => {}\n }\n }\n }\n Task::File(path) => {\n if let Ok(buffer) = permits.with(|| std::fs::read(&path)) {\n files += 1;\n bytes += buffer.len() + (work(&mut worker, &path, &buffer, settings) & 1);\n }\n }\n }\n let mut guard = state.lock().unwrap();\n let wakes = !found.is_empty() || guard.1 == 1;\n guard.0.extend(found);\n guard.1 -= 1;\n drop(guard);\n if wakes {\n ready.notify_all();\n }\n }\n let mut totals = totals.lock().unwrap();\n totals.0 += files;\n totals.1 += bytes;\n });\n }\n });\n totals.into_inner().unwrap()\n}\n\n#[derive(Clone, Copy)]\nenum Readers {\n Steal,\n Queue,\n Tasks,\n}\n\n/// Readers that share one queue of folder and file tasks, so that the files of one folder are read by all readers.\nfn task_paths(roots: &[PathBuf], threads: usize, settings: &Settings, send: &crossbeam_channel::Sender<(PathBuf, Vec)>) {\n enum Task {\n Folder(PathBuf),\n File(PathBuf),\n }\n let state = Mutex::new((roots.iter().cloned().map(Task::Folder).collect::>(), 0usize));\n let ready = Condvar::new();\n std::thread::scope(|scope| {\n for _ in 0..threads {\n let send = send.clone();\n let (state, ready) = (&state, &ready);\n scope.spawn(move || {\n let mut entries = Vec::new();\n loop {\n let task = {\n let mut guard = state.lock().unwrap();\n loop {\n if let Some(task) = guard.0.pop() {\n guard.1 += 1;\n break Some(task);\n }\n if guard.1 == 0 {\n break None;\n }\n guard = ready.wait(guard).unwrap();\n }\n };\n let Some(task) = task else { break };\n let mut found = Vec::new();\n match task {\n Task::Folder(dir) => {\n entries.clear();\n list_std(&dir, &mut entries);\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => found.push(Task::Folder(dir.join(name))),\n Kind::File => {\n let path = dir.join(name);\n let matches = path.extension().is_some_and(|extension| {\n settings.extensions.is_empty() || settings.extensions.iter().any(|it| extension == it.as_str())\n });\n if matches {\n found.push(Task::File(path));\n }\n }\n _ => {}\n }\n }\n }\n Task::File(path) => {\n if let Ok(bytes) = std::fs::read(&path) {\n let _ = send.send((path, bytes));\n }\n }\n }\n let mut guard = state.lock().unwrap();\n let wakes = !found.is_empty() || guard.1 == 1;\n guard.0.extend(found);\n guard.1 -= 1;\n drop(guard);\n if wakes {\n ready.notify_all();\n }\n }\n });\n }\n });\n}\n\n/// tidy's mutex queue of folders, as the design proposes for the readers, with each matching file read and sent.\nfn queue_paths(roots: &[PathBuf], threads: usize, settings: &Settings, send: &crossbeam_channel::Sender<(PathBuf, Vec)>) {\n let state = Mutex::new((roots.to_vec(), 0usize));\n let ready = Condvar::new();\n std::thread::scope(|scope| {\n for _ in 0..threads {\n let send = send.clone();\n let (state, ready) = (&state, &ready);\n scope.spawn(move || {\n let mut entries = Vec::new();\n loop {\n let dir = {\n let mut guard = state.lock().unwrap();\n loop {\n if let Some(dir) = guard.0.pop() {\n guard.1 += 1;\n break Some(dir);\n }\n if guard.1 == 0 {\n break None;\n }\n guard = ready.wait(guard).unwrap();\n }\n };\n let Some(dir) = dir else { break };\n entries.clear();\n list_std(&dir, &mut entries);\n let mut found = Vec::new();\n let mut files = Vec::new();\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => found.push(dir.join(name)),\n Kind::File => {\n let path = dir.join(name);\n let matches = path.extension().is_some_and(|extension| {\n settings.extensions.is_empty() || settings.extensions.iter().any(|it| extension == it.as_str())\n });\n if matches {\n files.push(path);\n }\n }\n _ => {}\n }\n }\n let mut guard = state.lock().unwrap();\n let wakes = !found.is_empty() || guard.1 == 1;\n guard.0.extend(found);\n guard.1 -= 1;\n drop(guard);\n if wakes {\n ready.notify_all();\n }\n for path in files {\n if let Ok(bytes) = std::fs::read(&path) {\n let _ = send.send((path, bytes));\n }\n }\n }\n });\n }\n });\n}\n\n/// `steal`, with each matching file read into a new buffer and sent.\nfn steal_paths(\n roots: &[PathBuf],\n threads: usize,\n settings: &Settings,\n _paths: &Mutex>,\n send: &crossbeam_channel::Sender<(PathBuf, Vec)>,\n) -> usize {\n let injector = Injector::new();\n for root in roots {\n injector.push(root.clone());\n }\n let pending = AtomicUsize::new(roots.len());\n let workers: Vec> = (0..threads).map(|_| Worker::new_lifo()).collect();\n let stealers: Vec> = workers.iter().map(Worker::stealer).collect();\n std::thread::scope(|scope| {\n for (index, local) in workers.into_iter().enumerate() {\n let (injector, pending, stealers, send) = (&injector, &pending, &stealers, send.clone());\n scope.spawn(move || {\n let mut entries = Vec::new();\n let backoff = Backoff::new();\n loop {\n let task = local.pop().or_else(|| {\n std::iter::repeat_with(|| {\n injector.steal_batch_and_pop(&local).or_else(|| {\n stealers\n .iter()\n .enumerate()\n .filter(|(other, _)| *other != index)\n .map(|(_, stealer)| stealer.steal())\n .collect()\n })\n })\n .find(|steal| !steal.is_retry())\n .and_then(|steal| steal.success())\n });\n let Some(dir) = task else {\n if pending.load(Ordering::Acquire) == 0 {\n break;\n }\n backoff.snooze();\n continue;\n };\n backoff.reset();\n entries.clear();\n list_std(&dir, &mut entries);\n for (name, kind) in entries.drain(..) {\n match kind {\n Kind::Dir if !skipped(&name) => {\n pending.fetch_add(1, Ordering::AcqRel);\n local.push(dir.join(name));\n }\n Kind::File => {\n let path = dir.join(name);\n let matches = path.extension().is_some_and(|extension| {\n settings.extensions.is_empty() || settings.extensions.iter().any(|it| extension == it.as_str())\n });\n if matches && let Ok(bytes) = std::fs::read(&path) {\n let _ = send.send((path, bytes));\n }\n }\n _ => {}\n }\n }\n pending.fetch_sub(1, Ordering::AcqRel);\n }\n });\n }\n });\n 0\n}\n\nfn ignore(roots: &[PathBuf], threads: usize, settings: &Settings) -> (usize, usize) {\n let mut builder = ignore::WalkBuilder::new(&roots[0]);\n for root in &roots[1..] {\n builder.add(root);\n }\n builder\n .standard_filters(false)\n .threads(threads)\n .filter_entry(|entry| !(entry.file_type().is_some_and(|it| it.is_dir()) && entry.depth() > 0 && skipped(entry.file_name())));\n let totals = Mutex::new((0, 0));\n builder.build_parallel().run(|| {\n let totals = &totals;\n let mut tally = Tally::default();\n Box::new(move |entry| {\n if let Ok(entry) = entry\n && entry.file_type().is_some_and(|it| it.is_file())\n {\n tally.file(entry.path(), settings);\n let mut totals = totals.lock().unwrap();\n totals.0 += std::mem::take(&mut tally.files);\n totals.1 += std::mem::take(&mut tally.bytes);\n }\n ignore::WalkState::Continue\n })\n });\n totals.into_inner().unwrap()\n}\n\n/// Runs one file system call on each path of a list file, on `threads` threads, or in `procs` processes of one\n/// thread each, to find what limits scaling.\nfn ops(list: &Path, op: &str, threads: usize, procs: usize, slice: (usize, usize)) -> (usize, usize) {\n let text = std::fs::read_to_string(list).unwrap();\n let paths: Vec = text\n .lines()\n .enumerate()\n .filter(|(index, _)| index % slice.1 == slice.0)\n .map(|(_, line)| std::ffi::CString::new(line).unwrap())\n .collect();\n if procs > 1 {\n let exe = std::env::current_exe().unwrap();\n let children: Vec<_> = (0..procs)\n .map(|index| {\n std::process::Command::new(&exe)\n .args([\"ops\", list.to_str().unwrap(), \"--op\", op, \"--threads\", \"1\", \"--slice\"])\n .arg(format!(\"{index}/{procs}\"))\n .stdout(std::process::Stdio::null())\n .spawn()\n .unwrap()\n })\n .collect();\n for mut child in children {\n child.wait().unwrap();\n }\n return (paths.len() * procs, 0);\n }\n let next = AtomicUsize::new(0);\n let kept = Mutex::new(Vec::new());\n std::thread::scope(|scope| {\n for _ in 0..threads {\n scope.spawn(|| {\n let mut mine = Vec::new();\n let mut buffer = vec![0u8; 1 << 20];\n let mut stat: libc::stat = unsafe { std::mem::zeroed() };\n loop {\n let index = next.fetch_add(1, Ordering::Relaxed);\n let Some(path) = paths.get(index) else { break };\n let path = path.as_ptr();\n unsafe {\n match op {\n \"stat\" => drop(libc::stat(path, &mut stat)),\n \"lstat\" => drop(libc::lstat(path, &mut stat)),\n \"access\" => drop(libc::access(path, libc::R_OK)),\n \"open\" => {\n let fd = libc::open(path, libc::O_RDONLY | libc::O_CLOEXEC);\n libc::close(fd);\n }\n \"openonly\" => mine.push(libc::open(path, libc::O_RDONLY | libc::O_CLOEXEC)),\n \"read\" => {\n let fd = libc::open(path, libc::O_RDONLY | libc::O_CLOEXEC);\n while libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len()) > 0 {}\n libc::close(fd);\n }\n \"rewrite\" => {\n let fd = libc::open(path, libc::O_RDONLY | libc::O_CLOEXEC);\n let read = libc::read(fd, buffer.as_mut_ptr().cast(), buffer.len());\n libc::close(fd);\n let fd = libc::open(path, libc::O_WRONLY | libc::O_TRUNC | libc::O_CLOEXEC);\n libc::write(fd, buffer.as_ptr().cast(), read.max(0) as usize);\n libc::close(fd);\n }\n \"readfstat\" => {\n let fd = libc::open(path, libc::O_RDONLY | libc::O_CLOEXEC);\n libc::fstat(fd, &mut stat);\n libc::read(fd, buffer.as_mut_ptr().cast(), (stat.st_size as usize + 1).min(buffer.len()));\n libc::close(fd);\n }\n other => panic!(\"unknown op {other}\"),\n }\n }\n }\n kept.lock().unwrap().extend(mine);\n });\n }\n });\n for fd in kept.into_inner().unwrap() {\n unsafe { libc::close(fd) };\n }\n (paths.len(), 0)\n}\n\nfn main() {\n let args: Vec = std::env::args().skip(1).collect();\n let strategy = args[0].clone();\n if strategy == \"ops\" {\n let limit = libc::rlimit { rlim_cur: 65536, rlim_max: libc::RLIM_INFINITY };\n unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) };\n let (mut op, mut threads, mut procs, mut slice) = (String::from(\"open\"), 1, 1, (0, 1));\n let mut rest = args[2..].iter();\n while let Some(arg) = rest.next() {\n match arg.as_str() {\n \"--op\" => op = rest.next().unwrap().clone(),\n \"--threads\" => threads = rest.next().unwrap().parse().unwrap(),\n \"--procs\" => procs = rest.next().unwrap().parse().unwrap(),\n \"--slice\" => {\n let (index, count) = rest.next().unwrap().split_once('/').unwrap();\n slice = (index.parse().unwrap(), count.parse().unwrap());\n }\n other => panic!(\"unknown option {other}\"),\n }\n }\n let (files, _) = ops(Path::new(&args[1]), &op, threads, procs, slice);\n println!(\"{files} files\");\n return;\n }\n let mut roots = Vec::new();\n let mut threads = std::thread::available_parallelism().map_or(1, |it| it.get());\n let mut settings = Settings {\n work: 0,\n readers: 4,\n read: false,\n open: false,\n extensions: Vec::new(),\n real: Real::None,\n markdown: mdtidy::Options::default(),\n capacity: 256,\n write: false,\n gate: 0,\n };\n let mut rest = args[1..].iter();\n while let Some(arg) = rest.next() {\n match arg.as_str() {\n \"--threads\" => threads = rest.next().unwrap().parse().unwrap(),\n \"--read\" => settings.read = true,\n \"--work\" => settings.work = rest.next().unwrap().parse().unwrap(),\n \"--readers\" => settings.readers = rest.next().unwrap().parse().unwrap(),\n \"--open\" => settings.open = true,\n \"--real\" => {\n settings.real = match rest.next().unwrap().as_str() {\n \"tidy\" => Real::Tidy,\n \"md\" => Real::Md,\n other => panic!(\"unknown formatter {other}\"),\n };\n settings.read = true;\n }\n \"--write\" => settings.write = true,\n \"--gate\" => settings.gate = rest.next().unwrap().parse().unwrap(),\n \"--cap\" => settings.capacity = rest.next().unwrap().parse().unwrap(),\n \"--ext\" => settings.extensions = rest.next().unwrap().split(',').map(str::to_owned).collect(),\n path => roots.push(PathBuf::from(path)),\n }\n }\n let (files, bytes) = match strategy.as_str() {\n \"serial\" => serial(&roots, list_std, &settings),\n \"bulk\" => serial(&roots, list_bulk, &settings),\n \"queue\" => queue(&roots, threads, &settings),\n \"steal\" => steal(&roots, threads, list_std, &settings),\n \"steal-bulk\" => steal(&roots, threads, list_bulk, &settings),\n \"ignore\" => ignore(&roots, threads, &settings),\n \"steal-at\" => steal_at(&roots, threads, &settings),\n \"pipe\" => pipe(&roots, threads, &settings, Readers::Steal),\n \"pipe-queue\" => pipe(&roots, threads, &settings, Readers::Queue),\n \"sem\" => sem(&roots, threads, &settings),\n \"pipe-tasks\" => pipe(&roots, threads, &settings, Readers::Tasks),\n other => panic!(\"unknown strategy {other}\"),\n };\n println!(\"{files} files, {bytes} bytes\");\n}\n","editable",false,"branch","documentNavigation","repo","lukeed/tidy","repoId","b2cac5cc-818f-4958-803a-7d401ccd9a90","ownerId","user:5855893","site","nexthub","private","admin","settings","maintainer","triage","starred","stars",0,"watchers",1,"watching","pinned","notify","views",[119,120,121,122,123,124,125,126,127,128,129,130,131],"navigation",{"_56":57,"_78":79,"_80":81,"_82":83,"_84":85,"_86":87,"_88":89},"sources",[],"viewer","meta",{"_71":72,"_73":74,"_75":72,"_43":32,"_76":14,"_77":32,"_49":50,"_51":52},"branches",[14],"tags",[],"cloneUrl","https://git.nexthub.sh/lukeed/tidy.git","description","","owner","lukeed","section","head","empty","pins",[101,102,103],"documents",{"_98":99,"_100":50},"code",{"_95":96,"_97":50},"automations",{"_92":93,"_94":-5},"delivery",{"_90":50,"_91":-5},"tag","W1_y18R9FUcyveBRlHeRXdVTtjo","releases","applications","workflows",2,"deployments","commits",350,"changes","counts",{},"total",{"_104":97,"_106":117,"_15":97,"_108":118,"_54":110,"_111":32,"_112":110,"_113":50},{"_104":114,"_106":115,"_15":114,"_108":116,"_54":110,"_111":32,"_112":110,"_113":50},{"_104":105,"_106":107,"_15":105,"_108":109,"_54":110,"_111":32,"_112":110,"_113":50},"id","issues","name","Issues","filters",{},true,"saved","pinnable","count","tasks","Tasks",{},"Changes",{},{"_104":97,"_106":117,"_15":97,"_108":162,"_54":110,"_111":32,"_112":110},{"_104":114,"_106":115,"_15":114,"_108":161,"_54":110,"_111":32,"_112":110},{"_104":105,"_106":107,"_15":105,"_108":160,"_54":110,"_111":32,"_112":110},{"_104":157,"_106":158,"_15":157,"_108":159,"_54":32,"_111":32,"_112":110},{"_104":154,"_106":155,"_15":154,"_108":156,"_54":32,"_111":32,"_112":110},{"_104":92,"_106":152,"_15":84,"_108":153,"_54":32,"_111":32,"_112":110},{"_104":94,"_106":149,"_15":150,"_108":151,"_54":32,"_111":32,"_112":110},{"_104":90,"_106":147,"_15":90,"_108":148,"_54":32,"_111":32,"_112":110},{"_104":143,"_106":144,"_15":145,"_108":146,"_54":32,"_111":32,"_112":110},{"_104":80,"_106":141,"_15":80,"_108":142,"_54":32,"_111":32,"_112":32},{"_104":138,"_106":139,"_15":138,"_108":140,"_54":32,"_111":32,"_112":32},{"_104":135,"_106":136,"_15":135,"_108":137,"_54":32,"_111":32,"_112":32},{"_104":132,"_106":133,"_15":132,"_108":134,"_54":32,"_111":32,"_112":32},"decisions","Decisions",{},"changelog","Changelog",{},"roadmap","Roadmap",{},"Documents",{},"activity","Activity","insights",{},"Releases",{},"Deployments","automations/deployments",{},"Workflows",{},"discussions","Discussions",{},"goals","Goals",{},{},{},{},"groups","user","agents",[],"now",1794564144436,"sidebar","expanded"]