// Times the JavaScript and TypeScript formatters against each other, and counts the files that each prints as Prettier // does. Each corpus has the files that Prettier parses. Every tool runs with Prettier's defaults: oxfmt with a width of // 80, Biome with spaces, tidy without JSDoc formatting, and none formats embedded code. // // The tools: tidy, Prettier 3.9.9, oxfmt, Biome and bun fmt. Each set runs in both orders; Prettier runs fewer times. // The results go to bench/results/format/js.md and ts.md. // // bun bench/js.ts [--runs ] [--oss ] [--only js,ts] import { $ } from "bun"; import { cpSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { dirname, join, relative } from "node:path"; import * as prettierApi from "prettier"; import { record, version } from "./results"; let args = Bun.argv.slice(2); let option = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : undefined); let runs = Number(option("--runs") ?? 5); let oss = option("--oss") ?? join(homedir(), "repos", "oss"); let only = option("--only")?.split(","); let root = new URL("..", import.meta.url).pathname; let bin = (name: string) => join(root, "node_modules/.bin", name); let target = join(root, "target"); await $`cargo build --release -p tidy_cli`.cwd(root).env({ ...process.env, CARGO_TARGET_DIR: target }).quiet(); let repos = ["vite", "storybook", "rolldown", "BrowserBox"]; let extensions: Record = { js: ["js", "jsx", "mjs", "cjs"], ts: ["ts", "tsx", "mts", "cts"], }; let base = join(tmpdir(), "tidy-bench-js"); rmSync(base, { recursive: true, force: true }); let config = join(base, "oxfmtrc.json"); await Bun.write(config, JSON.stringify({ printWidth: 80, embeddedLanguageFormatting: "off" })); let tidyConfig = join(base, "tidy.json"); await Bun.write(tidyConfig, JSON.stringify({ embedded: false, js: { jsdoc: false } })); /** The commands that check a folder, and the ones that format it in place. */ let commands = (dir: string, write: boolean): Record => { let check = write ? "" : "--check"; return { tidy: `${target}/release/tidy ${dir} ${write ? "--write" : "--check"} --config ${tidyConfig}`, "tidy (1 thread)": `${target}/release/tidy ${dir} ${write ? "--write" : "--check"} --config ${tidyConfig} --threads 1`, prettier: `${bin("prettier")} ${write ? "--write" : "--check"} ${dir} --no-config --embedded-language-formatting off --log-level silent`, oxfmt: `${bin("oxfmt")} ${check} -c ${config} --disable-nested-config ${dir}`, "oxfmt (1 thread)": `${bin("oxfmt")} ${check} -c ${config} --disable-nested-config --threads=1 ${dir}`, biome: `${bin("biome")} format ${write ? "--write" : ""} --indent-style=space --max-diagnostics=0 ${dir}`, bun: `bun fmt --cwd=${dir} ${check} --no-editorconfig --no-script --no-config --embedded-language-formatting off`, }; }; let versions: Record = { tidy: "tidy at the commit above", prettier: `Prettier ${prettierApi.version}`, oxfmt: `oxfmt ${await version([bin("oxfmt"), "--version"])}`, biome: `Biome ${await version([bin("biome"), "--version"])}`, bun: `bun fmt ${await version(["bun", "--version"])}`, }; for (let name of Object.keys(extensions).filter((name) => !only || only.includes(name))) { let corpus = join(base, name); await gather(corpus, extensions[name]); let [files, bytes] = size(corpus); let check = commands(corpus, false); let fast = Object.keys(check).filter((tool) => tool !== "prettier"); let means = { ...(await time(fast, check, runs)), ...(await time(["prettier"], check, Math.min(runs, 2))) }; let matches = await conformance(corpus, Object.keys(versions)); let fastest = Math.min(...Object.values(means).map(average)); let sorted = Object.entries(means).sort(([, left], [, right]) => average(left) - average(right)); let rows = sorted.map(([tool, values]) => { let match = matches[tool.replace(" (1 thread)", "")]; let share = tool === "prettier" ? "reference" : `${match} (${((match / files) * 100).toFixed(1)}%)`; return [tool, `${average(values).toFixed(0)} ms`, `${(average(values) / fastest).toFixed(2)}x`, share]; }); console.log(`\n${name}: ${files} files`); for (let row of rows) console.log(` ${row.join(" ")}`); let summary = `${files} files, ${(bytes / 1024 / 1024).toFixed(1)} MB, from ${repos.join(", ")}: the files that Prettier parses. Each tool checks every file; a time is the mean of ${runs} runs in each order, or ${Math.min(runs, 2)} for Prettier. "Same as Prettier" counts the files that a tool formats to Prettier's output. With one thread, tidy reads and formats the files in turn, while oxfmt still reads them on another thread, so its time doesn't include all of its reading.`; let head = ["Tool", "Time", "Against fastest", "Same as Prettier"]; let title = `Formatters: ${name === "js" ? "JavaScript" : "TypeScript"}`; await record(`format/${name}.md`, title, [{ summary, head, rows }], Object.values(versions)); } /** Copies the files of the repos with `list` extensions that Prettier parses into `corpus`. */ async function gather(corpus: string, list: string[]) { for (let repo of repos) { let patterns = list.map((extension) => `*.${extension}`); let listed = await $`git -C ${join(oss, repo)} ls-files -z -- ${patterns}`.nothrow().text(); for (let file of listed.split("\0").filter(Boolean)) { if (/node_modules|\/dist\/|\/build\/|(^|\/)\.|\.min\.js$|fixtures?\/|__snapshots__|\/vendor\//.test(file)) continue; let from = join(oss, repo, file); let source = await Bun.file(from).text().catch(() => undefined); if (source === undefined) continue; let parsed = await prettierApi.format(source, { filepath: from }).then(() => true, () => false); if (!parsed) continue; let to = join(corpus, repo, file); mkdirSync(dirname(to), { recursive: true }); cpSync(from, to); } } } function size(path: string): [number, number] { if (statSync(path).isFile()) return [1, statSync(path).size]; return readdirSync(path) .map((name) => size(join(path, name))) .reduce(([files, bytes], [more, extra]) => [files + more, bytes + extra], [0, 0]); } function list(path: string): string[] { if (statSync(path).isFile()) return [path]; return readdirSync(path).flatMap((name) => list(join(path, name))); } type Result = { command: string; mean: number }; async function time(names: string[], all: Record, count: number) { let means: Record = {}; for (let order of [names, names.toReversed()]) { let file = join(base, "hyperfine.json"); let flags = order.flatMap((tool) => ["--command-name", tool, all[tool]]); await $`hyperfine --warmup 1 --runs ${count} --ignore-failure --style none --export-json ${file} ${flags}`.quiet(); let { results } = (await Bun.file(file).json()) as { results: Result[] }; for (let result of results) (means[result.command] ??= []).push(result.mean); } return means; } /** Formats a copy of the corpus with each tool, and counts the files equal to Prettier's copy. */ async function conformance(corpus: string, tools: string[]) { let outputs: Record> = {}; for (let tool of tools) { let copy = `${corpus}-${tool}`; rmSync(copy, { recursive: true, force: true }); cpSync(corpus, copy, { recursive: true }); await $`sh -c ${commands(copy, true)[tool]}`.nothrow().quiet(); let texts = await Promise.all(list(copy).map(async (path) => [relative(copy, path), await Bun.file(path).text()])); outputs[tool] = new Map(texts as [string, string][]); } let reference = outputs.prettier; let matches: Record = {}; for (let tool of tools) { matches[tool] = [...reference].filter(([path, text]) => outputs[tool].get(path) === text).length; } return matches; } function average(values: number[]) { return (values.reduce((sum, value) => sum + value, 0) / values.length) * 1000; }