// Times the JavaScript and TypeScript formatters against each other, and counts the files that each prints as Prettier // does. tidy doesn't format scripts yet; this bench measures the field. Each corpus has the files that Prettier parses. // Every tool runs with Prettier's defaults: oxfmt with a width of 80, Biome with spaces, and none formats embedded code. // // The tools: Prettier 3.9.9, oxfmt, Biome and bun fmt. Each set runs in both orders; Prettier runs fewer times. // The results go to bench/results/format/js.md and ts.md. // // bun bench/js.ts [--runs ] [--oss ] [--only js,ts] import { $ } from "bun"; import { cpSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { dirname, join, relative } from "node:path"; import * as prettierApi from "prettier"; import { record, version } from "./results"; let args = Bun.argv.slice(2); let option = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : undefined); let runs = Number(option("--runs") ?? 5); let oss = option("--oss") ?? join(homedir(), "repos", "oss"); let only = option("--only")?.split(","); let root = new URL("..", import.meta.url).pathname; let bin = (name: string) => join(root, "node_modules/.bin", name); let repos = ["vite", "storybook", "rolldown", "BrowserBox"]; let extensions: Record = { js: ["js", "jsx", "mjs", "cjs"], ts: ["ts", "tsx", "mts", "cts"], }; let base = join(tmpdir(), "tidy-bench-js"); rmSync(base, { recursive: true, force: true }); let config = join(base, "oxfmtrc.json"); await Bun.write(config, JSON.stringify({ printWidth: 80, embeddedLanguageFormatting: "off" })); /** The commands that check a folder, and the ones that format it in place. */ let commands = (dir: string, write: boolean): Record => { let check = write ? "" : "--check"; return { prettier: `${bin("prettier")} ${write ? "--write" : "--check"} ${dir} --no-config --embedded-language-formatting off --log-level silent`, oxfmt: `${bin("oxfmt")} ${check} -c ${config} --disable-nested-config ${dir}`, "oxfmt (1 thread)": `${bin("oxfmt")} ${check} -c ${config} --disable-nested-config --threads=1 ${dir}`, biome: `${bin("biome")} format ${write ? "--write" : ""} --indent-style=space --max-diagnostics=0 ${dir}`, bun: `bun fmt --cwd=${dir} ${check} --no-editorconfig --no-script --no-config --embedded-language-formatting off`, }; }; let versions: Record = { prettier: `Prettier ${prettierApi.version}`, oxfmt: `oxfmt ${await version([bin("oxfmt"), "--version"])}`, biome: `Biome ${await version([bin("biome"), "--version"])}`, bun: `bun fmt ${await version(["bun", "--version"])}`, }; for (let name of Object.keys(extensions).filter((name) => !only || only.includes(name))) { let corpus = join(base, name); await gather(corpus, extensions[name]); let [files, bytes] = size(corpus); let check = commands(corpus, false); let fast = Object.keys(check).filter((tool) => tool !== "prettier"); let means = { ...(await time(fast, check, runs)), ...(await time(["prettier"], check, Math.min(runs, 2))) }; let matches = await conformance(corpus, Object.keys(versions)); let fastest = Math.min(...Object.values(means).map(average)); let sorted = Object.entries(means).sort(([, left], [, right]) => average(left) - average(right)); let rows = sorted.map(([tool, values]) => { let match = matches[tool.replace(" (1 thread)", "")]; let share = tool === "prettier" ? "reference" : `${match} (${((match / files) * 100).toFixed(1)}%)`; return [tool, `${average(values).toFixed(0)} ms`, `${(average(values) / fastest).toFixed(2)}x`, share]; }); console.log(`\n${name}: ${files} files`); for (let row of rows) console.log(` ${row.join(" ")}`); let summary = `${files} files, ${(bytes / 1024 / 1024).toFixed(1)} MB, from ${repos.join(", ")}: the files that Prettier parses. Each tool checks every file; a time is the mean of ${runs} runs in each order, or ${Math.min(runs, 2)} for Prettier. "Same as Prettier" counts the files that a tool formats to Prettier's output.`; let head = ["Tool", "Time", "Against fastest", "Same as Prettier"]; let title = `Formatters: ${name === "js" ? "JavaScript" : "TypeScript"}`; await record(`format/${name}.md`, title, [{ summary, head, rows }], Object.values(versions)); } /** Copies the files of the repos with `list` extensions that Prettier parses into `corpus`. */ async function gather(corpus: string, list: string[]) { for (let repo of repos) { let patterns = list.map((extension) => `*.${extension}`); let listed = await $`git -C ${join(oss, repo)} ls-files -z -- ${patterns}`.nothrow().text(); for (let file of listed.split("\0").filter(Boolean)) { if (/node_modules|\/dist\/|\/build\/|(^|\/)\.|\.min\.js$|fixtures?\/|__snapshots__|\/vendor\//.test(file)) continue; let from = join(oss, repo, file); let source = await Bun.file(from).text().catch(() => undefined); if (source === undefined) continue; let parsed = await prettierApi.format(source, { filepath: from }).then(() => true, () => false); if (!parsed) continue; let to = join(corpus, repo, file); mkdirSync(dirname(to), { recursive: true }); cpSync(from, to); } } } function size(path: string): [number, number] { if (statSync(path).isFile()) return [1, statSync(path).size]; return readdirSync(path) .map((name) => size(join(path, name))) .reduce(([files, bytes], [more, extra]) => [files + more, bytes + extra], [0, 0]); } function list(path: string): string[] { if (statSync(path).isFile()) return [path]; return readdirSync(path).flatMap((name) => list(join(path, name))); } type Result = { command: string; mean: number }; async function time(names: string[], all: Record, count: number) { let means: Record = {}; for (let order of [names, names.toReversed()]) { let file = join(base, "hyperfine.json"); let flags = order.flatMap((tool) => ["--command-name", tool, all[tool]]); await $`hyperfine --warmup 1 --runs ${count} --ignore-failure --style none --export-json ${file} ${flags}`.quiet(); let { results } = (await Bun.file(file).json()) as { results: Result[] }; for (let result of results) (means[result.command] ??= []).push(result.mean); } return means; } /** Formats a copy of the corpus with each tool, and counts the files equal to Prettier's copy. */ async function conformance(corpus: string, tools: string[]) { let outputs: Record> = {}; for (let tool of tools) { let copy = `${corpus}-${tool}`; rmSync(copy, { recursive: true, force: true }); cpSync(corpus, copy, { recursive: true }); await $`sh -c ${commands(copy, true)[tool]}`.nothrow().quiet(); let texts = await Promise.all(list(copy).map(async (path) => [relative(copy, path), await Bun.file(path).text()])); outputs[tool] = new Map(texts as [string, string][]); } let reference = outputs.prettier; let matches: Record = {}; for (let tool of tools) { matches[tool] = [...reference].filter(([path, text]) => outputs[tool].get(path) === text).length; } return matches; } function average(values: number[]) { return (values.reduce((sum, value) => sum + value, 0) / values.length) * 1000; }