// Compares tidy's CSS formatter with oxc's (oxc_formatter_css), Malva (the engine of dprint-plugin-malva), bun fmt // and Prettier, with lightningcss as a floor: it parses and prints CSS back, but doesn't format. Each corpus only // has the files that Prettier can parse. For each tool it shows the time, with all cores and with one, and how many // files it prints exactly as Prettier does. // // bun bench/css.ts [--runs ] [--oss ] import { $ } from "bun"; import { cpSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { dirname, extname, join, relative } from "node:path"; import * as prettier from "prettier"; let args = Bun.argv.slice(2); let option = (name: string) => (args.includes(name) ? args[args.indexOf(name) + 1] : undefined); let runs = Number(option("--runs") ?? 10); let oss = option("--oss") ?? join(homedir(), "repos", "oss"); let root = new URL("..", import.meta.url).pathname; let target = join(root, "target", "css"); await $`cargo build --release`.cwd(join(root, "bench/css")).env({ ...process.env, CARGO_TARGET_DIR: target }).quiet(); await $`cargo build --release -p tidy_cli`.cwd(root).env({ ...process.env, CARGO_TARGET_DIR: join(root, "target") }).quiet(); let tidy = join(root, "target/release/tidy"); let harness = join(target, "release/css-bench"); let parsers: Record = { ".css": "css", ".scss": "scss", ".less": "less" }; let corpus = join(tmpdir(), "tidy-css-bench"); rmSync(corpus, { recursive: true, force: true }); /** Where each corpus comes from: repositories and the extensions to take from them. */ let sources: Record = { frameworks: [{ repo: "bun", extensions: ["css"], path: "test/js/bun/css/files" }], css: ["vite", "svelte", "rolldown", "BrowserBox"].map((repo) => ({ repo, extensions: ["css"] })), scss: ["snowpack", "tastejs", "vite", "storybook"].map((repo) => ({ repo, extensions: ["scss"] })), less: ["bootstrap3", "antd4", "pwa", "vite", "taskr", "freshie"].map((repo) => ({ repo, extensions: ["less"] })), }; /** Prettier's output for each file of the corpus, by its path in the corpus. */ let expected = new Map(); for (let [name, list] of Object.entries(sources)) { for (let { repo, extensions, path } of list) { let patterns = extensions.map((extension) => `${path ? `${path}/` : ""}*.${extension}`); let files = (await $`git -C ${join(oss, repo)} ls-files -z -- ${patterns}`.nothrow().text()).split("\0").filter(Boolean); for (let file of files) { if (/node_modules|\/dist\/|\.min\.css$|(^|\/)\./.test(file)) continue; let source = await Bun.file(join(oss, repo, file)).text().catch(() => undefined); if (source === undefined) continue; let formatted = await prettier.format(source, { parser: parsers[extname(file)] }).catch(() => undefined); if (formatted === undefined) continue; let to = join(corpus, name, repo, file); mkdirSync(dirname(to), { recursive: true }); cpSync(join(oss, repo, file), to); expected.set(relative(corpus, to), formatted); } } } function size(path: string): [number, number] { if (statSync(path).isFile()) return [1, statSync(path).size]; return readdirSync(path) .map((name) => size(join(path, name))) .reduce(([files, bytes], [more, extra]) => [files + more, bytes + extra], [0, 0]); } let prettierBin = join(root, "node_modules/.bin/prettier"); let tools = (dir: string, name: string): Record => ({ tidy: `${tidy} ${dir} --check --no-config`, "tidy (1 thread)": `${tidy} ${dir} --check --no-config --threads 1`, oxc: `${harness} oxc ${dir}`, "oxc (1 thread)": `${harness} oxc ${dir} --threads 1`, malva: `${harness} malva ${dir}`, "malva (1 thread)": `${harness} malva ${dir} --threads 1`, ...(name === "scss" || name === "less" ? {} : { lightningcss: `${harness} lightningcss ${dir}`, "lightningcss (1 thread)": `${harness} lightningcss ${dir} --threads 1`, }), bun: `bun fmt --cwd=${dir} --check --no-config --no-editorconfig`, "bun (1 thread)": `bun fmt --cwd=${dir} --check --no-config --no-editorconfig --threads 1`, prettier: `${prettierBin} --check ${dir} --no-config --log-level silent`, }); /** How many files of the corpus a tool prints exactly as Prettier does. */ async function matches(tool: string, dir: string): Promise { if (tool === "prettier") return undefined; let out = join(corpus, "..", `tidy-css-out-${tool}`); rmSync(out, { recursive: true, force: true }); if (tool === "bun") { cpSync(dir, out, { recursive: true }); await $`bun fmt --cwd=${out} --write --no-config --no-editorconfig`.nothrow().quiet(); } else if (tool === "tidy") { cpSync(dir, out, { recursive: true }); await $`${tidy} ${out} --write --no-config`.nothrow().quiet(); } else { await $`${harness} ${tool} ${dir} --write ${out}`.nothrow().quiet(); } let same = 0; for (let [path, formatted] of expected) { if (!path.startsWith(relative(corpus, dir) + "/")) continue; let printed = await Bun.file(join(out, relative(relative(corpus, dir), path))).text().catch(() => undefined); if (printed === formatted) same++; } return same; } type Result = { command: string; mean: number; user: number; system: number }; let rows: string[] = []; for (let name of Object.keys(sources)) { let dir = join(corpus, name); let [files, bytes] = size(dir); let commands = tools(dir, name); let results: Record = {}; for (let order of [Object.keys(commands), Object.keys(commands).reverse()]) { let file = join(target, "bench.json"); let list = order.flatMap((tool) => ["--command-name", tool, commands[tool]]); await $`hyperfine --warmup 2 --runs ${runs} --ignore-failure --style none --export-json ${file} ${list}`.quiet(); let { results: measured } = (await Bun.file(file).json()) as { results: Result[] }; for (let result of measured) (results[result.command] ??= []).push(result); } let average = (values: number[]) => (values.reduce((sum, value) => sum + value, 0) / values.length) * 1000; rows.push(`\n${name}: ${files} files, ${(bytes / 1024).toFixed(0)} KB`); rows.push(` ${"tool".padEnd(24)} ${"wall".padStart(9)} ${"cpu".padStart(9)} ${"user".padStart(9)} same as Prettier`); for (let [tool, list] of Object.entries(results)) { let wall = average(list.map((it) => it.mean)); let cpu = average(list.map((it) => it.user + it.system)); let user = average(list.map((it) => it.user)); let same = tool.includes("(1 thread)") ? undefined : await matches(tool, dir); let column = same === undefined ? "" : `${same}/${files} (${((same / files) * 100).toFixed(0)}%)`; rows.push( ` ${tool.padEnd(24)} ${wall.toFixed(1).padStart(6)} ms ${cpu.toFixed(1).padStart(6)} ms ${user.toFixed(1).padStart(6)} ms ${column}`, ); } } console.log(rows.join("\n"));