1
#!/usr/bin/env python3
2
 
3
import json
4
import logging
5
import os
6
import subprocess
7
from argparse import ArgumentParser, Namespace
8
from concurrent.futures import ThreadPoolExecutor, as_completed
9
from dataclasses import dataclass
10
from pathlib import Path
11
from sys import exit
12
from typing import Callable, Optional
13
 
14
# This file is symlinked into the internal repo as tools/format.py, so the root may be two levels up
15
# or three.
16
ROOT = Path(__file__).parents[1]
17
if not (ROOT / ".git").exists():
18
    ROOT = ROOT.parent
19
BAZEL_BIN = ROOT / "bazel-bin"
20
 
21
 
22
def parse_args() -> Namespace:
23
    parser = ArgumentParser()
24
    parser.add_argument(
25
        "--check",
26
        help="only check for files requiring formatting; don't actually format them",
27
        action="store_true",
28
        default=False,
29
    )
30
    subparsers = parser.add_subparsers(dest="subcommand")
31
    git_parser = subparsers.add_parser(
32
        "git", help="Apply format to changes tracked by git"
33
    )
34
    git_parser.add_argument(
35
        "--source",
36
        help=(
37
            "consider files modified in the specified commit-ish; "
38
            "if not specified, defaults to all changes in the working directory"
39
        ),
40
        type=str,
41
        required=False,
42
        default=None,
43
    )
44
    git_parser.add_argument(
45
        "--target",
46
        help="consider files modified since the specified commit-ish; defaults to HEAD",
47
        type=str,
48
        required=False,
49
        default="HEAD",
50
    )
51
    git_parser.add_argument(
52
        "--staged",
53
        help="consider files with staged modifications only",
54
        action="store_true",
55
        default=False,
56
    )
57
    options = parser.parse_args()
58
    if (
59
        options.subcommand == "git"
60
        and options.staged
61
        and (options.source is not None or options.target != "HEAD")
62
    ):
63
        logging.error(
64
            "--staged cannot be used with --source or --target; "
65
            "use --staged with --source=HEAD"
66
        )
67
        exit(1)
68
    return options
69
 
70
 
71
def filter_files_by_globs(
72
    files: list[Path], dir_path: Path, globs: tuple[str, ...], excludes: tuple[str, ...]
73
) -> list[Path]:
74
    return [
75
        file
76
        for file in files
77
        if file.is_relative_to(dir_path)
78
        and matches_any_glob(globs, file)
79
        and not relative_to_any(excludes, file)
80
    ]
81
 
82
 
83
def relative_to_any(excludes: tuple[str, ...], file: Path) -> bool:
84
    return any(file.is_relative_to(exclude) for exclude in excludes)
85
 
86
 
87
def matches_any_glob(globs: tuple[str, ...], file: Path) -> bool:
88
    return any(file.match(glob) for glob in globs)
89
 
90
 
91
def _ensure_bazel_tool(tool_name: str, build_target: str | None = None) -> Path:
92
    """Ensure a bazel-built formatter tool exists and return its path."""
93
    tool_suffix = Path("build") / "deps" / "formatters" / tool_name
94
    internal_tool_path = (
95
        BAZEL_BIN / "external" / "+local_repository+workerd" / tool_suffix
96
    )
97
    workerd_tool_path = BAZEL_BIN / tool_suffix
98
 
99
    if internal_tool_path.exists():
100
        return internal_tool_path
101
    if workerd_tool_path.exists():
102
        return workerd_tool_path
103
 
104
    # Tool not cached; build it once.
105
    if build_target is None:
106
        build_target = f"@workerd//build/deps/formatters:{tool_name}@rule"
107
    download_result = subprocess.run(["bazel", "build", build_target])
108
    if download_result.returncode != 0:
109
        raise RuntimeError(f"Failed to download {tool_name}")
110
 
111
    if internal_tool_path.exists():
112
        return internal_tool_path
113
    return workerd_tool_path
114
 
115
 
116
def run_bazel_tool(
117
    tool_name: str, args: list[str], build_target: str | None = None
118
) -> subprocess.CompletedProcess:
119
    tool_path = _ensure_bazel_tool(tool_name, build_target)
120
    return subprocess.run([tool_path, *args], cwd=ROOT)
121
 
122
 
123
def _run_parallel(
124
    run_fn: Callable, files: list[Path], cmd: list, max_workers: int = 16
125
) -> bool:
126
    """Split files across parallel invocations of run_fn(cmd + chunk)."""
127
    n_workers = min(os.cpu_count() or 1, len(files), max_workers)
128
    if n_workers <= 1:
129
        return run_fn(cmd + files).returncode == 0
130
 
131
    chunks = [files[i::n_workers] for i in range(n_workers)]
132
    with ThreadPoolExecutor(max_workers=n_workers) as pool:
133
        results = list(pool.map(lambda chunk: run_fn(cmd + chunk), chunks))
134
    return all(r.returncode == 0 for r in results)
135
 
136
 
137
def clang_format(files: list[Path], check: bool = False) -> bool:
138
    cmd = ["--dry-run", "--Werror"] if check else ["-i"]
139
    tool = _ensure_bazel_tool("clang-format")
140
    return _run_parallel(
141
        lambda args: subprocess.run([tool, *args], cwd=ROOT), files, cmd
142
    )
143
 
144
 
145
def prettier(files: list[Path], check: bool = False) -> bool:
146
    PRETTIER = BAZEL_BIN / "node_modules/prettier/bin/prettier.cjs"
147
 
148
    if not PRETTIER.exists():
149
        subprocess.run(["bazel", "build", "//:node_modules/prettier"])
150
    cmd = [PRETTIER, "--log-level=warn", "--check" if check else "--write"]
151
    return _run_parallel(
152
        lambda args: subprocess.run(args, cwd=ROOT), files, cmd, max_workers=8
153
    )
154
 
155
 
156
def buildifier(files: list[Path], check: bool = False) -> bool:
157
    cmd = ["--mode=check" if check else "--mode=fix"]
158
    result = run_bazel_tool("buildifier", cmd + files)
159
    return result.returncode == 0
160
 
161
 
162
def rustfmt(files: list[Path], check: bool = False) -> bool:
163
    if not files:
164
        return True
165
    cmd = ["--edition", "2024"]
166
    if check:
167
        cmd.append("--check")
168
    result = run_bazel_tool("rustfmt", cmd + files)
169
    return result.returncode == 0
170
 
171
 
172
def ruff(files: list[Path], check: bool = False) -> bool:
173
    if not files:
174
        return True
175
 
176
    cmd = ["check"]
177
    if not check:
178
        cmd.append("--fix")
179
    result1 = run_bazel_tool("ruff", cmd + files)
180
 
181
    # format
182
    cmd = ["format"]
183
    if check:
184
        cmd.append("--diff")
185
 
186
    result2 = run_bazel_tool("ruff", cmd + files)
187
    return result1.returncode == 0 and result2.returncode == 0
188
 
189
 
190
def git_get_modified_files(
191
    target: str, source: Optional[str], staged: bool
192
) -> list[Path]:
193
    if staged:
194
        files_in_diff = subprocess.check_output(
195
            ["git", "diff", "--diff-filter=d", "--name-only", "--cached"],
196
            encoding="utf-8",
197
            cwd=ROOT,
198
        ).splitlines()
199
        return [Path(file) for file in files_in_diff]
200
    else:
201
        merge_base = subprocess.check_output(
202
            ["git", "merge-base", target, source or "HEAD"],
203
            encoding="utf-8",
204
            cwd=ROOT,
205
        ).strip()
206
        files_in_diff = subprocess.check_output(
207
            ["git", "diff", "--diff-filter=d", "--name-only", merge_base]
208
            + ([source] if source else []),
209
            encoding="utf-8",
210
            cwd=ROOT,
211
        ).splitlines()
212
        return [Path(file) for file in files_in_diff]
213
 
214
 
215
def git_get_all_files() -> list[Path]:
216
    files = subprocess.check_output(
217
        ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
218
        encoding="utf-8",
219
        cwd=ROOT,
220
    ).splitlines()
221
    return [Path(file) for file in files]
222
 
223
 
224
@dataclass
225
class FormatConfig:
226
    directory: str
227
    globs: tuple[str, ...]
228
    formatter: str
229
    excludes: tuple[str, ...] = ()
230
 
231
 
232
FORMATTERS = {
233
    "clang-format": clang_format,
234
    "prettier": prettier,
235
    "ruff": ruff,
236
    "buildifier": buildifier,
237
    "rustfmt": rustfmt,
238
}
239
 
240
 
241
def format(config: FormatConfig, files: list[Path], check: bool) -> tuple[bool, str]:
242
    matching_files = filter_files_by_globs(
243
        files, Path(config.directory), config.globs, config.excludes
244
    )
245
 
246
    if not matching_files:
247
        return (
248
            True,
249
            f"No matching files for {config.directory} ({', '.join(config.globs)})",
250
        )
251
 
252
    result = FORMATTERS[config.formatter](matching_files, check)
253
    message = (
254
        f"{len(matching_files)} files in {config.directory} ({', '.join(config.globs)})"
255
    )
256
    return (
257
        result,
258
        f"{'Checked' if check else 'Formatted'} {message}",
259
    )
260
 
261
 
262
def main() -> None:
263
    options = parse_args()
264
 
265
    if options.subcommand == "git":
266
        files = git_get_modified_files(options.target, options.source, options.staged)
267
    else:
268
        files = git_get_all_files()
269
 
270
    with (Path(__file__).parent / "format.json").open() as fp:
271
        configs = json.load(fp, object_hook=lambda o: FormatConfig(**o))
272
 
273
    # Ensure all required tools are downloaded before running formatters in
274
    # parallel.  Otherwise a slow `bazel build` for a missing tool races with
275
    # the other formatters and its output gets interleaved.
276
    needed_formatters = set()
277
    for config in configs:
278
        matched = filter_files_by_globs(
279
            files, Path(config.directory), config.globs, config.excludes
280
        )
281
        if matched:
282
            needed_formatters.add(config.formatter)
283
    for name in needed_formatters:
284
        if name in ("clang-format", "buildifier", "ruff", "rustfmt"):
285
            _ensure_bazel_tool(name)
286
 
287
    all_ok = True
288
 
289
    with ThreadPoolExecutor() as executor:
290
        future_to_config = {
291
            executor.submit(format, config, files, options.check): config
292
            for config in configs
293
        }
294
        for future in as_completed(future_to_config):
295
            config = future_to_config[future]
296
            try:
297
                result, message = future.result()
298
                all_ok &= result
299
                logging.info(message)
300
            except Exception:
301
                logging.exception(
302
                    f"Formatter for {config.directory} generated an exception"
303
                )
304
                all_ok = False
305
 
306
    if not all_ok:
307
        logging.error(
308
            "Code has linting issues. Fix with python ./tools/cross/format.py"
309
        )
310
        exit(1)
311
 
312
 
313
if __name__ == "__main__":
314
    main()