Skip to content
rachel/workerd
File

Blob: build/deps/update-deps.py

python672 lines
1#!/usr/bin/python3
2"""
3Usage: update-deps.py [dep_name]
4"""
5 
6import base64
7import datetime
8import hashlib
9import io
10import json
11import os
12import re
13import subprocess
14import sys
15import tarfile
16import urllib.request
17import zipfile
18from pathlib import Path
19 
20TARGET_FILTER = None if len(sys.argv) < 2 else sys.argv[1]
21 
22SCRIPT_DIR = Path(__file__).parent
23if "BUILD_WORKSPACE_DIRECTORY" in os.environ:
24    SCRIPT_DIR = Path(os.environ["BUILD_WORKSPACE_DIRECTORY"]) / "build" / "deps"
25 
26GEN_DIR = SCRIPT_DIR / "gen"
27ALL_DEPS = ["deps.jsonc", "build_deps.jsonc", "shared_deps.jsonc"]
28 
29 
30TOP = """# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT
31
32http = use_extension("@//:build/exts/http.bzl", "http")
33
34git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
35
36"""
37 
38GITHUB_TAR_URL_TEMPLATE = "https://github.com/{owner}/{repo}/tarball/{commit}"
39 
40GITHUB_RELEASE_FILE_URL_TEMPLATE = (
41    "https://github.com/{owner}/{repo}/releases/download/v{version}/{file}"
42)
43 
44EXT_DEP_TEMPLATE = """# {name}
45{ext_name}.{rule_name}({attrs}
46)
47use_repo({ext_name}, "{name}")
48
49"""
50 
51 
52REPO_RULE_DEP_TEMPLATE = """# {name}
53{rule_name}({attrs}
54)
55
56"""
57 
58 
59BAZEL_DEP_TEMPLATE = """# {name}
60bazel_dep({attrs})
61
62"""
63 
64BAZEL_DEP_OVERRIDE_TEMPLATE = """# {name}
65bazel_dep({bazel_dep_attrs})
66{override_type}({override_attrs}
67)
68
69"""
70 
71GITHUB_ACCESS_TOKEN = ""
72 
73 
74def format_attr_list(attrs, single_line=False):
75    if not attrs:
76        return ""
77 
78    # buildifier (Bazel build file formatter) requires keys to be sorted, except name and module_name go first
79    attr_list = sorted(
80        attrs.items(), key=lambda kv: "" if kv[0] in {"name", "module_name"} else kv[0]
81    )
82    attr_strs = (f"{k} = {format_attr(v)}" for k, v in attr_list)
83 
84    if single_line:
85        # Print attributes on a single line (for short declarations)
86        return ", ".join(attr_strs)
87    else:
88        # Print one attribute per line (for long declarations)
89        return "\n" + "\n".join(f"    {kv}," for kv in attr_strs)
90 
91 
92def format_attr(v):
93    if isinstance(v, (bool, int)):
94        return str(v)
95    elif isinstance(v, list):
96        # Format lists as multiline with proper indentation and trailing comma
97        return json.dumps(v, indent=8).replace("\n]", ",\n    ]")
98    else:
99        return json.dumps(v)
100 
101 
102def format_repo_rule_dep(repo, rule_name, attrs):
103    return REPO_RULE_DEP_TEMPLATE.format(
104        rule_name=rule_name,
105        name=repo["name"],
106        attrs=format_attr_list(repo_attributes(repo) | attrs),
107    )
108 
109 
110def format_ext_dep(repo, ext_name, rule_name, attrs):
111    return EXT_DEP_TEMPLATE.format(
112        ext_name=ext_name,
113        rule_name=rule_name,
114        name=repo["name"],
115        attrs=format_attr_list(repo_attributes(repo) | attrs),
116    )
117 
118 
119class RateLimitedException(Exception):
120    pass
121 
122 
123class AssetsException(Exception):
124    pass
125 
126 
127class UnsupportedException(Exception):
128    pass
129 
130 
131def github_urlopen(url):
132    """
133    A wrapper around urllib.request.urlopen() which parses GitHub rate limit errors and
134    provides a more human-friendly explanation.
135    """
136    if GITHUB_ACCESS_TOKEN != "":
137        url = urllib.request.Request(url)
138        url.add_header("Authorization", f"Bearer {GITHUB_ACCESS_TOKEN}")
139    try:
140        return urllib.request.urlopen(url)
141    except urllib.error.HTTPError as e:
142        reset_ts = e.headers["x-ratelimit-reset"]
143        if e.code != 403 or not reset_ts:
144            raise
145        reset_dt = datetime.datetime.fromtimestamp(int(reset_ts))
146        reset_iso_utc = reset_dt.astimezone(datetime.UTC).isoformat(" ")
147        reset_iso_local = reset_dt.isoformat(" ")
148        raise RateLimitedException(
149            f"""
150We have been rate-limited by GitHub. We can make API calls again at:
151  {reset_iso_utc} UTC ({reset_iso_local} local time).
152"""
153            + """
154You can try re-running the script and specifying an access token since authenticated
155GitHub API requests have a higher rate limit.
156"""
157            if GITHUB_ACCESS_TOKEN == ""
158            else ""
159        ) from e
160 
161 
162def github_last_commit(repo):
163    owner = repo["owner"]
164    github_repo = repo["repo"]
165    branch = repo.get("branch", "master")
166    api_url = f"https://api.github.com/repos/{owner}/{github_repo}/commits/{branch}"
167    commits = json.loads(github_urlopen(api_url).read())
168    return commits["sha"]
169 
170 
171def get_url_content_sha256(url):
172    return hashlib.sha256(urllib.request.urlopen(url).read()).hexdigest()
173 
174 
175def get_bcr_module_bazel_url(module_name, version):
176    """Generate the URL for a MODULE.bazel file from BCR."""
177    return f"https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/refs/heads/main/modules/{module_name}/{version}/MODULE.bazel"
178 
179 
180def get_bcr_module_bazel_integrity(module_name, version):
181    """Fetch MODULE.bazel from BCR and compute its SHA256 integrity hash."""
182    url = get_bcr_module_bazel_url(module_name, version)
183    content = urllib.request.urlopen(url).read()
184    sha256 = hashlib.sha256(content).digest()
185    return f"sha256-{base64.b64encode(sha256).decode()}"
186 
187 
188def repo_attributes(repo):
189    repo_attrs = {}
190 
191    for option in (
192        "name",
193        "build_file",
194        "repo_mapping",
195        "downloaded_file_path",
196        "build_file_content",
197        "patches",
198    ):
199        if option in repo:
200            repo_attrs[option] = repo[option]
201 
202    if "patches" in repo_attrs:
203        repo_attrs["patch_strip"] = 1
204 
205    if "use_module_bazel_from_bcr" in repo:
206        url = get_bcr_module_bazel_url(repo["name"], repo["use_module_bazel_from_bcr"])
207        integrity = get_bcr_module_bazel_integrity(
208            repo["name"], repo["use_module_bazel_from_bcr"]
209        )
210        repo_attrs["remote_file_urls"] = {"MODULE.bazel": [url]}
211        repo_attrs["remote_file_integrity"] = {"MODULE.bazel": integrity}
212 
213    return repo_attrs
214 
215 
216def format_bazel_dep_with_override(repo, override_type, override_attrs):
217    """Format bazel_dep + override."""
218    name = repo["name"]
219 
220    # bazel_dep attributes
221    bazel_dep_attrs = {"name": name}
222    if "repo_name" in repo:
223        bazel_dep_attrs["repo_name"] = repo["repo_name"]
224 
225    # Override attributes - use repo_attributes but swap name for module_name
226    base_attrs = repo_attributes(repo)
227    base_attrs["module_name"] = name
228    del base_attrs["name"]
229 
230    return BAZEL_DEP_OVERRIDE_TEMPLATE.format(
231        name=name,
232        bazel_dep_attrs=format_attr_list(bazel_dep_attrs, single_line=True),
233        override_type=override_type,
234        override_attrs=format_attr_list(base_attrs | override_attrs),
235    )
236 
237 
238def gen_github_tarball(repo):
239    owner = repo["owner"]
240    github_repo = repo["repo"]
241 
242    commit = github_last_commit(repo)
243    if "freeze_commit" in repo:
244        if repo["freeze_commit"] != commit:
245            print(
246                "frozen, update available ",
247                repo["freeze_commit"][:7],
248                " -> ",
249                commit[:7],
250                end="",
251            )
252        commit = repo["freeze_commit"]
253    else:
254        print(commit[:7], end="")
255 
256    prefix = f"{owner}-{github_repo}-{commit[:7]}"
257    if "extra_strip_prefix" in repo:
258        prefix = prefix + repo["extra_strip_prefix"]
259 
260    url = GITHUB_TAR_URL_TEMPLATE.format(
261        owner=owner,
262        repo=github_repo,
263        commit=commit,
264    )
265 
266    if "freeze_sha256" in repo:
267        sha256 = repo["freeze_sha256"]
268    else:
269        sha256 = get_url_content_sha256(url)
270 
271    attrs = dict(
272        url=url,
273        strip_prefix=prefix,
274        sha256=sha256,
275        type="tgz",
276    )
277 
278    if repo.get("use_bazel_dep"):
279        return format_bazel_dep_with_override(
280            repo,
281            override_type="archive_override",
282            override_attrs=attrs,
283        )
284    else:
285        return format_ext_dep(
286            repo,
287            ext_name="http",
288            rule_name="archive",
289            attrs=attrs,
290        )
291 
292 
293def github_last_release(repo):
294    owner = repo["owner"]
295    github_repo = repo["repo"]
296    api_url = f"https://api.github.com/repos/{owner}/{github_repo}/releases/latest"
297    return json.loads(github_urlopen(api_url).read())
298 
299 
300def github_release(repo, tag_name):
301    owner = repo["owner"]
302    github_repo = repo["repo"]
303    api_url = (
304        f"https://api.github.com/repos/{owner}/{github_repo}/releases/tags/{tag_name}"
305    )
306    return json.loads(github_urlopen(api_url).read())
307 
308 
309def gen_github_release(repo):
310    try:
311        release = github_last_release(repo)
312    except urllib.error.HTTPError as e:
313        # If a repo only has pre-releases, github_last_release will throw a 404 error.
314        # In that case, we must specify a "freeze_version".
315        if e.code != 404 or "freeze_version" not in repo:
316            raise
317        release = None
318 
319    if "freeze_version" in repo:
320        frozen_release = github_release(repo, repo["freeze_version"])
321        if release is not None and frozen_release["tag_name"] != release["tag_name"]:
322            print(
323                "frozen, update available: {} -> {}".format(
324                    frozen_release["tag_name"], release["tag_name"]
325                ),
326                end="",
327            )
328        release = frozen_release
329    else:
330        print(release["tag_name"], end="")
331 
332    if "file_regex" in repo:
333        # Using file_regex to select a user-uploaded asset
334        url = get_release_asset(repo, release)
335    else:
336        # Using Github-generated tarball
337        url = release["tarball_url"]
338 
339    type = "tgz"
340    if url.endswith(".zip"):
341        type = "zip"
342    elif url.endswith(".xz"):
343        type = "xz"
344    elif url.endswith(".tar.bz2"):
345        type = "tar.bz2"
346 
347    content = urllib.request.urlopen(url).read()
348 
349    if "freeze_sha256" in repo:
350        sha256 = repo["freeze_sha256"]
351    else:
352        sha256 = hashlib.sha256(content).hexdigest()
353 
354    file_type = repo.get("file_type", "archive")
355    if file_type == "archive":
356        if "strip_prefix" in repo:
357            prefix = repo["strip_prefix"]
358        elif url.endswith(".zip"):
359            with zipfile.ZipFile(io.BytesIO(content)) as zip:
360                prefix = os.path.commonprefix(zip.namelist())
361        else:
362            with tarfile.open(fileobj=io.BytesIO(content)) as tgz:
363                prefix = os.path.commonprefix(tgz.getnames())
364 
365        attrs = dict(
366            url=url,
367            strip_prefix=prefix,
368            sha256=sha256,
369            type=type,
370        )
371 
372        if repo.get("use_bazel_dep"):
373            return format_bazel_dep_with_override(
374                repo,
375                override_type="archive_override",
376                override_attrs=attrs,
377            )
378        else:
379            return format_ext_dep(
380                repo,
381                ext_name="http",
382                rule_name="archive",
383                attrs=attrs,
384            )
385    elif file_type == "executable":
386        if repo.get("use_bazel_dep"):
387            raise UnsupportedException(
388                "use_bazel_dep is not supported for executable file_type"
389            )
390        return format_ext_dep(
391            repo,
392            ext_name="http",
393            rule_name="file",
394            attrs=dict(url=url, sha256=sha256, executable=True),
395        )
396    else:
397        raise UnsupportedException("Unsupported file_type: " + file_type)
398 
399 
400def get_release_asset(repo, release):
401    file_regex = re.compile(repo["file_regex"])
402    assets = [a for a in release["assets"] if file_regex.match(a["name"])]
403 
404    if len(assets) == 0:
405        raise AssetsException("No assets found: " + json.dumps(release))
406 
407    if len(assets) > 1:
408        raise AssetsException(
409            "Too many assets, use more specific file_regex: "
410            + str([a["name"] for a in assets])
411        )
412 
413    return assets[0]["browser_download_url"]
414 
415 
416def gen_git_clone(repo):
417    url = repo["url"]
418 
419    # We used to clone the repository here to get a shallow_since timestamp, but based
420    # on # https://github.com/bazelbuild/bazel/issues/12857 it is unclear if this is
421    # actually helpful.
422    ls_remote = subprocess.run(
423        ["git", "ls-remote", url, repo["branch"]], capture_output=True, text=True
424    )
425    ls_remote.check_returncode()
426    commit = ls_remote.stdout.strip().split()[0]
427 
428    if "freeze_commit" in repo:
429        freeze_commit = repo["freeze_commit"]
430        if freeze_commit != commit:
431            print(
432                "frozen, update available ",
433                repo["freeze_commit"][:7],
434                " -> ",
435                commit[:7],
436                end="",
437            )
438            commit = freeze_commit
439        else:
440            print(commit[:7], end="")
441 
442    attrs = dict(
443        remote=url,
444        commit=commit,
445    )
446 
447    if repo.get("use_bazel_dep"):
448        return format_bazel_dep_with_override(
449            repo,
450            override_type="git_override",
451            override_attrs=attrs,
452        )
453    else:
454        return format_repo_rule_dep(
455            repo,
456            rule_name="git_repository",
457            attrs=attrs,
458        )
459 
460 
461def get_bcr_version(name: str) -> str:
462    module_versions_url = f"https://bcr.bazel.build/modules/{name}/metadata.json"
463    with urllib.request.urlopen(module_versions_url) as res:
464        meta = json.load(res)
465 
466        # Assuming the newer versions are appended to the end of the list
467        for version in reversed(meta["versions"]):
468            # Do not recommend a yanked version
469            if version in meta["yanked_versions"]:
470                continue
471 
472            # Make a best efforts attempt to exclude pre-releases
473            if re.search(r"(beta|rc|alpha|dev)", version):
474                continue
475 
476            # TODO: Consider parsing versions with `packaging`. However, there is no standardized
477            # version scheme for BCR dependencies, so this would definitely fail on some deps.
478 
479            return version
480 
481 
482def gen_bazel_dep(repo):
483    name = repo["name"]
484 
485    latest_version = get_bcr_version(name)
486 
487    if "freeze_version" in repo:
488        frozen_version = repo["freeze_version"]
489        if frozen_version != latest_version:
490            print(
491                f"frozen, update available: {frozen_version} -> {latest_version}",
492                end="",
493            )
494        version = frozen_version
495    else:
496        print(latest_version, end="")
497        version = latest_version
498 
499    if "patches" in repo:
500        return BAZEL_DEP_OVERRIDE_TEMPLATE.format(
501            name=name,
502            bazel_dep_attrs=format_attr_list(
503                dict(name=name, version=version), single_line=True
504            ),
505            override_type="single_version_override",
506            override_attrs=format_attr_list(
507                {
508                    "module_name": name,
509                    "patch_strip": 1,
510                    "patches": repo["patches"],
511                }
512            ),
513        )
514 
515    return BAZEL_DEP_TEMPLATE.format(
516        name=name,
517        attrs=format_attr_list(dict(name=name, version=version), single_line=True),
518    )
519 
520 
521def gen_repo_str(repo):
522    if repo["type"] == "github_tarball":
523        return gen_github_tarball(repo)
524    elif repo["type"] == "github_release":
525        return gen_github_release(repo)
526    elif repo["type"] == "git_clone":
527        return gen_git_clone(repo)
528    elif repo["type"] == "bazel_dep":
529        return gen_bazel_dep(repo)
530    else:
531        raise UnsupportedException(f"Unsupported repo type: {repo['type']}")
532 
533 
534def gen_repo_bzl(repo, current_dep):
535    """Generate and return the content for a repository dependency."""
536    print("Checking", repo["name"], "... ", end="", flush=True)
537    if TARGET_FILTER is not None and not repo["name"].startswith(TARGET_FILTER):
538        print("skipped")
539        return current_dep
540 
541    content = gen_repo_str(repo)
542    print()
543    return content
544 
545 
546def gen_deps_bzl(repo_contents, deps_bzl):
547    """Generate the index file by concatenating all repository contents."""
548    deps_bzl_content = TOP
549 
550    # Concatenate all repository contents
551    for content in repo_contents:
552        if content:
553            deps_bzl_content += content
554 
555    with deps_bzl.open("w") as f:
556        # Strip extra trailing newline but keep one for POSIX compliance
557        f.write(deps_bzl_content.rstrip("\n") + "\n")
558 
559 
560def process_deps(deps, current_deps, deps_bzl):
561    # Sort repositories by name for consistent ordering (buildifier preference)
562    sorted_repos = sorted(deps["repositories"], key=lambda r: r["name"])
563 
564    # Generate content for each repository
565    repo_contents = []
566    for repo in sorted_repos:
567        content = gen_repo_bzl(repo, current_deps.get(repo["name"]))
568        repo_contents.append(content)
569 
570    gen_deps_bzl(repo_contents, deps_bzl)
571 
572 
573def split_bzl_file(file: Path) -> dict[str, str]:
574    # Creates a map from dependency name to generated code for that dep
575    deps = {}
576    text = file.read_text()
577 
578    blocks = iter(re.finditer(r"^# (.*)$", text, re.MULTILINE))
579    a = next(blocks)
580 
581    for b in blocks:
582        # Key: dependency name from comment line
583        # Value: generated code (all text until the next comment line)
584        deps[a.groups()[0]] = text[a.start() : b.start()]
585        a = b
586 
587    deps[a.groups()[0]] = text[a.start() :]
588 
589    return deps
590 
591 
592def strip_comments(text):
593    # capture string literals first, comments send
594    regex = re.compile(r"(\".*\")|(//.*$)", re.MULTILINE)
595    return regex.sub(
596        lambda match: "" if match.group(2) is not None else match.group(1), text
597    )
598 
599 
600def read_access_token():
601    if not sys.stdin.isatty():
602        return ""
603 
604    # 1. Try to obtain token from the gh tool
605    try:
606        res = subprocess.run(["gh", "auth", "token"], capture_output=True)
607        if res.returncode == 0:
608            return res.stdout.decode().strip()
609        else:
610            # User has gh but is not logged in
611            print("Please log in to Github")
612            print("$ gh auth login")
613            raise SystemExit
614    except FileNotFoundError:
615        pass  # User does not have gh tool installed
616 
617        print(
618            """Follow these steps to obtain a GitHub API access token with
619appropriate permissions:
620
6211. On github.com, go to
622Settings > Developer Settings > Personal access tokens > Fine-grained tokens.
6232. Generate a new token with default settings.
624
625Alternatively, install the gh CLI tool to save time <https://github.com/cli/cli#installation>.
626"""
627        )
628        print(
629            "Please enter GitHub API access token (or empty to skip): ",
630            end="",
631            flush=True,
632        )
633 
634        return sys.stdin.readline().strip("\n")
635 
636 
637def process_config(deps_file):
638    deps_path = SCRIPT_DIR / deps_file
639    bzl_path = GEN_DIR / Path(deps_file).with_suffix(".MODULE.bazel").name
640 
641    # Create output directory if it doesn't exist
642    GEN_DIR.mkdir(parents=True, exist_ok=True)
643 
644    # Load existing generated deps file (if any)
645    try:
646        current_deps = split_bzl_file(bzl_path)
647    except FileNotFoundError:
648        current_deps = {}
649 
650    try:
651        with deps_path.open() as fp:
652            json_text = strip_comments(fp.read())
653            process_deps(json.loads(json_text), current_deps, bzl_path)
654    except FileNotFoundError:
655        pass
656 
657 
658def run():
659    if TARGET_FILTER is None:
660        global GITHUB_ACCESS_TOKEN
661        GITHUB_ACCESS_TOKEN = read_access_token()
662 
663        # Clean all generated .bazel files
664        for f in GEN_DIR.glob("*.bazel"):
665            f.unlink()
666 
667    for deps in ALL_DEPS:
668        process_config(deps)
669 
670 
671run()