1
load("@aspect_rules_esbuild//esbuild:defs.bzl", "esbuild")
2
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
3
load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
4
load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
5
load("//:build/capnp_embed.bzl", "capnp_embed")
6
load("//:build/js_file.bzl", "js_file")
7
load("//:build/python_metadata.bzl", "BUNDLE_VERSION_INFO", "PYODIDE_VERSIONS", "PYTHON_LOCKFILES")
8
load("//:build/wd_js_bundle.bzl", "wd_js_bundle")
9
load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle")
10
 
11
def _out_name(src):
12
    src = src.removesuffix("//file")
13
    src = src.removeprefix("@")
14
    return src.rsplit(":", 2)[-1].rsplit("/", 2)[-1]
15
 
16
def _out_path(name, version):
17
    res = "generated/" + name
18
    if version:
19
        res = version + "/" + res
20
    return res
21
 
22
def _ts_bundle_out(prefix, name, version):
23
    return ":" + _out_path(prefix + name.removeprefix("internal/").replace("/", "_"), version)
24
 
25
def _copy_to_generated(src, version = None, out_name = None, name = None):
26
    out_name = out_name or _out_name(src)
27
    if name == None:
28
        name = out_name + "@copy"
29
    if version:
30
        name += "@" + version
31
    copy_file(name = name, src = src, out = _out_path(out_name, version))
32
 
33
def _copy_and_capnp_embed(src):
34
    out_name = _out_name(src)
35
    _copy_to_generated(src)
36
    capnp_embed(
37
        name = out_name + "@capnp",
38
        src = _out_path(out_name, None),
39
        deps = [out_name + "@copy"],
40
    )
41
 
42
def _fmt_python_snapshot_release(
43
        pyodide_version,
44
        pyodide_date,
45
        packages,
46
        backport,
47
        baseline_snapshot_hash,
48
        flag,
49
        real_pyodide_version,
50
        **_kwds):
51
    content = ", ".join(
52
        [
53
            'pyodide = "%s"' % pyodide_version,
54
            'realPyodideVersion = "%s"' % real_pyodide_version,
55
            'pyodideRevision = "%s"' % pyodide_date,
56
            'packages = "%s"' % packages,
57
            "backport = %s" % backport,
58
            'baselineSnapshotHash = "%s"' % baseline_snapshot_hash,
59
            'flagName = "%s"' % flag,
60
        ],
61
    )
62
    return "(%s)" % content
63
 
64
def pyodide_extra():
65
    _copy_to_generated(
66
        "pyodide_extra.capnp",
67
        out_name = "pyodide_extra_tmpl.capnp",
68
    )
69
 
70
    package_tags = [info["tag"] for info in PYTHON_LOCKFILES]
71
 
72
    expand_template(
73
        name = "pyodide_extra_expand_template@rule",
74
        out = "generated/pyodide_extra.capnp",
75
        substitutions = {
76
            "%PACKAGE_LOCKS": ",".join(
77
                [
78
                    '(packageDate = "%s", lock = embed "pyodide-lock_%s.json")' %
79
                    (tag, tag)
80
                    for tag in package_tags
81
                ],
82
            ),
83
            "%PYTHON_RELEASES": ", ".join(
84
                [_fmt_python_snapshot_release(**info) for info in BUNDLE_VERSION_INFO.values()],
85
            ),
86
        },
87
        template = "generated/pyodide_extra_tmpl.capnp",
88
    )
89
 
90
    for tag in package_tags:
91
        _copy_and_capnp_embed("@pyodide-lock_" + tag + ".json//file")
92
 
93
    cc_capnp_library(
94
        name = "pyodide_extra_capnp",
95
        srcs = ["generated/pyodide_extra.capnp"],
96
        visibility = ["//visibility:public"],
97
        deps = [
98
        ] + [
99
            ":pyodide-lock_%s.json@capnp" % tag
100
            for tag in package_tags
101
        ],
102
        target_compatible_with = select({
103
            "@//build/config:no_build": ["@platforms//:incompatible"],
104
            "//conditions:default": [],
105
        }),
106
    )
107
 
108
def python_bundles(overrides = {}):
109
    srcs = [_python_bundle_helper(info, overrides) for info in PYODIDE_VERSIONS]
110
    native.filegroup(
111
        name = "python_bundles",
112
        srcs = srcs + [":bundle_version_info"],
113
    )
114
 
115
def _python_bundle_helper(info, overrides):
116
    version = info["version"]
117
    override = overrides.get(version, {})
118
    return _python_bundle(version, **override)
119
 
120
def pyodide_static():
121
    internal_data_modules = native.glob([
122
        "internal/*.py",
123
        "internal/workers-api/src/*.py",
124
        "internal/workers-api/src/workers/*.py",
125
        "internal/patches/*.py",
126
        "internal/topLevelEntropy/*.py",
127
    ])
128
    internal_modules = native.glob(
129
        [
130
            "internal/*.ts",
131
            "internal/topLevelEntropy/*.ts",
132
            "types/*.ts",
133
            "types/*/*.ts",
134
        ],
135
    )
136
    modules = ["python-entrypoint-helper.ts"]
137
 
138
    wd_ts_bundle(
139
        name = "pyodide_static",
140
        eslintrc_json = "eslint.config.mjs",
141
        import_name = "pyodide",
142
        internal_data_modules = internal_data_modules,
143
        internal_modules = internal_modules,
144
        lint = True,
145
        modules = modules,
146
        schema_id = "0xdc8d02dfbdf14025",
147
        tsconfig_json = "tsconfig.json",
148
    )
149
 
150
_PRELUDE = """
151
import {
152
    addEventListener,
153
    getRandomValues,
154
    location,
155
    monotonicDateNow,
156
    newWasmModule,
157
    patchedApplyFunc,
158
    patchedLoadLibData,
159
    reportUndefinedSymbolsPatched,
160
    wasmInstantiate,
161
    patched_PyEM_CountFuncParams,
162
} from "pyodide-internal:pool/builtin_wrappers";
163
"""
164
 
165
# pyodide.asm.js patches
166
# TODO: all of these should be fixed by linking our own Pyodide or by upstreaming.
167
_REPLACEMENTS = [
168
    [
169
        # Convert pyodide.asm.js into an es6 module.
170
        # When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us
171
        # automatically.
172
        "var _createPyodideModule",
173
        _PRELUDE + "export const _createPyodideModule",
174
    ],
175
    [
176
        "globalThis._createPyodideModule = _createPyodideModule;",
177
        "",
178
    ],
179
    [
180
        "new WebAssembly.Module",
181
        "newWasmModule",
182
    ],
183
    [
184
        "WebAssembly.instantiate",
185
        "wasmInstantiate",
186
    ],
187
    [
188
        "Date.now",
189
        "monotonicDateNow",
190
    ],
191
    [
192
        "reportUndefinedSymbols()",
193
        "reportUndefinedSymbolsPatched(Module)",
194
    ],
195
    [
196
        "crypto.getRandomValues(",
197
        "getRandomValues(Module, ",
198
    ],
199
    [
200
        # Direct eval disallowed in esbuild, see:
201
        # https://esbuild.github.io/content-types/#direct-eval
202
        "eval(func)",
203
        "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()",
204
    ],
205
    [
206
        "eval(data)",
207
        "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()",
208
    ],
209
    [
210
        "eval(UTF8ToString(ptr))",
211
        "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()",
212
    ],
213
    # Dynamic linking patches:
214
    # library lookup
215
    [
216
        "function loadLibData(){",
217
        """
218
        function loadLibData(){
219
            var libData = patchedLoadLibData(Module, libName, flags.rpath);
220
            return flags.loadAsync ? Promise.resolve(libData) : libData;
221
        }
222
        function dummiedOutOrigLoadLibData(){
223
        """,
224
    ],
225
    # for ensuring memory base of dynlib is stable when restoring snapshots
226
    [
227
        "getMemory(",
228
        "Module.getMemoryPatched(Module, libName, ",
229
    ],
230
    # to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7
231
    [
232
        "nullToUndefined(func.apply(",
233
        "nullToUndefined(patchedApplyFunc(API, func, ",
234
    ],
235
    [
236
        "nullToUndefined(Function.prototype.apply.apply",
237
        "nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply",
238
    ],
239
    [
240
        "function _PyEM_CountFuncParams(func){",
241
        "function _PyEM_CountFuncParams(func){ return patched_PyEM_CountFuncParams(Module, func);",
242
    ],
243
    [
244
        "var tableBase=metadata.tableSize?wasmTable.length:0;",
245
        "var tableBase=metadata.tableSize?wasmTable.length:0;" +
246
        "Module.snapshotDebug && console.log('loadWebAssemblyModule', libName, memoryBase, tableBase);",
247
    ],
248
    # to ensure we report every fatal error, not just the first one
249
    [
250
        'console.error("Recursive call to fatal_error. Inner error was:");',
251
        'console.error("Recursive call to fatal_error. Inner error was:");\n' +
252
        "try { API.on_fatal?.(e); } catch(e2) { console.error(e2); }\n",
253
    ],
254
]
255
 
256
def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, python_stdlib_zip = None, emscripten_setup_override = None):
257
    pyodide_package = "@pyodide-%s//" % version
258
    if not pyodide_asm_wasm:
259
        pyodide_asm_wasm = pyodide_package + ":pyodide/pyodide.asm.wasm"
260
 
261
    if not pyodide_asm_js:
262
        pyodide_asm_js = pyodide_package + ":pyodide/pyodide.asm.js"
263
 
264
    if not python_stdlib_zip:
265
        python_stdlib_zip = pyodide_package + ":pyodide/python_stdlib.zip"
266
 
267
    _copy_to_generated(pyodide_asm_wasm, version, out_name = "pyodide.asm.wasm")
268
 
269
    _copy_to_generated(python_stdlib_zip, version, out_name = "python_stdlib.zip")
270
 
271
    expand_template(
272
        name = "pyodide.asm.js@rule@" + version,
273
        out = _out_path("pyodide.asm.js", version),
274
        substitutions = dict(_REPLACEMENTS),
275
        template = pyodide_asm_js,
276
    )
277
 
278
    js_file(
279
        name = "pyodide.asm.js@rule_js@" + version,
280
        srcs = [_out_path("pyodide.asm.js", version)],
281
        deps = ["pyodide.asm.js@rule@" + version],
282
    )
283
 
284
    if emscripten_setup_override:
285
        _copy_to_generated(out_name = "emscriptenSetup.js", name = "emscriptenSetup", src = emscripten_setup_override, version = version)
286
    else:
287
        expand_template(
288
            name = "esbuild.config.mjs@" + version,
289
            out = _out_path("esbuild.config.mjs", version),
290
            substitutions = {
291
                "%PYODIDE_VERSION%": version,
292
            },
293
            template = "internal/pool/esbuild.config.mjs",
294
        )
295
 
296
        esbuild(
297
            name = "emscriptenSetup@" + version,
298
            # exclude emscriptenSetup from source set so that rules_ts won't also try to create a JS output
299
            # for it. The file is provided in entry_point instead.
300
            srcs = native.glob([
301
                "internal/pool/*.ts",
302
            ], exclude = ["internal/pool/emscriptenSetup.ts"]) + [
303
                _out_path("pyodide.asm.js", version),
304
                "internal/util.ts",
305
                "internal/const.ts",
306
            ],
307
            config = _out_path("esbuild.config.mjs", version),
308
            entry_point = "internal/pool/emscriptenSetup.ts",
309
            external = [
310
                "child_process",
311
                "crypto",
312
                "fs",
313
                "path",
314
                "url",
315
                "vm",
316
                "ws",
317
                "node:child_process",
318
                "node:crypto",
319
                "node:fs",
320
                "node:path",
321
                "node:url",
322
                "node:vm",
323
                "internal:unsafe-eval",
324
            ],
325
            format = "esm",
326
            output = _out_path("emscriptenSetup.js", version),
327
            target = "esnext",
328
            deps = ["pyodide.asm.js@rule_js@" + version],
329
        )
330
 
331
    import_name = "pyodideRuntime"
332
    wd_js_bundle(
333
        name = "pyodide@" + version,
334
        import_name = import_name,
335
        builtin_modules = [],
336
        schema_id = "0xbcc8f57c63814005",
337
        internal_modules = [
338
            _out_path("emscriptenSetup.js", version),
339
        ],
340
        internal_wasm_modules = [
341
            _out_path("pyodide.asm.wasm", version),
342
        ],
343
        internal_data_modules = [
344
            _out_path("python_stdlib.zip", version),
345
        ],
346
        deps = [
347
            "emscriptenSetup@" + version,
348
            "pyodide.asm.wasm@copy@" + version,
349
            "python_stdlib.zip@copy@" + version,
350
        ],
351
        out_dir = _out_path("", version),
352
    )
353
 
354
    pyodide_cappn_bin_rule = "pyodide.capnp.bin@rule@" + version
355
    native.genrule(
356
        name = pyodide_cappn_bin_rule,
357
        srcs = [
358
            ":pyodide@%s.capnp" % version,
359
            "//src/workerd/jsg:modules.capnp",
360
            _ts_bundle_out(import_name + "-internal_", "emscriptenSetup", version),
361
            _ts_bundle_out(import_name + "-internal_", "pyodide.asm.wasm", version),
362
            _ts_bundle_out(import_name + "-internal_", "python_stdlib.zip", version),
363
        ],
364
        outs = [_out_path("pyodide.capnp.bin", version)],
365
        cmd = " ".join([
366
            # Annoying logic to deal with different paths in workerd vs downstream.
367
            # Either need "-I src" in workerd or -I external/+local_repository+workerd/src downstream
368
            "INCLUDE=$$(stat src > /dev/null 2>&1 && echo src || echo external/+local_repository+workerd/src);",
369
            "$(execpath @capnp-cpp//src/capnp:capnp_tool)",
370
            "eval",
371
            "$(location :pyodide@%s.capnp)" % version,
372
            import_name + "Bundle",
373
            "-I $$INCLUDE",
374
            "-o binary",
375
            "> $@",
376
        ]),
377
        tools = ["@capnp-cpp//src/capnp:capnp_tool"],
378
        visibility = ["//visibility:public"],
379
        target_compatible_with = select({
380
            "@//build/config:no_build": ["@platforms//:incompatible"],
381
            "//conditions:default": [],
382
        }),
383
    )
384
    return pyodide_cappn_bin_rule