1
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
2
load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
3
load("//:build/python_metadata.bzl", "BUNDLE_VERSION_INFO")
4
load("//:build/wd_test.bzl", "wd_test")
5
 
6
def _get_enable_flags(python_flag):
7
    flags = [BUNDLE_VERSION_INFO[python_flag]["enable_flag_name"]]
8
    if "python_workers" not in flags:
9
        flags.append("python_workers")
10
    for key, value in BUNDLE_VERSION_INFO.items():
11
        # With all-compat-flags variant we might end up accidently using a newer python bundle than
12
        # intended. To make sure we get the speicific intended version we also need to disable newer
13
        # python versions.
14
        if python_flag != key and value["enable_flag_name"] not in ["python_workers", "python_workers_development"]:
15
            flags.append("no_" + value["enable_flag_name"])
16
    return flags
17
 
18
def _py_wd_test_helper(
19
        name,
20
        src,
21
        python_flag,
22
        *,
23
        make_snapshot,
24
        use_snapshot,
25
        args,
26
        feature_flags,
27
        data = [],
28
        **kwargs):
29
    name_flag = name + "_" + python_flag
30
    templated_src = name_flag.replace("/", "-") + "@template"
31
    templated_src = "/".join(src.split("/")[:-1] + [templated_src])
32
 
33
    pkg_tag = BUNDLE_VERSION_INFO[python_flag]["packages"]
34
    data = data + ["@all_pyodide_wheels_%s//:whls" % pkg_tag]
35
    args = args + ["--pyodide-package-disk-cache-dir"]
36
 
37
    # +pyodide+ is a bzlmod canonical repository name
38
    args.append("../+pyodide+all_pyodide_wheels_%s" % pkg_tag)
39
 
40
    load_snapshot = None
41
    pyodide_version = BUNDLE_VERSION_INFO[python_flag]["real_pyodide_version"]
42
    if use_snapshot == "stacked":
43
        if pyodide_version == "0.26.0a2":
44
            use_snapshot = None
45
        else:
46
            use_snapshot = "baseline"
47
            if make_snapshot:
48
                feature_flags = feature_flags + ["python_dedicated_snapshot"]
49
    if use_snapshot:
50
        version_info = BUNDLE_VERSION_INFO[python_flag]
51
 
52
        snapshot = version_info[use_snapshot + "_snapshot"]
53
        data = data + [":python_snapshots"]
54
        load_snapshot = snapshot
55
 
56
    if load_snapshot and not make_snapshot:
57
        args += ["--python-load-snapshot", "load_snapshot.bin"]
58
 
59
    flags = _get_enable_flags(python_flag) + feature_flags
60
    feature_flags_txt = ",".join(['"{}"'.format(flag) for flag in flags])
61
 
62
    expand_template(
63
        name = name_flag + "@rule",
64
        out = templated_src,
65
        template = src,
66
        substitutions = {"%PYTHON_FEATURE_FLAGS": feature_flags_txt},
67
    )
68
 
69
    # Since we bumped the development flag to point to 0.28.2, it doesn't work on windows CI.
70
    # TODO: Fix this.
71
    if python_flag == "development":
72
        kwargs["target_compatible_with"] = select({
73
            "@platforms//os:windows": ["@platforms//:incompatible"],
74
            "//conditions:default": [],
75
        })
76
 
77
    wd_test(
78
        src = templated_src,
79
        name = name_flag,
80
        args = args,
81
        python_snapshot_test = make_snapshot,
82
        data = data,
83
        load_snapshot = load_snapshot,
84
        # TODO(soon): at the time of disabling these they all passed but because of how slow python
85
        #             tests are we disabled them for now. We should re-enable them when we have
86
        #             a better way to run them.
87
        generate_all_autogates_variant = False,
88
        generate_all_compat_flags_variant = False,
89
        **kwargs
90
    )
91
 
92
def _snapshot_file(snapshot):
93
    if not snapshot:
94
        return []
95
    copy_file(
96
        name = "pyodide-snapshot-%s@copy" % snapshot,
97
        src = "@pyodide-snapshot-%s//file" % snapshot,
98
        out = snapshot,
99
        visibility = ["//visibility:public"],
100
    )
101
    return [":" + snapshot]
102
 
103
def _snapshot_files(
104
        name,
105
        baseline_snapshot = None,
106
        numpy_snapshot = None,
107
        fastapi_snapshot = None,
108
        dedicated_fastapi_snapshot = None,
109
        **_kwds):
110
    if name == "development":
111
        return []
112
    result = []
113
    result += _snapshot_file(baseline_snapshot)
114
    result += _snapshot_file(numpy_snapshot)
115
    result += _snapshot_file(fastapi_snapshot)
116
    result += _snapshot_file(dedicated_fastapi_snapshot)
117
    return result
118
 
119
def _snapshot_file_group():
120
    snapshots = []
121
    for x in BUNDLE_VERSION_INFO.values():
122
        snapshots += _snapshot_files(**x)
123
 
124
    native.filegroup(
125
        name = "python_snapshots",
126
        data = snapshots,
127
        visibility = ["//visibility:public"],
128
    )
129
 
130
def _capnp_bundle(id, **_kwds):
131
    if id == "dev":
132
        return
133
    name = "pyodide_%s.capnp.bin" % id
134
    copy_file(
135
        name = name + "@rule",
136
        src = "@%s//file" % name,
137
        out = "pyodide-bundle-cache/" + name,
138
        visibility = ["//visibility:public"],
139
    )
140
 
141
def _capnp_bundles_file_group():
142
    # pyodide_dev.capnp.bin represents a custom pyodide version "dev" that is generated
143
    # at build time using the latest contents of the src/pyodide directory.
144
    # This is used to run tests to ensure that they are always run against the latest build of
145
    # the Pyodide bundle.
146
    copy_file(
147
        name = "pyodide_dev.capnp.bin@rule",
148
        src = "//src/pyodide:pyodide.capnp.bin_cross",
149
        out = "pyodide-bundle-cache/pyodide_dev.capnp.bin",
150
        visibility = ["//visibility:public"],
151
    )
152
    for info in BUNDLE_VERSION_INFO.values():
153
        _capnp_bundle(**info)
154
 
155
def _capnp_rules():
156
    return ["//src/workerd/server/tests/python:pyodide_%s.capnp.bin@rule" % info["id"] for info in BUNDLE_VERSION_INFO.values()]
157
 
158
def python_test_setup():
159
    _capnp_bundles_file_group()
160
    _snapshot_file_group()
161
 
162
def compute_python_flags(python_flags, skip_python_flags):
163
    if python_flags == "all":
164
        python_flags = BUNDLE_VERSION_INFO.keys()
165
    python_flags = [flag for flag in python_flags if flag not in skip_python_flags and flag in BUNDLE_VERSION_INFO]
166
    return python_flags
167
 
168
def py_wd_test(
169
        directory = None,
170
        *,
171
        src = None,
172
        data = None,
173
        name = None,
174
        python_flags = "all",
175
        skip_python_flags = [],
176
        feature_flags = [],
177
        args = [],
178
        size = "enormous",
179
        tags = [],
180
        make_snapshot = True,
181
        use_snapshot = "stacked",
182
        skip_default_data = False,
183
        **kwargs):
184
    python_flags = compute_python_flags(python_flags, skip_python_flags)
185
    if data == None:
186
        data = []
187
    if directory and not skip_default_data:
188
        data += native.glob(
189
            [
190
                directory + "/**",
191
            ],
192
            exclude = ["**/*.wd-test"],
193
        )
194
    if src == None:
195
        src = native.glob([directory + "/*.wd-test"])[0]
196
    if name == None and directory != None:
197
        name = directory
198
    elif name == None:
199
        name = src.removesuffix(".wd-test")
200
    data += _capnp_rules()
201
    args = args + [
202
        "--pyodide-bundle-disk-cache-dir",
203
        "$(location //src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule)/..",
204
        "--experimental",
205
        "--python-snapshot-dir",
206
        ".",
207
    ]
208
 
209
    # Python tests are extremely slow with coverage instrumentation, skip them
210
    tags = tags + ["py_wd_test", "python", "no-coverage"]
211
 
212
    for python_flag in python_flags:
213
        _py_wd_test_helper(
214
            name,
215
            src,
216
            python_flag,
217
            make_snapshot = make_snapshot,
218
            use_snapshot = use_snapshot,
219
            feature_flags = feature_flags,
220
            data = data,
221
            args = args,
222
            size = size,
223
            tags = tags,
224
            **kwargs
225
        )