Skip to content
rachel/workerd
File

Blob: build/wpt_test.bzl

22.2 KB
1# Copyright (c) 2017-2022 Cloudflare, Inc.
2# Licensed under the Apache 2.0 license found in the LICENSE file or at:
3#     https://opensource.org/licenses/Apache-2.0
4 
5load("@bazel_skylib//lib:paths.bzl", "paths")
6load("//:build/wd_test.bzl", "wd_test")
7 
8PORT_BINDINGS = [
9    "HTTP_PORT",
10    "HTTPS_PORT",
11    # The remaining ports are not currently used by tests but need to be assigned to wptserve anyway
12    "HTTP_PORT_2",
13    "HTTPS_PORT_2",
14    "HTTP_PUBLIC_PORT",
15    "HTTPS_PUBLIC_PORT",
16    "HTTP_LOCAL_PORT",
17    "WSS_PORT",
18    "WS_PORT",
19]
20 
21### wpt_test macro
22### (Invokes wpt_js_test_gen, wpt_wd_test_gen and wd_test to assemble a complete test suite.)
23### -----------------------------------------------------------------------------------------
24 
25def wpt_test(name, wpt_directory, config, compat_date = "", compat_flags = [], autogates = [], start_server = False, **kwargs):
26    """
27    Main entry point.
28 
29    1. Generates a workerd test suite in JS. This contains the logic to run
30       each WPT test file, applying the relevant test config.
31    2. Generates a wd-test file for this test suite. This contains all of the
32       paths to modules needed to run the test: generated test suite, test config
33       file, WPT test scripts, associated JSON resources.
34    """
35 
36    js_test_gen_rule = "{}@_wpt_js_test_gen".format(name)
37    test_config_as_js = config.removesuffix(".ts") + ".js"
38    wpt_tsproject = "//src/wpt:wpt-all@tsproject"
39    harness = "//src/wpt:harness@js"
40    wpt_cacert = "@wpt//:tools/certs/cacert.pem"
41    wpt_common = "@wpt//:common@module"
42    wpt_resources = "@wpt//:resources@module"
43    wpt_interfaces = "@wpt//:interfaces@module"
44 
45    if compat_date == "":
46        compat_date = "2999-12-31"
47 
48    _wpt_js_test_gen(
49        name = js_test_gen_rule,
50        test_name = name,
51        wpt_directory = wpt_directory,
52        test_config = test_config_as_js,
53        wpt_tsproject = wpt_tsproject,
54    )
55 
56    wd_test_gen_rule = "{}@_wpt_wd_test_gen".format(name)
57    _wpt_wd_test_gen(
58        name = wd_test_gen_rule,
59        test_name = name,
60        wpt_directory = wpt_directory,
61        test_config = test_config_as_js,
62        test_js_generated = js_test_gen_rule,
63        harness = harness,
64        autogates = autogates,
65        wpt_cacert = wpt_cacert,
66        wpt_common = wpt_common,
67        wpt_resources = wpt_resources,
68        wpt_interfaces = wpt_interfaces,
69        compat_flags = compat_flags + ["immutable_api_prototypes", "spec_compliant_property_attributes", "pedantic_wpt", "experimental", "nodejs_compat", "unsupported_process_actual_platform"],
70    )
71 
72    data = [
73        test_config_as_js,  # e.g. "url-test.js"
74        js_test_gen_rule,  # e.g. "url-test.generated.js",
75        wpt_directory,  # e.g. wpt/url/**",
76        harness,  # e.g. wpt/harness/*.ts
77        wpt_cacert,  # i.e. "wpt/tools/certs/cacert.pem",
78        wpt_common,  # i.e. "wpt/common/**"
79        wpt_resources,  # i.e. "wpt/resources/**"
80        wpt_interfaces,  # i.e. "wpt/interfaces/**"
81    ]
82 
83    wd_test(
84        name = "{}".format(name),
85        src = wd_test_gen_rule,
86        args = ["--experimental"],
87        sidecar_port_bindings = PORT_BINDINGS if start_server else [],
88        sidecar = "@wpt//:entrypoint" if start_server else None,
89        compat_date = compat_date,
90        generate_all_compat_flags_variant = False,  # Already using future date where possible.
91        # Predictable mode enables V8's --expose-gc, which can trigger additional JIT events
92        # that produce INFO log lines on stdout. Those lines land after the JSON report/stats
93        # blobs emitted by the WPT harness and break tools/cross/wpt_logs.py parsing.
94        predictable = False,
95        data = data,
96        **kwargs
97    )
98 
99### wpt_module macro and rule
100### (Discovers test files within the WPT directory tree)
101### ----------------------------------------------------
102 
103def wpt_module(name):
104    """
105    Given the name of a directory within the WPT tree, creates a rule providing:
106 
107    srcs: All files within the directory
108    dir: A reference to the directory itself.
109 
110    This info is later used by wpt_test rules to generate JS code and wd-test config
111    for each test.
112    """
113    return _wpt_module(
114        name = "{}@module".format(name),
115        dir = name,
116        srcs = native.glob(["{}/**/*".format(name)]),
117        visibility = ["//visibility:public"],
118    )
119 
120WPTModuleInfo = provider(fields = ["base"])
121 
122def _wpt_module_impl(ctx):
123    return [
124        DefaultInfo(files = depset(ctx.files.srcs)),
125        WPTModuleInfo(base = ctx.attr.dir.files.to_list()[0]),
126    ]
127 
128_wpt_module = rule(
129    implementation = _wpt_module_impl,
130    attrs = {
131        "dir": attr.label(allow_single_file = True),
132        "srcs": attr.label_list(allow_files = True),
133    },
134)
135 
136### wpt_js_test_gen rule
137### (Generates a .js file for each test)
138### ------------------------------------
139 
140def _wpt_js_test_gen_impl(ctx):
141    """
142    Generates a workerd test suite in JS.
143 
144    This contains the logic to run each WPT test file, applying the relevant test config.
145    """
146 
147    src = ctx.actions.declare_file("{}-test.generated.js".format(ctx.attr.test_name))
148    base = ctx.attr.wpt_directory[WPTModuleInfo].base
149 
150    files = ctx.attr.wpt_directory.files.to_list()
151    test_files = [file for file in files if is_test_file(file)]
152 
153    ctx.actions.write(
154        output = src,
155        content = WPT_JS_TEST_TEMPLATE.format(
156            test_config = ctx.file.test_config.basename,
157            cases = generate_external_cases(base, test_files),
158            all_test_files = generate_external_file_list(base, test_files),
159            test_name = ctx.attr.test_name,
160        ),
161    )
162 
163    return DefaultInfo(
164        files = depset([src]),
165    )
166 
167_wpt_js_test_gen = rule(
168    implementation = _wpt_js_test_gen_impl,
169    attrs = {
170        # A string to use as the test name. Used in the wd-test filename and the worker's name
171        "test_name": attr.string(),
172        # A file group representing a directory of wpt tests. All files in the group will be embedded.
173        "wpt_directory": attr.label(),
174        # A JS file containing the test configuration.
175        "test_config": attr.label(allow_single_file = True),
176        # Dependency: The ts_project rule that compiles the tests to JS
177        "wpt_tsproject": attr.label(),
178    },
179)
180 
181WPT_JS_TEST_TEMPLATE = """// This file is autogenerated by wpt_test.bzl
182// DO NOT EDIT.
183import {{ createRunner }} from 'harness/harness';
184import config from '{test_config}';
185 
186const allTestFiles = {all_test_files};
187const {{ run, printResults }} = createRunner(config, '{test_name}', allTestFiles);
188 
189{cases}
190 
191export const zzz_results = printResults();
192"""
193 
194def is_test_file(file):
195    if not file.path.endswith(".js"):
196        # Not JS code, not a test
197        return False
198 
199    if is_in_resources_directory(file):
200        # If it's in a subdirectory named resources/, then it's meant to be included by tests,
201        # not to run on its own. This logic still isn't perfect; sometimes resources are dropped
202        # into the main directory, and would need to manually be marked as skipAllTests
203        return False
204 
205    if ".tentative." in file.path or "/tentative/" in file.path:
206        # Tentative tests are for proposed features that are not yet standardized.
207        # We skip these to avoid noise from unstable specifications.
208        return False
209 
210    # Probably an actual test
211    return True
212 
213def generate_external_cases(base, files):
214    """
215    Generate a workerd test case that runs each test file in the WPT module.
216    """
217 
218    result = []
219    for file in files:
220        relative_path = module_relative_path(base, file)
221        result.append("export const {} = run('{}');".format(test_case_name(relative_path), relative_path))
222 
223    return "\n".join(result)
224 
225def generate_external_file_list(base, files):
226    """
227    Generate a JS list containing the name of every test file in the WPT module
228    """
229 
230    return "[{}];".format(", ".join([
231        "'{}'".format(module_relative_path(base, file))
232        for file in files
233    ]))
234 
235### wpt_wd_test_gen rule
236### (Generates a .wd-test file for each test)
237### -----------------------------------------
238 
239def _wpt_wd_test_gen_impl(ctx):
240    """
241    Generates a wd-test file for this test suite.
242 
243    This contains all of the paths to modules needed to run the test: generated test suite, test
244    config file, WPT test scripts, associated JSON resources.
245    """
246    src = ctx.actions.declare_file("{}.wd-test".format(ctx.attr.test_name))
247    base = ctx.attr.wpt_directory[WPTModuleInfo].base
248 
249    # Generate bindings for test directory files plus common/**/*.js, resources/**/*.js, and interfaces/**/*.idl
250    bindings = generate_external_bindings(src, base, ctx.attr.wpt_directory.files)
251    common_base = ctx.attr.wpt_common[WPTModuleInfo].base
252    common_bindings = generate_common_bindings(src, common_base, ctx.attr.wpt_common.files)
253    resources_base = ctx.attr.wpt_resources[WPTModuleInfo].base
254    resources_bindings = generate_resources_bindings(src, resources_base, ctx.attr.wpt_resources.files)
255    interfaces_base = ctx.attr.wpt_interfaces[WPTModuleInfo].base
256    interfaces_bindings = generate_interfaces_bindings(src, interfaces_base, ctx.attr.wpt_interfaces.files)
257    all_bindings = ",\n".join([b for b in [bindings, common_bindings, resources_bindings, interfaces_bindings] if b])
258    ctx.actions.write(
259        output = src,
260        content = WPT_WD_TEST_TEMPLATE.format(
261            test_name = ctx.attr.test_name,
262            test_config = ctx.file.test_config.basename,
263            test_js_generated = ctx.file.test_js_generated.basename,
264            bindings = all_bindings,
265            harness_modules = generate_harness_modules(src, ctx.attr.harness.files),
266            wpt_cacert = wd_test_relative_path(src, ctx.file.wpt_cacert),
267            autogates = generate_autogates_field(ctx.attr.autogates),
268            compat_flags = generate_compat_flags_field(ctx.attr.compat_flags),
269        ),
270    )
271 
272    return DefaultInfo(
273        files = depset([src]),
274    )
275 
276_wpt_wd_test_gen = rule(
277    implementation = _wpt_wd_test_gen_impl,
278    attrs = {
279        # A string to use as the test name. Used in the wd-test filename and the worker's name
280        "test_name": attr.string(),
281        # A file group representing a directory of wpt tests. All files in the group will be embedded.
282        "wpt_directory": attr.label(),
283        # A JS file containing the test configuration.
284        "test_config": attr.label(allow_single_file = True),
285        # An auto-generated JS file containing the test logic.
286        "test_js_generated": attr.label(allow_single_file = True),
287        # Target specifying the files in the WPT test harness
288        "harness": attr.label(),
289        # Target specifying the location of the WPT CA certificate
290        "wpt_cacert": attr.label(allow_single_file = True),
291        # Target specifying the WPT common directory module
292        "wpt_common": attr.label(),
293        # Target specifying the WPT resources directory module
294        "wpt_resources": attr.label(),
295        # Target specifying the WPT interfaces directory module
296        "wpt_interfaces": attr.label(),
297        # A list of autogates to specify in the generated wd-test file
298        "autogates": attr.string_list(),
299        # A list of compatibility flags to specify in the generated wd-test file
300        "compat_flags": attr.string_list(),
301    },
302)
303 
304WPT_WD_TEST_TEMPLATE = """
305using Workerd = import "/workerd/workerd.capnp";
306const unitTests :Workerd.Config = (
307  services = [
308    ( name = "{test_name}",
309      worker = (
310        modules = [
311          (name = "worker", esModule = embed "{test_js_generated}"),
312          (name = "{test_config}", esModule = embed "{test_config}"),
313          {harness_modules}
314        ],
315        bindings = [
316          (name = "wpt", service = "wpt"),
317          (name = "unsafe", unsafeEval = void),
318          (name = "SIDECAR_HOSTNAME", fromEnvironment = "SIDECAR_HOSTNAME"),
319          (name = "HTTP_PORT", fromEnvironment = "HTTP_PORT"),
320          (name = "HTTPS_PORT", fromEnvironment = "HTTPS_PORT"),
321          (name = "WS_PORT", fromEnvironment = "WS_PORT"),
322          (name = "WSS_PORT", fromEnvironment = "WSS_PORT"),
323          (name = "GEN_TEST_CONFIG", fromEnvironment = "GEN_TEST_CONFIG"),
324          (name = "GEN_TEST_REPORT", fromEnvironment = "GEN_TEST_REPORT"),
325          (name = "GEN_TEST_STATS", fromEnvironment = "GEN_TEST_STATS"),
326          {bindings}
327        ],
328        {compat_flags}
329      )
330    ),
331    ( name = "internet",
332      network = (
333        allow = ["private"],
334        tlsOptions = (
335          trustedCertificates = [
336            embed "{wpt_cacert}"
337          ]
338        )
339      )
340    ),
341    (
342      name = "wpt",
343      disk = ".",
344    )
345  ],
346  v8Flags = ["--expose-gc"],
347  {autogates}
348);"""
349 
350def generate_autogates_field(autogates):
351    """
352    Generates a capnproto fragment listing the specified autogates.
353    """
354 
355    if not autogates:
356        return ""
357 
358    autogate_list = ", ".join(['"{}"'.format(autogate) for autogate in autogates])
359    return "autogates = [{}],".format(autogate_list)
360 
361def generate_compat_flags_field(flags):
362    """
363    Generates a capnproto fragment listing the specified compatibility flags.
364    """
365 
366    if not flags:
367        return ""
368 
369    flag_list = ", ".join(['"{}"'.format(flag) for flag in flags])
370    return "compatibilityFlags = [{}],".format(flag_list)
371 
372def generate_external_bindings(wd_test_file, base, files):
373    """
374    Generates appropriate bindings for each file in the WPT module:
375 
376    - JS files: text binding to allow code to be evaluated
377    - JSON files: JSON binding to allow test code to fetch resources
378    """
379 
380    result = []
381 
382    for file in files.to_list():
383        if file.extension == "js":
384            binding_type = "text"
385        elif file.extension == "json":
386            binding_type = "json"
387        else:
388            # Unknown binding type, skip for now
389            continue
390 
391        result.append('(name = "{}", {} = embed "{}")'.format(module_relative_path(base, file), binding_type, wd_test_relative_path(wd_test_file, file)))
392 
393    return ",\n".join(result)
394 
395def generate_common_bindings(wd_test_file, base, files):
396    """
397    Generates appropriate bindings for each file in the WPT common module.
398    Files are prefixed with /common/ path.
399 
400    Only JS files are included as some JSON files in common/ contain
401    non-standard JSON (e.g., with comments) that would cause parsing errors.
402    """
403 
404    result = []
405 
406    for file in files.to_list():
407        if file.extension == "js":
408            binding_type = "text"
409        else:
410            # Skip non-JS files to avoid issues with non-standard JSON
411            continue
412 
413        relative_path = module_relative_path(base, file)
414        result.append('(name = "/common/{}", {} = embed "{}")'.format(relative_path, binding_type, wd_test_relative_path(wd_test_file, file)))
415 
416    return ",\n".join(result)
417 
418def generate_resources_bindings(wd_test_file, base, files):
419    """
420    Generates appropriate bindings for each file in the WPT resources module.
421    Files are prefixed with /resources/ path.
422 
423    Only JS files are included. Also adds an alias for /resources/WebIDLParser.js
424    which points to /resources/webidl2/lib/webidl2.js (matching the WPT server
425    rewrite rule in tools/serve/serve.py).
426    """
427 
428    result = []
429    webidl2_file = None
430 
431    for file in files.to_list():
432        if file.extension != "js":
433            # Skip non-JS files
434            continue
435 
436        relative_path = module_relative_path(base, file)
437        file_path = wd_test_relative_path(wd_test_file, file)
438        result.append('(name = "/resources/{}", text = embed "{}")'.format(relative_path, file_path))
439 
440        # Track the webidl2.js file for the alias
441        if relative_path == "webidl2/lib/webidl2.js":
442            webidl2_file = file_path
443 
444    # Add the WebIDLParser.js alias (WPT server rewrites this to webidl2/lib/webidl2.js)
445    if webidl2_file:
446        result.append('(name = "/resources/WebIDLParser.js", text = embed "{}")'.format(webidl2_file))
447 
448    return ",\n".join(result)
449 
450def generate_interfaces_bindings(wd_test_file, base, files):
451    """
452    Generates appropriate bindings for each file in the WPT interfaces module.
453    Files are prefixed with /interfaces/ path.
454 
455    Only IDL files are included (*.idl) which are used by idlharness tests.
456    """
457 
458    result = []
459 
460    for file in files.to_list():
461        if file.extension != "idl":
462            # Skip non-IDL files
463            continue
464 
465        relative_path = module_relative_path(base, file)
466        file_path = wd_test_relative_path(wd_test_file, file)
467        result.append('(name = "/interfaces/{}", text = embed "{}")'.format(relative_path, file_path))
468 
469    return ",\n".join(result)
470 
471def generate_harness_modules(wd_test_file, files):
472    """
473    Generates a module entry for each file in the harness package
474    """
475    result = []
476 
477    for file in files.to_list():
478        relative_path = wd_test_relative_path(wd_test_file, file)
479        import_path = paths.basename(relative_path).removesuffix(".js")
480        result.append('(name = "harness/{}", esModule = embed "{}")'.format(import_path, relative_path))
481 
482    return ",\n".join(result)
483 
484### WPT server entrypoint
485### ---------------------
486### (Create a single no-args script that starts the WPT server)
487 
488WPT_ENTRYPOINT_SCRIPT_TEMPLATE = """
489# Make /usr/sbin/sysctl visible (Python needs to call it on macOS)
490export PATH="$PATH:/usr/sbin"
491 
492cd $(dirname $0)
493{python} wpt.py serve --no-h2 --config /dev/stdin <<EOF
494{{
495  "server_host": "$SIDECAR_HOSTNAME",
496  "check_subdomains": false,
497  "ports": {{
498    "http": [$HTTP_PORT, $HTTP_PORT_2],
499    "https": [$HTTPS_PORT, $HTTPS_PORT_2],
500    "http-public": [$HTTP_PUBLIC_PORT],
501    "https-public": [$HTTPS_PUBLIC_PORT],
502    "http-local": [$HTTP_LOCAL_PORT],
503    "wss": [$WSS_PORT],
504    "ws": [$WS_PORT]
505  }}
506}}
507EOF
508"""
509 
510def _wpt_server_entrypoint_impl(ctx):
511    """
512    This rule generates a script that starts the wpt server.
513 
514    This script is passed as the sidecar to wd_test.
515    We generate a script because Bazel doesn't want arguments in this context, and we use a custom
516    rule to do so because we want more control over paths and dependencies than with a genrule.
517    """
518    start_src = ctx.actions.declare_file("start.sh")
519    ctx.actions.write(
520        output = start_src,
521        is_executable = True,
522        content = WPT_ENTRYPOINT_SCRIPT_TEMPLATE.format(
523            python = ctx.file.python.short_path,
524        ),
525    )
526 
527    return DefaultInfo(
528        runfiles = ctx.runfiles(
529            files = [start_src],
530            transitive_files = depset(ctx.files.srcs + [ctx.file.python]),
531        ),
532        executable = start_src,
533    )
534 
535wpt_server_entrypoint = rule(
536    implementation = _wpt_server_entrypoint_impl,
537    attrs = {
538        # Python interpreter to use to run the WPT server
539        "python": attr.label(allow_single_file = True),
540        # All the Python code that should be visible
541        "srcs": attr.label_list(allow_files = True),
542    },
543)
544 
545### Path manipulation
546### -----------------
547 
548def module_relative_path(module_base, file):
549    """
550    Return the relative path of a file inside its parent WPT module.
551 
552    For example, within the 'dom/abort' module, the path
553    'dom/abort/resources/abort-signal-any-tests.js' would be referred to as
554    'resources/abort-signal-any-tests.js'
555    """
556 
557    return paths.relativize(file.short_path, module_base.short_path)
558 
559def relative_path(from_path, to_path):
560    """
561    Finds the relative path from from_path to to_path.
562    Based on <https://github.com/nodejs/node/blob/0a91e988cfe2a994e60780a4ce001e44812a9ad9/lib/path.js#L1280>
563    """
564 
565    # The algorithm we got from Node assumes both paths are absolute,
566    # so add a slash rather than change all the logic below
567    from_path = "/{}".format(from_path)
568    to_path = "/{}".format(to_path)
569 
570    from_start = 1
571    from_end = len(from_path)
572    from_len = from_end - from_start
573    to_start = 1
574    to_len = len(to_path) - to_start
575 
576    length = from_len if from_len < to_len else to_len
577    last_common_sep = -1
578    i = 0
579 
580    for i in range(length):
581        from_ch = from_path[from_start + i]
582        if from_ch != to_path[to_start + i]:
583            break
584        elif from_ch == "/":
585            last_common_sep = i
586 
587    # Compare paths to find the longest common path from root
588    if i == length:
589        if to_len > length:
590            if to_path[to_start + i] == "/":
591                # We get here if `from_path` is the exact base path for `to_path`.
592                # For example: from='/foo/bar'; to='/foo/bar/baz'
593                return to_path[to_start + i + 1:]
594 
595            if i == 0:
596                # We get here if `from` is the root
597                # For example: from='/'; to='/foo'
598                return to_path[to_start + i:]
599    elif from_len > length:
600        if from_path[from_start + i] == "/":
601            # We get here if `to` is the exact base path for `from`.
602            # For example: from='/foo/bar/baz'; to='/foo/bar'
603            last_common_sep = i
604        elif i == 0:
605            # We get here if `to` is the root.
606            # For example: from='/foo/bar'; to='/'
607            last_common_sep = 0
608 
609    out = ""
610 
611    # Generate the relative path based on the path difference between `to`
612    # and `from`.
613    for i in range(from_start + last_common_sep + 1, from_end + 1):
614        if i == from_end or from_path[i] == "/":
615            out += ".." if len(out) == 0 else "/.."
616 
617    # Lastly, append the rest of the destination (`to`) path that comes after
618    # the common path parts.
619    return out + to_path[to_start + last_common_sep:]
620 
621def wd_test_relative_path(wd_test_file, file):
622    """
623    Generates a path that can be used in a .wd-test file to refer to another file.
624 
625    Paths are relative to the .wd-test file.
626    """
627    return relative_path(paths.dirname(wd_test_file.short_path), file.short_path)
628 
629def is_in_resources_directory(file):
630    """
631    True if a given file is in the resources/ directory of a WPT module.
632    """
633    immediate_parent = paths.basename(file.dirname)
634    return immediate_parent == "resources"
635 
636def test_case_name(filename):
637    """
638    Converts a JS filename to a valid JS identifier for use as a test case name.
639 
640    WPT files are named with the convention some-words-with-hyphens.some-suffix.js.
641    We would turn this into someWordsWithHyphensSomeSuffix.
642    """
643 
644    words = (filename
645        .removesuffix(".js")
646        .replace(".", "-")
647        .replace("/", "-")
648        .split("-"))
649 
650    return words[0] + "".join([word.capitalize() for word in words[1:]])