Skip to content
rachel/workerd
File

Blob: src/wpt/harness/harness.ts

typescript669 lines
1// Copyright (c) 2017-2022 Cloudflare, Inc.
2// Licensed under the Apache 2.0 license found in the LICENSE file or at:
3//     https://opensource.org/licenses/Apache-2.0
4// Copyright © web-platform-tests contributors. BSD license
5// Adapted from Node.js. Copyright Joyent, Inc. and other Node contributors.
6//
7// Permission is hereby granted, free of charge, to any person obtaining a
8// copy of this software and associated documentation files (the
9// "Software"), to deal in the Software without restriction, including
10// without limitation the rights to use, copy, modify, merge, publish,
11// distribute, sublicense, and/or sell copies of the Software, and to permit
12// persons to whom the Software is furnished to do so, subject to the
13// following conditions:
14//
15// The above copyright notice and this permission notice shall be included
16// in all copies or substantial portions of the Software.
17//
18// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
21// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
22// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24// USE OR OTHER DEALINGS IN THE SOFTWARE.
25 
26import { default as path } from 'node:path';
27import {
28  FilterList,
29  type UnknownFunc,
30  sanitize_unpaired_surrogates,
31  getHostInfo,
32  getBindingPath,
33} from './common';
34 
35import { Test } from './test';
36 
37// These imports introduce functions into the global scope, so that WPT tests can call them
38 
39// Test, promise_test, async_test, test
40import './test';
41 
42// WPT assert_ functions
43import './assertions';
44 
45// Other functions from the WPT harness that need to be exposed to tests
46import './utils';
47 
48// Modifications we need to make to global types for testing
49import './globals';
50 
51type CommonOptions = {
52  // Brief explanation of why these options were set for a test
53  comment?: string;
54  // Print information about each subtest within this test as it runs
55  verbose?: boolean;
56  // Function executed before running the test
57  before?: () => void;
58  // Function executed after running the test
59  after?: () => void;
60  // Function that modifies the test code before it's run
61  replace?: (code: string) => string;
62  // Execute only this test
63  only?: boolean;
64  // If true, evaluate the test code within a top-level block. Otherwise, the test runs within the
65  // scope of an anonymous function.
66  runInGlobalScope?: boolean;
67};
68 
69type SuccessOptions = {
70  expectedFailures?: undefined;
71  disabledTests?: undefined;
72  omittedTests?: undefined;
73};
74 
75type ErrorOptions = {
76  // A comment is mandatory when there are expected failure, skipped tests or excluded tests
77  comment: string;
78 
79  // Known errors when running the tests
80  expectedFailures?: (string | RegExp)[] | true;
81 
82  // Tests that cannot be run for now, either due to harness limitations, workerd bugs or
83  // features not yet implemented.
84  disabledTests?: (string | RegExp)[] | true;
85 
86  // Tests that we have decided not to use. These are omitted from results and coverage
87  // calculations.
88  omittedTests?: (string | RegExp)[] | true;
89};
90 
91type TestRunnerOptions = CommonOptions & (SuccessOptions | ErrorOptions);
92 
93export type TestRunnerConfig = {
94  [key: string]: TestRunnerOptions;
95};
96 
97type Env = {
98  unsafe: { eval: (code: string) => unknown };
99  SIDECAR_HOSTNAME: string | null;
100  HTTP_PORT: string | null;
101  HTTPS_PORT: string | null;
102  [key: string]: unknown;
103};
104 
105type TestCase = {
106  test(_: unknown, env: Env): Promise<void> | void;
107};
108 
109// Singleton object used to pass test state between the runner and the harness functions available
110// to the evaled  WPT test code.
111class RunnerState {
112  // URL corresponding to the current test file, based on the WPT directory structure.
113  testUrl: URL;
114 
115  // Filename of the current test. Used in error messages.
116  testFileName: string;
117 
118  // The worker environment. Used to allow fetching resources in the test.
119  env: Env;
120 
121  // Makes test options available from assertion functions
122  options: TestRunnerOptions;
123 
124  // A test is pushed to this list as soon as it is discovered
125  subtests: Test[] = [];
126 
127  // Callbacks to be run once the entire test is done.
128  completionCallbacks: UnknownFunc[] = [];
129 
130  constructor(
131    testUrl: URL,
132    testFileName: string,
133    env: Env,
134    options: TestRunnerOptions
135  ) {
136    this.testUrl = testUrl;
137    this.testFileName = testFileName;
138    this.env = env;
139    this.options = options;
140  }
141 
142  async validate(): Promise<void> {
143    // Exception handling is set up on every promise in the test function that created it.
144    await Promise.all(this.subtests.map((t) => t.promise ?? Promise.resolve()));
145 
146    for (const cleanFn of this.completionCallbacks) {
147      cleanFn();
148    }
149 
150    const expectedFailures = new FilterList(this.options.expectedFailures);
151    const unexpectedFailures = [];
152 
153    for (const subtest of this.subtests) {
154      const err = subtest.error;
155 
156      if (err instanceof Error) {
157        if (!expectedFailures.delete(err.message)) {
158          err.message = sanitize_unpaired_surrogates(err.message);
159          console.warn(err);
160          unexpectedFailures.push(err.message);
161        } else if (this.options.verbose) {
162          err.message = sanitize_unpaired_surrogates(err.message);
163          console.warn('Expected failure: ', err);
164        }
165      }
166    }
167 
168    if (unexpectedFailures.length > 0) {
169      console.error(
170        'The following tests unexpectedly failed:',
171        JSON.stringify(unexpectedFailures, null, 2)
172      );
173    }
174 
175    const unexpectedSuccess = expectedFailures.getUnmatched();
176    // TODO(soon): Once we have a list of successful assertions, we should throw an error if a
177    // regex also matches successful tests. This can be done once we implement wpt.fyi report
178    // generation.
179 
180    if (unexpectedSuccess.size > 0) {
181      console.error(
182        'The following tests were expected to fail but instead succeeded:',
183        JSON.stringify(
184          [...unexpectedSuccess].map((v) => v.toString()),
185          null,
186          2
187        )
188      );
189    }
190 
191    if (unexpectedFailures.length > 0 || unexpectedSuccess.size > 0) {
192      throw new Error(
193        `${this.testFileName} failed. Please update the test config.`
194      );
195    }
196  }
197}
198 
199declare global {
200  // Current RunnerState
201  var state: RunnerState;
202  // All RunnerStates (to get results later)
203  var results: { [file: string]: RunnerState };
204}
205 
206const COLORS = {
207  FOREGROUND: 30,
208  BACKGROUND: 40,
209  BLACK: 0,
210  RED: 1,
211  GREEN: 2,
212  YELLOW: 3,
213  BLUE: 4,
214  MAGENTA: 5,
215  CYAN: 6,
216  WHITE: 7,
217};
218 
219function colorPrint(
220  fn: (...args: unknown[]) => void,
221  color: number
222): (...args: unknown[]) => void {
223  return (...args: unknown[]): void => {
224    fn(`\x1b[${color}m`, ...args, '\x1b[0m');
225  };
226}
227 
228globalThis.console.info = colorPrint(
229  console.info,
230  COLORS.FOREGROUND + COLORS.BLUE
231);
232globalThis.console.warn = colorPrint(
233  console.warn,
234  COLORS.FOREGROUND + COLORS.YELLOW
235);
236globalThis.console.error = colorPrint(
237  console.error,
238  COLORS.FOREGROUND + COLORS.RED
239);
240 
241class WPTMetadata {
242  // Specifies the Javascript global scopes for the test. (Not currently supported)
243  global: string[] = [];
244  // Specifies additional JS scripts to execute.
245  scripts: string[] = [];
246  // Adjusts the timeout for the tests in this file. (Not currently supported)
247  timeout?: string;
248  // Sets a human-readable title for the entire test file. (Not currently supported.)
249  title?: string;
250  // Specifies a variant of this test, which can be used in subsetTestByKey (Not currently supported.)
251  variants: URLSearchParams[] = [];
252}
253 
254function parseWptMetadata(code: string): WPTMetadata {
255  const meta = new WPTMetadata();
256 
257  for (const { groups } of code.matchAll(
258    /\/\/ META: (?<key>\w+)=(?<value>.+)/g
259  )) {
260    if (!groups || !groups.key || !groups.value) {
261      continue;
262    }
263 
264    switch (groups.key) {
265      case 'global':
266        meta.global = groups.value.split(',');
267        break;
268 
269      case 'script': {
270        meta.scripts.push(groups.value);
271        break;
272      }
273 
274      case 'timeout':
275        meta.timeout = groups.value;
276        break;
277 
278      case 'title':
279        meta.title = groups.value;
280        break;
281 
282      case 'variant':
283        meta.variants.push(new URLSearchParams(groups.value));
284        break;
285    }
286  }
287 
288  return meta;
289}
290 
291const EXCLUDED_PATHS = new Set([
292  // Implemented in harness.ts
293  '/common/subset-tests-by-key.js',
294  '/common/subset-tests.js',
295  '/resources/utils.js',
296  '/common/utils.js',
297  '/common/get-host-info.sub.js',
298  '/common/sab.js',
299]);
300 
301function replaceInterpolation(code: string): string {
302  const hostInfo = getHostInfo();
303 
304  return code
305    .replace(/\{\{host\}\}/g, hostInfo.REMOTE_HOST)
306    .replace(/\{\{hosts\[alt\]\[www\]\}\}/g, hostInfo.REMOTE_HOST)
307    .replace(/\{\{ports\[http\]\[0\]\}\}/g, hostInfo.HTTP_PORT)
308    .replace(/\{\{ports\[http\]\[1\]\}\}/g, hostInfo.HTTP_PORT)
309    .replace(/\{\{ports\[https\]\[0\]\}\}/g, hostInfo.HTTPS_PORT)
310    .replace(/\{\{ports\[ws\]\[0\]\}\}/g, hostInfo.WS_PORT)
311    .replace(/\{\{ports\[wss\]\[0\]\}\}/g, hostInfo.WSS_PORT)
312    .replace(/\{\{ports\[h2\]\[0\]\}\}/g, hostInfo.HTTPS_PORT);
313}
314 
315function getCodeAtPath(
316  env: Env,
317  base: string,
318  rawPath: string,
319  replace?: (code: string) => string
320): string {
321  const bindingPath = getBindingPath(base, rawPath);
322 
323  if (EXCLUDED_PATHS.has(bindingPath)) {
324    return '';
325  }
326 
327  if (typeof env[bindingPath] != 'string') {
328    // We only have access to the files explicitly declared in the .wd-test, not the full WPT
329    // checkout, so it's possible for tests to include things we can't load.
330    throw new Error(
331      `Test file ${bindingPath} not found. Update wpt_test.bzl to handle this case.`
332    );
333  }
334 
335  let code = env[bindingPath];
336  if (replace) {
337    code = replace(code);
338  }
339 
340  return replaceInterpolation(code);
341}
342 
343function evalAsBlock(
344  env: Env,
345  runInGlobalScope: boolean,
346  files: string[]
347): void {
348  const code = files.join('\n');
349 
350  if (runInGlobalScope) {
351    // In this mode, the scope of const, let and function declarations is limited to stop tests
352    // from interfering with each other.
353    env.unsafe.eval(`{ ${code} }`);
354  } else {
355    // Additionally, use an IIFE to limit the scope of var and manipulation of the global scope
356    env.unsafe.eval(`(function () { ${code} })();`);
357  }
358}
359 
360type Runner = {
361  run: (file: string) => TestCase | Record<string, never>;
362  printResults: () => TestCase;
363};
364 
365export function createRunner(
366  config: TestRunnerConfig,
367  moduleBase: string,
368  allTestFiles: string[]
369): Runner {
370  const testsNotFound = new Set(Object.keys(config)).difference(
371    new Set(allTestFiles)
372  );
373 
374  if (testsNotFound.size > 0) {
375    throw new Error(
376      `Configuration was provided for the following tests that have not been found in the WPT repo: ${[...testsNotFound].join(', ')}`
377    );
378  }
379 
380  const onlyFlagUsed = Object.values(config).some((options) => options.only);
381  globalThis.results = {};
382 
383  return {
384    run(file: string): TestCase | Record<string, never> {
385      if (onlyFlagUsed && !config[file]?.only) {
386        // Return an empty object to avoid printing extra output from all the other disabled test cases.
387        return {};
388      }
389 
390      return {
391        async test(_: unknown, env: Env): Promise<void> {
392          return runTest(config[file], env, moduleBase, file);
393        },
394      };
395    },
396    printResults(): TestCase {
397      return {
398        test(_: unknown, env: Env): void {
399          printResults(config, allTestFiles, moduleBase, env);
400        },
401      };
402    },
403  };
404}
405 
406async function runTest(
407  options: TestRunnerOptions | undefined,
408  env: Env,
409  moduleBase: string,
410  file: string
411): Promise<void> {
412  if (!options) {
413    throw new Error(
414      `Missing test configuration for ${file}. Specify '${file}': {} for default options.`
415    );
416  }
417 
418  const testUrl = new URL(
419    path.join(moduleBase, file),
420    `http://${env.SIDECAR_HOSTNAME ?? 'localhost'}`
421  );
422 
423  // If the environment variable HTTP_PORT is set, the wpt server is running as a sidecar.
424  // Update the URL's port so we can connect to it
425  testUrl.port = env.HTTP_PORT ?? '';
426 
427  globalThis.state = new RunnerState(testUrl, file, env, options);
428  globalThis.results[file] = globalThis.state;
429 
430  if (options.disabledTests === true) {
431    console.warn(
432      `All tests in ${file} have been disabled because "${options.comment}".`
433    );
434    return;
435  }
436 
437  if (options.omittedTests === true) {
438    console.warn(
439      `All tests in ${file} have been omitted because "${options.comment}".`
440    );
441    return;
442  }
443 
444  const meta = parseWptMetadata(String(env[file]));
445 
446  if (options.before) {
447    options.before();
448  }
449 
450  const files = [];
451 
452  for (const script of meta.scripts) {
453    files.push(getCodeAtPath(env, path.dirname(file), script));
454  }
455 
456  files.push(getCodeAtPath(env, './', file, options.replace));
457  evalAsBlock(env, options.runInGlobalScope ?? false, files);
458 
459  if (options.after) {
460    options.after();
461  }
462 
463  // The WPT IDL harness (idlharness.js) uses eval() inside async promise callbacks
464  // (e.g., to create test objects via `eval('new URL("http://foo")')`).  The workerd
465  // runtime only allows eval() while env.unsafe.eval() is on the call stack (the
466  // evalAllowed flag is reset once env.unsafe.eval returns).  Async test callbacks
467  // run when state.validate() awaits their promises, so native eval() would fail.
468  // Install the shim here — after evalAsBlock and options.after have finished — so
469  // it covers exactly the async-callback phase.
470  const originalEval = globalThis.eval;
471  globalThis.eval = ((code: string): unknown =>
472    env.unsafe.eval(code)) as typeof globalThis.eval;
473 
474  try {
475    await globalThis.state.validate();
476  } finally {
477    globalThis.eval = originalEval;
478  }
479}
480 
481function printResults(
482  config: TestRunnerConfig,
483  allTestFiles: string[],
484  moduleBase: string,
485  env: Env
486): void {
487  const results: string[] = [];
488 
489  if (env.GEN_TEST_CONFIG) {
490    results.push(generateConfig(config, allTestFiles, moduleBase));
491  }
492 
493  if (env.GEN_TEST_REPORT) {
494    results.push(generateReport(config));
495  }
496 
497  if (env.GEN_TEST_STATS) {
498    results.push(generateStats(moduleBase, config));
499  }
500 
501  console.log(results.join('\n\n***\n\n'));
502}
503 
504function generateConfig(
505  config: TestRunnerConfig,
506  allTestFiles: string[],
507  moduleBase: string
508): string {
509  const generatedConfig: TestRunnerConfig = {};
510  for (const file of allTestFiles) {
511    // Copy config if the user has created any so far, else initialize to blank
512    generatedConfig[file] = config[file] ?? {};
513  }
514 
515  return `\x1b[1;7;35m*** Copy this config to src/wpt/${moduleBase}-test.ts ***\x1b[0m
516
517// Copyright (c) 2017-2022 Cloudflare, Inc.
518// Licensed under the Apache 2.0 license found in the LICENSE file or at:
519//     https://opensource.org/licenses/Apache-2.0
520
521import { type TestRunnerConfig } from 'harness/harness';
522
523export default ${JSON.stringify(generatedConfig, null, 2)} satisfies TestRunnerConfig;`;
524}
525 
526class WPTTestResult {
527  test: string;
528  status: 'OK' | 'ERROR';
529  subtests: WPTSubtestResult[] = [];
530  // TODO(soon): Track elapsed time
531  duration: number = 0;
532 
533  constructor(result: RunnerState, options: TestRunnerOptions) {
534    this.test = WPTTestResult.#getTestNameFromUrl(result.testUrl);
535    this.status = options.disabledTests === true ? 'ERROR' : 'OK';
536    this.subtests = result.subtests.map((r) => new WPTSubtestResult(r));
537  }
538 
539  static #getTestNameFromUrl(testUrl: URL): string {
540    const testNameUrl = new URL(testUrl);
541    testNameUrl.pathname = testNameUrl.pathname.replace('.js', '.html');
542    return testNameUrl.href.slice(testNameUrl.origin.length);
543  }
544}
545 
546class WPTSubtestResult {
547  name: string;
548  status: 'PASS' | 'FAIL';
549  message?: string;
550  isExpectedFailure?: true;
551 
552  constructor(result: Test) {
553    this.name = sanitize_unpaired_surrogates(result.name);
554    if (result.error instanceof Error) {
555      this.message = result.error.message;
556      // TODO(soon): This is true in main, but not necessarily if you run a report in local dev
557      this.isExpectedFailure = true;
558      this.status = 'FAIL';
559    } else if (result.error === 'DISABLED') {
560      this.isExpectedFailure = true;
561      this.status = 'FAIL';
562    } else {
563      this.status = 'PASS';
564    }
565  }
566}
567 
568function generateReport(config: TestRunnerConfig): string {
569  const results: WPTTestResult[] = [];
570  for (const [file, options] of Object.entries(config)) {
571    const testResult = globalThis.results[file];
572    if (!testResult) {
573      throw new Error(
574        `Unable to find test results for ${file}. This is probably a harness bug`
575      );
576    }
577 
578    results.push(new WPTTestResult(testResult, options));
579  }
580 
581  return JSON.stringify({ results }, null, 2);
582}
583 
584class Stats {
585  moduleBase: string;
586  coverage = new CoverageStats();
587  pass = new PassStats();
588 
589  constructor(moduleBase: string) {
590    this.moduleBase = moduleBase;
591  }
592 
593  toList(): unknown[] {
594    return [this.moduleBase, ...this.coverage.toList(), ...this.pass.toList()];
595  }
596}
597class CoverageStats {
598  ok: number = 0;
599  disabled: number = 0;
600 
601  get total(): number {
602    return this.ok + this.disabled;
603  }
604 
605  get ok_percent(): number {
606    return (this.ok / this.total) * 100;
607  }
608 
609  toList(): unknown[] {
610    return [this.ok, this.disabled, this.total, this.ok_percent.toFixed(0)];
611  }
612}
613 
614class PassStats {
615  pass: number = 0;
616  fail: number = 0;
617  disabled: number = 0;
618 
619  get total(): number {
620    return this.pass + this.fail + this.disabled;
621  }
622 
623  get pass_percent(): number {
624    return (this.pass / this.total) * 100;
625  }
626 
627  toList(): unknown[] {
628    return [
629      this.pass,
630      this.fail,
631      this.disabled,
632      this.total,
633      this.pass_percent.toFixed(0),
634    ];
635  }
636}
637 
638function generateStats(moduleBase: string, config: TestRunnerConfig): string {
639  const stats = new Stats(moduleBase);
640 
641  for (const [file, options] of Object.entries(config)) {
642    if (options.disabledTests === true) {
643      stats.coverage.disabled++;
644    } else if (options.omittedTests !== true) {
645      stats.coverage.ok++;
646    }
647 
648    const testResult = globalThis.results[file];
649 
650    if (!testResult) {
651      throw new Error(
652        `Unable to find test results for ${file}. This is probably a harness bug`
653      );
654    }
655 
656    for (const subtestResult of testResult.subtests) {
657      if (subtestResult.error === 'DISABLED') {
658        stats.pass.disabled++;
659      } else if (subtestResult.error instanceof Error) {
660        stats.pass.fail++;
661      } else if (subtestResult.error === undefined) {
662        stats.pass.pass++;
663      }
664    }
665  }
666 
667  return JSON.stringify(stats.toList());
668}