// Copyright (c) 2024 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 import { strictEqual, deepStrictEqual, notStrictEqual, rejects, throws, } from 'node:assert'; const arrayIterator = [][Symbol.iterator](); const arrayIteratorPrototype = Object.getPrototypeOf( Object.getPrototypeOf(arrayIterator) ); export const passthroughWithContent = { async test() { const response = new Response('hello', { headers: { foo: 'bar', }, }); strictEqual(response.headers.get('foo'), 'bar'); const rewriter = new HTMLRewriter(); const newResponse = rewriter.transform(response); notStrictEqual(response, newResponse); strictEqual(newResponse.headers.get('foo'), 'bar'); strictEqual(await newResponse.text(), 'hello'); }, }; export const passthroughWithContentAndHandler = { async test() { const response = new Response('

hello

', { headers: { foo: 'bar', }, }); strictEqual(response.headers.get('foo'), 'bar'); const rewriter = new HTMLRewriter().on('h3', { element() {}, comments() {}, text() {}, }); const newResponse = rewriter.transform(response); notStrictEqual(response, newResponse); strictEqual(newResponse.headers.get('foo'), 'bar'); strictEqual(await newResponse.text(), '

hello

'); }, }; export const passthroughWithContentAndAsyncHandler = { async test() { const response = new Response('

hello

', { headers: { foo: 'bar', }, }); strictEqual(response.headers.get('foo'), 'bar'); const rewriter = new HTMLRewriter().on('h3', { async test() { await scheduler.wait(10); }, async element() { await scheduler.wait(10); }, async comments() { await scheduler.wait(10); }, }); const newResponse = rewriter.transform(response); notStrictEqual(response, newResponse); strictEqual(newResponse.headers.get('foo'), 'bar'); strictEqual(await newResponse.text(), '

hello

'); }, }; export const passthroughWithContentAndAsyncHandler2 = { async test() { const response = new Response('

hello

', { headers: { foo: 'bar', }, }); strictEqual(response.headers.get('foo'), 'bar'); const { promise, resolve } = Promise.withResolvers(); const rewriter = new HTMLRewriter().on('h3', { async test() { await promise; }, async element() { await scheduler.wait(10); }, async comments() { await scheduler.wait(10); }, }); const newResponse = rewriter.transform(response); // We can resolve the promise after calling transform and the transform // will complete as expected. resolve(); notStrictEqual(response, newResponse); strictEqual(newResponse.headers.get('foo'), 'bar'); strictEqual(await newResponse.text(), '

hello

'); }, }; export const passthroughWithContentStream = { async test() { const { readable, writable } = new TransformStream(); const rewriter = new HTMLRewriter().on('h3', { async text(content) { await scheduler.wait(10); }, }); const response = rewriter.transform(new Response(readable)); const writer = writable.getWriter(); const enc = new TextEncoder(); const results = await Promise.all([ response.text(), writer.write(enc.encode('

hello

')), writer.close(), ]); strictEqual(results[0], '

hello

'); }, }; export const passthroughWithEmptyStream = { async test() { const { readable, writable } = new TransformStream(); const rewriter = new HTMLRewriter().on('h3', { async text(content) { await scheduler.wait(10); }, }); const response = rewriter.transform(new Response(readable)); const writer = writable.getWriter(); const results = await Promise.all([response.text(), writer.close()]); strictEqual(results[0], ''); }, }; export const asyncElementHandler = { async test() { const rewriter = new HTMLRewriter().on('body', { async element(e) { await scheduler.wait(10); e.setInnerContent('world'); }, }); const response = rewriter.transform(new Response('hello')); strictEqual(await response.text(), 'world'); }, }; export const asyncCommentHandler = { async test() { const rewriter = new HTMLRewriter().on('body', { async comments(comment) { await scheduler.wait(10); if (comment.text == 'hello') { comment.text = 'world'; } }, }); const response = rewriter.transform( new Response('') ); strictEqual(await response.text(), ''); }, }; export const objectHandlers = { async test() { class DocumentContentHandlers { deadTokens = {}; doctypeCount = 0; commentCount = 0; textCount = 0; expectedErrors = []; doctype(token) { if (!this.deadTokens.doctype) { this.deadTokens.doctype = token; } ++this.doctypeCount; this.sawDoctype = JSON.stringify(token); } comments(token) { if (!this.deadTokens.comment) { this.deadTokens.comment = token; } ++this.commentCount; } text(token) { if (!this.deadTokens.text) { this.deadTokens.text = token; } ++this.textCount; } end(token) { this.reachedEnd = true; } } class ElementContentHandlers { deadTokens = {}; elementCount = 0; commentCount = 0; textCount = 0; expectedErrors = []; element(token) { if (!this.deadTokens.element) { this.deadTokens.element = token; } if (!this.deadTokens.attributesIterator) { this.deadTokens.attributesIterator = token.attributes; } ++this.elementCount; // Exercise all the different methods on Element. if ( token.tagName === 'body' && token.hasAttribute('foo') && !token.hasAttribute('baz') && token.getAttribute('foo') === 'bar' ) { token.removeAttribute('foo'); token.setAttribute('baz', 'qux'); try { token.tagName = 'should throw'; throw new Error('should have thrown'); } catch (e) { this.expectedErrors.push(e.message); } token.tagName = 'tail'; // These will show up in order in the response body. token.before('<1>'); token.before('<2>', { html: false }); token.before('<3>\n', null); token.before('', { html: true }); // These will show up in reverse order in the response body. token.prepend('hello, ', { html: true }); token.prepend('<6>\n'); token.prepend('<5>', { html: false }); token.prepend('\n<4>', null); // Iterator tests. this.sawAttributes = JSON.stringify([...token.attributes]); let iterator = token.attributes; let iteratorPrototype = Object.getPrototypeOf( Object.getPrototypeOf(iterator) ); if (iteratorPrototype !== arrayIteratorPrototype) { throw new Error( 'attributes iterator does not have iterator prototype' ); } // Run the iterator down until it's done. for (let [_k, _v] of iterator) { // intentionally empty } // .next() should now be idempotent. let result = iterator.next(); let result2 = iterator.next(); if ( result.done !== result2.done || result.value !== result2.value || !result.done || result.value ) { throw new Error( 'exhausted iterator should continually return done' ); } } else if (token.tagName === 'remove') { let mode = token.getAttribute('mode'); if (mode === null) { throw new Error("missing attribute on 'remove' element"); } if (token.removed) { throw new Error('element should not have been removed yet'); } if (mode === 'all') { token.remove(); } else { token.removeAndKeepContent(); } if (!token.removed) { throw new Error('element should have been removed now'); } } else if (token.tagName === 'after') { let isHtml = token.getAttribute('is-html'); let html = isHtml === 'true' ? true : false; token.after('', { html }); } else if (token.tagName === 'append') { let isHtml = token.getAttribute('is-html'); let html = isHtml === 'true' ? true : false; token.append('', { html }); } else if (token.tagName === 'replace') { let isHtml = token.getAttribute('is-html'); let html = isHtml === 'true' ? true : false; token.replace('', { html }); } else if (token.tagName === 'set-inner-content') { let isHtml = token.getAttribute('is-html'); let html = isHtml === 'true' ? true : false; token.setInnerContent('', { html }); } else if (token.tagName === 'set-attribute') { if (!token.hasAttribute('foo')) { throw new Error('element should have had attribute'); } let attr = token.getAttribute('foo'); if (attr !== '') { throw new Error('element attribute should have been empty'); } token.setAttribute('foo', 'bar'); if (token.getAttribute('nonexistent')) { throw new Error('attribute should not exist'); } } } comments(token) { if (!this.deadTokens.comment) { this.deadTokens.comment = token; } ++this.commentCount; // Exercise all the different methods on Comment. if (token.text === ' SET TEXT PROPERTY ') { token.text = ' text property has been set '; } else if (token.text === ' REMOVE ME ') { if (token.removed) { throw new Error("Shouldn't be removed yet"); } token.remove(); if (!token.removed) { throw new Error('Should be removed now'); } } else if (token.text === ' REPLACE ME ') { if (token.removed) { throw new Error("Shouldn't be removed yet"); } token.replace('this will get overwritten'); if (!token.removed) { throw new Error('Should be removed now'); } token.replace('', null); if (!token.removed) { throw new Error('Should still be removed'); } token.before('', { html: true }); } } text(token) { if (!this.deadTokens.text) { this.deadTokens.text = token; } ++this.textCount; if (token.lastInTextNode && token.text.length > 0) { throw new Error('last text chunk has non-zero length'); } else if (!token.lastInTextNode && token.text.length === 0) { throw new Error('non-last text chunk has zero length'); } if (token.text === 'world') { token.before('again, '); token.after('...'); if (token.removed) { throw new Error("Shouldn't be removed yet"); } token.replace('this will get overwritten'); if (!token.removed) { throw new Error('Should be removed now'); } token.replace('', { html: true }); if (!token.removed) { throw new Error('Should still be removed'); } } else if (token.text === 'REMOVE ME\n') { if (token.removed) { throw new Error("Shouldn't be removed yet"); } token.remove(); if (!token.removed) { throw new Error('Should be removed now'); } } } } let documentHandlers = new DocumentContentHandlers(); let elementHandlers = new ElementContentHandlers(); const rewriter = new HTMLRewriter() .onDocument(documentHandlers) .on('*', elementHandlers); let _count = 0; const enc = new TextEncoder(); const kInput = [ '', '', 'document-level text', 'world

REMOVE ME', 'inner content', 'inner content', 'inner content', 'inner content', 'inner content', 'inner content', 'inner content', 'inner content', 'inner content', 'inner content', '', '', '', '', '', '', '', ]; const kResult = `document-level text<1><2><3> <4><5><6> hello, again, ...

REMOVE MEinner contentinner contentinner content<after>inner contentinner content<append><replace><set-inner-content>`; const readable = new ReadableStream({ async pull(controller) { await scheduler.wait(1); if (kInput.length > 0) { controller.enqueue(enc.encode(kInput.shift())); } else { controller.close(); } }, }); const response = rewriter.transform(new Response(readable)); // At this point, we should not have seen any tokens. strictEqual(elementHandlers.deadTokens.element, undefined); strictEqual(await response.text(), kResult); // Now we've seen tokens notStrictEqual(elementHandlers.deadTokens.element, undefined); // Verify that tokens are invalidated outside handler execution scope. throws(() => documentHandlers.deadTokens.doctype.publicId, { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); throws(() => documentHandlers.deadTokens.comment.text, { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); throws(() => documentHandlers.deadTokens.text.text, { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); throws(() => elementHandlers.deadTokens.element.getAttribute('foo'), { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); throws(() => elementHandlers.deadTokens.attributesIterator.next(), { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); throws(() => elementHandlers.deadTokens.comment.text, { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); throws(() => elementHandlers.deadTokens.text.text, { message: 'This content token is no longer valid. Content tokens are only ' + 'valid during the execution of the relevant content handler.', }); strictEqual(documentHandlers.doctypeCount, 1); strictEqual(documentHandlers.commentCount, 4); strictEqual(documentHandlers.textCount, 26); strictEqual(documentHandlers.reachedEnd, true); strictEqual(elementHandlers.elementCount, 15); strictEqual(elementHandlers.commentCount, 3); strictEqual(elementHandlers.textCount, 24); deepStrictEqual(elementHandlers.expectedErrors, [ 'Parser error: ` ` character is forbidden in the tag name', ]); }, }; export const manualWriting = { async test() { const { readable, writable } = new IdentityTransformStream(); const response = new HTMLRewriter() .on('*', { element(element) { element.prepend('foo '); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); // Because this variation uses IdentityTransformStream, we must // initiate the read before doing the writes. const promise = response.text(); await writer.write(encoder.encode('')); await writer.write(encoder.encode('bar')); await writer.write(encoder.encode('')); await writer.close(); strictEqual(await promise, 'foo bar'); }, }; export const manualWriting2 = { async test() { const { readable, writable } = new TransformStream(); const response = new HTMLRewriter() .on('*', { element(element) { element.prepend('foo '); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); await writer.write(encoder.encode('')); await writer.write(encoder.encode('bar')); await writer.write(encoder.encode('')); await writer.close(); // This variation uses the JavaScript TransformStream, so we can // initiate the read after doing the writes. const promise = response.text(); strictEqual(await promise, 'foo bar'); }, }; export const streamingReplacement = { async test() { const { readable, writable } = new TransformStream(); const response = new HTMLRewriter() .on('*', { async element(element) { const dataStream = ( await fetch('data:,the quick brown fox jumped over the lazy dog%20') ).body; element.prepend(dataStream, { html: false }); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); await writer.write(encoder.encode('')); await writer.write(encoder.encode('bar')); await writer.write(encoder.encode('')); await writer.close(); // This variation uses the JavaScript TransformStream, so we can // initiate the read after doing the writes. const promise = response.text(); strictEqual( await promise, `the quick brown fox jumped over the lazy dog bar` ); }, }; export const streamingReplacementHTML = { async test() { const { readable, writable } = new TransformStream(); const response = new HTMLRewriter() .on('*', { async element(element) { const dataStream = ( await fetch('data:,such markup much wow ') ).body; element.prepend(dataStream, { html: true }); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); await writer.write(encoder.encode('')); await writer.write(encoder.encode('bar')); await writer.write(encoder.encode('')); await writer.close(); // This variation uses the JavaScript TransformStream, so we can // initiate the read after doing the writes. const promise = response.text(); strictEqual( await promise, `such markup much wowbar` ); }, }; export const streamingReplacementReplace = { async test() { const { readable, writable } = new TransformStream(); const response = new HTMLRewriter() .on('.dinosaur', { async element(element) { const dataStream = (await fetch('data:,goodbye world')).body; element.replace(dataStream, { html: false }); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); await writer.write(encoder.encode('')); await writer.write( encoder.encode('

hello world
') ); await writer.write(encoder.encode('')); await writer.close(); // This variation uses the JavaScript TransformStream, so we can // initiate the read after doing the writes. const promise = response.text(); strictEqual(await promise, `goodbye world`); }, }; export const streamingReplacementMultiple = { async test() { const { readable, writable } = new TransformStream(); const response = new HTMLRewriter() .on('*', { async element(element) { element.prepend(await fetch('data:,alpha%20')); element.append(await fetch('data:,%20gamma')); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); await writer.write(encoder.encode('')); await writer.write(encoder.encode('beta')); await writer.write(encoder.encode('')); await writer.close(); // This variation uses the JavaScript TransformStream, so we can // initiate the read after doing the writes. const promise = response.text(); strictEqual(await promise, `alpha beta gamma`); }, }; export const streamingReplacementBadUTF8 = { async test() { const { readable, writable } = new TransformStream(); const response = new HTMLRewriter() .on('*', { async element(element) { element.prepend(await fetch('data:,garbage%e2%28%a1')); }, }) .transform(new Response(readable)); const writer = writable.getWriter(); const encoder = new TextEncoder(); await writer.write(encoder.encode('')); await writer.write(encoder.encode('bar')); await writer.write(encoder.encode('')); await writer.close(); // This variation uses the JavaScript TransformStream, so we can // initiate the read after doing the writes. await rejects(response.text(), { message: 'Parser error: Invalid UTF-8' }); }, }; export const appendOnEnd = { async test() { const kInput = ''; const kSuffix = ''; const result = new HTMLRewriter() .onDocument({ end(end) { end.append(kSuffix, { html: true }); }, }) .transform(new Response(kInput)); strictEqual(await result.text(), kInput + kSuffix); }, }; export const interleavedAsyncHandlers = { async test() { class OneTimeBarrier { constructor(limit) { this.limit = limit; this.current = 0; let resolve; this.promise = new Promise((r) => (resolve = r)); this.resolver = resolve; } async wait() { this.current += 1; if (this.current >= this.limit) { this.resolver(); } else { await this.promise; } } } const barrier = new OneTimeBarrier(2); const responses = await Promise.all([ new HTMLRewriter() .on('body', { async element(e) { await barrier.wait(); e.setInnerContent('foo bar'); }, }) .transform(new Response('body value')) .text(), new HTMLRewriter() .on('body', { async element(e) { await barrier.wait(); e.remove(); }, }) .transform( new Response('') ) .arrayBuffer(), ]); const body = responses[0]; const blank = responses[1]; const inserted = new HTMLRewriter() .on('html', { element(e) { e.append(body, { html: true }); }, }) .transform(new Response(blank)); strictEqual( await inserted.text(), 'foo bar' ); }, }; export const exceptionInHandler = { async test() { const response = new HTMLRewriter() .on('*', { text() { throw new Error('boom'); }, }) .transform(new Response('hello')); await rejects(response.text(), { message: 'boom', }); }, }; export const exceptionInAsyncHandler = { async test() { const response = new HTMLRewriter() .on('*', { async text() { throw new Error('boom'); }, }) .transform(new Response('hello')); await rejects(response.text(), { message: 'boom', }); }, }; export const invalidEncoding = { async test() { const response = new Response('hello', { headers: { 'content-type': 'text/html; charset=invalid', }, }); throws( () => { new HTMLRewriter().on('*', {}).transform(response); }, { message: 'Parser error: Unknown character encoding has been provided.', } ); }, }; export const exceptionPropagation = { async test() { const { readable, writable } = new IdentityTransformStream(); const response = new HTMLRewriter().transform(new Response(readable)); response.body.cancel(new Error('boom')); const writer = writable.getWriter(); const enc = new TextEncoder(); await writer.write(enc.encode('test')); rejects(writer.write(enc.encode('test')), { message: 'boom', }); }, }; // handled HTMLRewriter errors must not poison the IoContext. export const continueAfterException = { async test() { const errorResponse = new HTMLRewriter() .on('*', { element() { throw new Error('intentional error for testing'); }, }) .transform(new Response('
test
')); await rejects(errorResponse.text(), { message: 'intentional error for testing', }); const successResponse = new HTMLRewriter() .on('div', { element(el) { el.setInnerContent('success'); }, }) .transform(new Response('
original
')); strictEqual(await successResponse.text(), '
success
'); }, }; export const continueAfterException2 = { async test() { const errorResponse = new HTMLRewriter() .on('*', { element() { throw new Error('intentional error for pipeTo testing'); }, }) .transform(new Response('
test
')); const { writable } = new TransformStream(); await rejects(errorResponse.body.pipeTo(writable), { message: 'intentional error for pipeTo testing', }); const successResponse = new HTMLRewriter() .on('div', { element(el) { el.setInnerContent('success after pipeTo'); }, }) .transform(new Response('
original
')); strictEqual( await successResponse.text(), '
success after pipeTo
' ); }, }; export const sameToken = { async test() { const obj = {}; let element; const r = new HTMLRewriter() .on('*', { element(e) { element = e; strictEqual(e.hi, undefined); e.hi = 'test'; strictEqual(e.hi, 'test'); e.hi = 'hi'; e.obj = obj; e.replace('foo'); }, }) .on('img', { element(e) { notStrictEqual(e, element); notStrictEqual(e.hi, 'hi'); notStrictEqual(e.obj, obj); // The HTMLRewriter creates a fresh new Element/Doctype/Text // object for each handler, thus `e.hi` will yield undefined even if we // assigned it in the first handler. // See https://jira.cfdata.org/browse/EW-2200. e.replace(e.hi); }, }) .transform(new Response('')) .text(); await r; }, }; // Regression test for VULN-122672: HTMLRewriter AttributesIterator UAF. // When element attributes are modified during iteration (adding new attributes // that cause the underlying Vec to reallocate), the iterator's stale pointers // would read from freed memory. The fix invalidates all live iterators when // setAttribute() or removeAttribute() is called. export const attributesIteratorInvalidatedOnSetAttribute = { async test() { const html = `
test
`; const rewriter = new HTMLRewriter().on('div', { element(el) { const iter = el.attributes[Symbol.iterator](); // Read first attribute - valid. const first = iter.next(); strictEqual(first.done, false); // Mutate attributes — this must invalidate the iterator. el.setAttribute('newattr', 'value'); // Subsequent next() must throw, not read freed memory. throws(() => iter.next(), { message: 'The attributes of this element have been modified during iteration. ' + 'You must create a new iterator after modifying attributes.', }); }, }); await rewriter.transform(new Response(html)).text(); }, }; export const attributesIteratorInvalidatedOnRemoveAttribute = { async test() { const html = `
test
`; const rewriter = new HTMLRewriter().on('div', { element(el) { const iter = el.attributes[Symbol.iterator](); iter.next(); // consume first // removeAttribute also must invalidate the iterator. el.removeAttribute('b'); throws(() => iter.next(), { message: 'The attributes of this element have been modified during iteration. ' + 'You must create a new iterator after modifying attributes.', }); }, }); await rewriter.transform(new Response(html)).text(); }, }; // After mutation, creating a fresh iterator must work normally. export const attributesIteratorFreshAfterMutation = { async test() { const html = `
test
`; let attrs = []; const rewriter = new HTMLRewriter().on('div', { element(el) { el.setAttribute('b', '2'); // A new iterator created after mutation should work fine. for (const [name, value] of el.attributes) { attrs.push([name, value]); } }, }); await rewriter.transform(new Response(html)).text(); // Should see both original and newly-set attribute. strictEqual(attrs.length, 2); deepStrictEqual(attrs[0], ['a', '1']); deepStrictEqual(attrs[1], ['b', '2']); }, }; // Iterating without mutation must still work as before. export const attributesIteratorNormalIteration = { async test() { const html = `
test
`; let attrs = []; const rewriter = new HTMLRewriter().on('div', { element(el) { for (const [name, value] of el.attributes) { attrs.push([name, value]); } }, }); await rewriter.transform(new Response(html)).text(); deepStrictEqual(attrs, [ ['x', '1'], ['y', '2'], ['z', '3'], ]); }, }; export const svgNamespace = { async test() { const response = new Response(` `); let namespace; await new HTMLRewriter() .on('a', { element(e) { namespace = e.namespaceURI; }, }) .transform(response) .text(); strictEqual(namespace, 'http://www.w3.org/2000/svg'); }, }; export const handlerPerformingManySmallWrites = { async test() { const promiseDepth = 128 * 100; const numReads = 2; const input = new Response('
'); const output = new HTMLRewriter() .onDocument({ end(e) { for (let i = 0; i < promiseDepth; i++) { e.append('
', { html: true }); } }, }) .transform(input); // Begin reading but do not fully consume const reader = output.body.getReader(); for (let i = 0; i < numReads; i++) { await reader.read(); } }, }; export const hugeNumberOfHandlersForAnElement = { // Many small handlers async test() { const promiseDepth = 128 * 100; const numReads = promiseDepth; const input = new Response('
'.repeat(promiseDepth)); const output = new HTMLRewriter() .on('div', { element(e) { e.append('
', { html: true }); }, }) .transform(input); // Begin reading but do not fully consume const reader = output.body.getReader(); for (let i = 0; i < numReads; i++) { await reader.read(); } }, }; // Test HTMLRewriter with JS-backed ReadableStream // This test was moved from streams-respond-test.js because it triggers // a flaky ASAN failure related to V8's cppgc memory validation. export const htmlRewriterStream = { async test() { const enc = new TextEncoder(); const readable = new ReadableStream({ pull(controller) { controller.enqueue(enc.encode('Hello World!')); controller.close(); }, }); let response = new Response(readable, { status: 200, headers: { 'Content-Type': 'text/html; charset=utf-8', }, }); response = new HTMLRewriter().transform(response); strictEqual(await response.text(), 'Hello World!'); }, };