Skip to content
rachel/workerd
File

Blob: src/workerd/server/tests/python/sdk/worker.py

python597 lines
1# The tests in this file are primarily spread across Default.fetch() (in this module) and
2# Default.fetch() (in server.py).
3#
4# The code in `Default.test()` (in this module) is used to actually perform the testing and its
5# behaviour doesn't need to strictly be held consistent. In fact it uses the JS fetch, so it's not
6# going to follow the SDK at all.
7 
8from contextlib import asynccontextmanager
9from http import HTTPMethod, HTTPStatus
10 
11import js
12from workers import Blob, File, FormData, Request, Response, WorkerEntrypoint
13 
14import pyodide.http
15from pyodide.ffi import JsProxy, to_js
16 
17 
18@asynccontextmanager
19async def _mock_fetch(check):
20    async def mocked_fetch(original_fetch, url, opts):
21        check(url, opts)
22        return await original_fetch(url, opts)
23 
24    original_fetch = pyodide.http._jsfetch
25    pyodide.http._jsfetch = lambda url, opts: mocked_fetch(original_fetch, url, opts)
26    try:
27        yield
28    finally:
29        pyodide.http._jsfetch = original_fetch
30 
31 
32class Default(WorkerEntrypoint):
33    # Each path in this handler is its own test. The URLs that are being fetched
34    # here are defined in server.py.
35    async def fetch(self, request):
36        assert isinstance(request, Request)
37        if request.url.endswith("/modify"):
38            resp = await self.env.SERVER.fetch("https://example.com/sub")
39            return Response(
40                resp.body,
41                status=201,
42                headers={"Custom-Header-That-Should-Passthrough": "modified"},
43            )
44        elif request.url.endswith("/modify_tuple"):
45            resp = await self.env.SERVER.fetch("https://example.com/sub")
46            return Response(
47                resp.body,
48                headers=[
49                    ("Custom-Header-That-Should-Passthrough", "modified"),
50                    ("Custom-Header-That-Should-Passthrough", "another"),
51                ],
52            )
53        elif request.url.endswith("/fetch_opts"):
54            resp = await self.env.SERVER.fetch(
55                "https://example.com/sub_headers",
56                method=HTTPMethod.GET,
57                body=None,
58                headers={"Custom-Req-Header": 123},
59            )
60            return resp
61        elif request.url.endswith("/jsism"):
62            # Headers should be specified via a keyword argument, but it's done
63            # differently in JS. So we test what happens when someone does it
64            # that way accidentally to make sure we give them a reasonable error
65            # message.
66            try:
67                resp = await self.env.SERVER.fetch(
68                    "https://example.com/sub",
69                    {"headers": {"Custom-Req-Header": 123}},
70                )
71            except TypeError as exc:
72                if str(exc) == "fetch() takes 1 positional argument but 2 were given":
73                    return Response("success")
74 
75            return Response("fail")
76        elif request.url.endswith("/undefined_opts"):
77            # This tests two things:
78            #   * `Response.redirect` static method
79            #   * that other options can be passed into `fetch` (so that we can support
80            #       new options without updating this code)
81 
82            # Mock pyodide.http._jsfetch to ensure `foobarbaz` gets passed in.
83            def fetch_check(url, opts):
84                assert opts.foobarbaz == 42
85 
86            async with _mock_fetch(fetch_check):
87                resp = await self.env.SERVER.fetch(
88                    "https://example.com/redirect", redirect="manual", foobarbaz=42
89                )
90 
91            return resp
92        elif request.url.endswith("/response_inherited"):
93            expected = "test123"
94            resp = Response(expected)
95            text = await resp.text()
96            if text == expected:
97                return Response("success")
98            else:
99                return Response("invalid")
100        elif request.url.endswith("/redirect_invalid_input"):
101            try:
102                return Response.redirect("", HTTPStatus.BAD_GATEWAY)
103            except ValueError:
104                return Response("success")
105            return Response("invalid")
106        elif request.url.endswith("/response_json"):
107            return Response.json({"obj": {"field": 123}}, status=HTTPStatus.NOT_FOUND)
108        elif request.url.endswith("/formdata"):
109            # server.py creates a new FormData and verifies that it can be passed
110            # to Response.
111            resp = await self.env.SERVER.fetch("https://example.com/formdata")
112            data = await resp.formData()
113            if data["field"] == "value":
114                return Response("success")
115            else:
116                return Response("fail")
117        elif request.url.endswith("/formdatablob"):
118            # server.py creates a new FormData and verifies that it can be passed
119            # to Response.
120            resp = await self.env.SERVER.fetch("https://example.com/formdatablob")
121            data = await resp.formData()
122            assert data["field"] == "value"
123            assert (await data["blob.py"].text()) == "print(42)"
124            assert data["blob.py"].content_type == "text/python"
125            assert data["metadata"].name == "metadata.json"
126 
127            return Response("success")
128        elif request.url.endswith("/cf_opts"):
129            resp = await self.env.SERVER.fetch(
130                "http://example.com/redirect",
131                redirect="manual",
132                cf={
133                    "cacheTtl": 5,
134                    "cacheEverything": True,
135                    "cacheKey": "someCustomKey",
136                },
137            )
138            assert resp.status == 301
139            return Response("success")
140        elif request.url.endswith("/event_decorator"):
141            # Verify that the `@event`` decorator has transformed the `request` parameter to the correct
142            # type.
143            #
144            # Try to grab headers which should contain a duplicated header.
145            headers = request.headers.get_all("X-Custom-Header")
146            assert "some_value, some_other_value" in headers
147            return Response("success")
148        else:
149            resp = await self.env.SERVER.fetch("https://example.com/sub")
150            return resp
151 
152    async def scheduled(self, ctrl, env, ctx):
153        assert ctrl.scheduledTime == 1000
154        assert ctrl.cron == "* * * * 30"
155 
156    async def test(self):
157        js_env = self.env._env
158        await can_support_scheduled_cron_trigger(js_env)
159        await can_return_custom_fetch_response(js_env)
160        await can_modify_response(js_env)
161        await can_use_duplicate_headers(js_env)
162        await can_use_fetch_opts(js_env)
163        await gets_nice_error_on_jsism(js_env)
164        await can_use_undefined_options_and_redirect(js_env)
165        await can_use_inherited_response_methods(js_env)
166        await errors_on_invalid_input_to_redirect(js_env)
167        await can_use_response_json(js_env)
168        await can_request_form_data(js_env)
169        await form_data_unit_tests(js_env)
170        await blob_unit_tests(js_env)
171        await can_request_form_data_blob(js_env)
172        await replace_body_unit_tests(js_env)
173        await can_use_cf_fetch_opts(js_env)
174        await request_unit_tests(js_env)
175        await can_use_event_decorator(js_env)
176        await response_unit_tests(js_env)
177        await response_buffer_source_unit_tests(js_env)
178 
179 
180# TODO: Right now the `fetch` that's available on a binding is the JS fetch.
181# We may wish to rewrite it to be the same as the `fetch` defined in
182# `cloudflare.workers`. Doing so will require a feature flag.
183#
184# WARNING: Don't expect the below code itself to make use of the features in the SDK, it's mostly
185# calling the JS fetch via the FFI.
186 
187 
188async def can_return_custom_fetch_response(env):
189    assert isinstance(env, JsProxy), (
190        "Expecting the env for these tests not to be wrapped"
191    )
192    response = await env.SELF.fetch(
193        "http://example.com/",
194    )
195    text = await response.text()
196    assert text == "Hi there!"
197 
198 
199async def can_modify_response(env):
200    response = await env.SELF.fetch(
201        "http://example.com/modify",
202    )
203    text = await response.text()
204    assert text == "Hi there!"
205    assert response.status == 201
206    assert response.headers.get("Custom-Header-That-Should-Passthrough") == "modified"
207 
208 
209async def can_use_duplicate_headers(env):
210    response = await env.SELF.fetch(
211        "http://example.com/modify_tuple",
212    )
213    text = await response.text()
214    assert text == "Hi there!"
215    assert (
216        response.headers.get("Custom-Header-That-Should-Passthrough")
217        == "modified, another"
218    )
219 
220 
221async def can_use_fetch_opts(env):
222    response = await env.SELF.fetch(
223        "http://example.com/fetch_opts",
224    )
225    text = await response.text()
226    assert text == "Hi there!"
227    assert response.headers.get("Custom-Header-That-Should-Passthrough") == "true"
228 
229 
230async def gets_nice_error_on_jsism(env):
231    response = await env.SELF.fetch(
232        "http://example.com/jsism",
233    )
234    text = await response.text()
235    assert text == "success"
236 
237 
238async def can_use_undefined_options_and_redirect(env):
239    response = await env.SELF.fetch(
240        "http://example.com/undefined_opts", redirect="manual"
241    )
242    # The above path in this worker hits server's /redirect path,
243    # which returns a 301 to /sub. The code uses a fetch option which
244    # instructs it to not follow redirects, so we expect to get
245    # a 301 here.
246    assert response.status == 301
247 
248 
249async def can_use_inherited_response_methods(env):
250    response = await env.SELF.fetch(
251        "http://example.com/response_inherited",
252    )
253    text = await response.text()
254    assert text == "success"
255 
256 
257async def errors_on_invalid_input_to_redirect(env):
258    response = await env.SELF.fetch(
259        "http://example.com/redirect_invalid_input",
260    )
261    text = await response.text()
262    assert text == "success"
263 
264 
265async def can_use_response_json(env):
266    response = await env.SELF.fetch(
267        "http://example.com/response_json",
268    )
269    text = await response.text()
270    assert text == '{"obj": {"field": 123}}'
271 
272 
273async def can_request_form_data(env):
274    response = await env.SELF.fetch(
275        "http://example.com/formdata",
276    )
277    text = await response.text()
278    assert text == "success"
279 
280 
281async def form_data_unit_tests(env):
282    # Verify that existing JS formdata is loaded correctly.
283    js_data = js.FormData.new()
284    js_data.append("foobar", 123)
285    js_data.append("key", "lorem ipsum")
286    js_data.append("key", "dolor sit amet")
287    data = FormData(js_data)
288    assert data["foobar"] == "123"
289    assert data["key"] == "lorem ipsum"
290    assert data.get_all("key") == ["lorem ipsum", "dolor sit amet"]
291    assert "unknown key" not in data
292    assert "key" in data
293 
294    # Verify that dictionary can instantiate form data.
295    data = FormData({"key": "foobar"})
296    assert data["key"] == "foobar"
297    assert "key" in data
298 
299    # Test iterators
300    data.append("key", "another")
301    data.append("key2", "foobar2")
302    data_items = set(data.items())
303    assert ("key", "foobar") in data_items
304    assert ("key", "another") in data_items
305    assert ("key2", "foobar2") in data_items
306    data_keys = set(data.keys())
307    assert len(list(data.keys())) == 3
308    assert "key" in data_keys
309    assert "key2" in data_keys
310    data_values = set(data.values())
311    assert "foobar" in data_values
312    assert "another" in data_values
313    assert "foobar2" in data_values
314 
315 
316async def blob_unit_tests(env):
317    # Verify that we can create a FormData and add Blob in there.
318    data = FormData()
319    data["test"] = Blob(["some string"])
320    test_contents = await data["test"].text()
321    assert test_contents == "some string"
322    data["js"] = js.Blob.new(to_js(["another string"]))
323    # Even though we added a JS Blob, we should get a Python Blob.
324    assert isinstance(data["js"], Blob)
325    data.append("blah", js.Blob.new(to_js(["another string"])))
326    # Iterating through the items should give us Python Blobs.
327    for key, val in data.items():
328        assert isinstance(key, str)
329        assert isinstance(val, Blob)
330    # Verify that content type can be set.
331    content_type_blob = Blob(["test"], content_type="application/json")
332    assert content_type_blob.js_object.type == "application/json"
333    # Verify instance properties.
334    assert content_type_blob.size == 4
335    assert content_type_blob.content_type == "application/json"
336    # Verify instance methods.
337    assert (await content_type_blob.bytes()) == b"test"
338    assert (await content_type_blob.slice(1, 3).text()) == "es"
339    # Verify that Blob can be created with a memoryview.
340    memory_data = b"foobar"
341    memory_view_blob = Blob([memoryview(memory_data)])
342    assert await memory_view_blob.text() == "foobar"
343    # Verify that Blob constructor inherits type.
344    inherited = Blob(content_type_blob)
345    not_inherited = Blob(content_type_blob, content_type="other/type")
346    assert inherited.content_type == "application/json"
347    assert not_inherited.content_type == "other/type"
348    not_inherited2 = Blob([content_type_blob])
349    assert not_inherited2.content_type == ""
350    # Verify that we can create Files.
351    file_blob = File("my file", filename="test.txt", content_type="text/plain")
352    assert file_blob.content_type == "text/plain"
353    assert file_blob.name == "test.txt"
354 
355 
356async def can_request_form_data_blob(env):
357    response = await env.SELF.fetch(
358        "http://example.com/formdatablob",
359    )
360    text = await response.text()
361    assert text == "success"
362 
363 
364async def replace_body_unit_tests(env):
365    response = Response("test", status=201, status_text="Created")
366    cloned = response.replace_body("other")
367    assert cloned.status == 201
368    assert cloned.status_text == "Created"
369    t = await cloned.text()
370    assert t == "other"
371 
372 
373async def can_use_cf_fetch_opts(env):
374    response = await env.SELF.fetch(
375        "http://example.com/cf_opts",
376    )
377    text = await response.text()
378    assert text == "success"
379 
380 
381async def request_unit_tests(env):
382    req = Request("https://test.com", method=HTTPMethod.POST)
383    assert req.method == HTTPMethod.POST
384    assert repr(req) == "Request(method='POST', url='https://test.com/')"
385 
386    # Verify that we can pass JS headers to Request
387    js_headers = js.Headers.new()
388    js_headers.set("foo", "bar")
389    req_with_headers = Request("http://example.com", headers=js_headers)
390    assert req_with_headers.headers["foo"] == "bar"
391 
392    # Verify that we can pass a dictionary as headers to Request
393    req_with_headers = Request("http://example.com", headers={"aaaa": "test"})
394    assert req_with_headers.headers["aaaa"] == "test"
395 
396    # Verify that BodyUserError is thrown correctly.
397    req_used_twice = Request(
398        "http://example.com", body='{"field": 42}', method=HTTPMethod.POST
399    )
400    data = await req_used_twice.json()
401    assert data["field"] == 42
402    try:
403        req_used_twice.clone()
404        raise ValueError("Expected to throw")  # noqa: TRY301
405    except Exception as exc:
406        assert exc.__class__.__name__ == "OSError"  # TODO: BodyUsedError when available
407 
408    # Verify that duplicate header keys are returned correctly.
409    js_headers = js.Headers.new()
410    js_headers.append("Accept-encoding", "deflate")
411    js_headers.append("Accept-encoding", "gzip")
412    req_with_dup_headers = Request("http://example.com", headers=js_headers)
413    assert req_with_dup_headers.url == "http://example.com/"
414    encoding = req_with_dup_headers.headers.get_all("Accept-encoding")
415    assert encoding == ["deflate, gzip"]
416 
417    # Verify that header values containing commas are preserved.
418    js_headers = js.Headers.new()
419    js_headers.set("User-Agent", "Example, Agent/1.0")
420    req_with_user_agent = Request("http://example.com", headers=js_headers)
421    assert req_with_user_agent.headers.get("User-Agent") == "Example, Agent/1.0"
422    assert req_with_user_agent.headers.get_all("User-Agent") == ["Example, Agent/1.0"]
423 
424    # Verify that header values with commas are preserved when using Python dict.
425    req_dict_comma = Request(
426        "http://example.com",
427        headers={
428            "User-Agent": "Python, Client/2.0",
429            "Accept": "text/html, application/json",
430        },
431    )
432    assert req_dict_comma.headers.get("User-Agent") == "Python, Client/2.0"
433    assert "text/html" in req_dict_comma.headers.get("Accept")
434    assert "application/json" in req_dict_comma.headers.get("Accept")
435 
436    # Verify that Set-Cookie headers are preserved as distinct values.
437    js_headers = js.Headers.new()
438    js_headers.append("Set-Cookie", "a=b, c=d")
439    js_headers.append("Set-Cookie", "e=f")
440    req_with_set_cookie = Request("http://example.com", headers=js_headers)
441    assert req_with_set_cookie.headers.get_all("Set-Cookie") == ["a=b, c=d", "e=f"]
442 
443    # Verify that Set-Cookie headers work with Python list of tuples.
444    req_tuple_cookies = Request(
445        "http://example.com",
446        headers=[
447            ("Set-Cookie", "session=abc123"),
448            ("Set-Cookie", "token=xyz789"),
449            ("X-Custom", "value"),
450        ],
451    )
452    assert req_tuple_cookies.headers.get_all("Set-Cookie") == [
453        "session=abc123",
454        "token=xyz789",
455    ]
456    assert req_tuple_cookies.headers.get("X-Custom") == "value"
457 
458    # Verify that we can get a Blob.
459    req_for_blob = Request("http://example.com", body="foobar", method="POST")
460    blob = await req_for_blob.blob()
461    assert (await blob.text()) == "foobar"
462 
463    # Verify that we can get a FormData back.
464    js_form_data = js.FormData.new()
465    js_form_data.append("foobar", 123)
466    req_with_form_data = Request("http://example.com", body=js_form_data, method="POST")
467    form_data = await req_with_form_data.form_data()
468    assert form_data["foobar"] == "123"
469 
470 
471async def can_use_event_decorator(env):
472    js_headers = js.Headers.new()
473    js_headers.append("X-Custom-Header", "some_value")
474    js_headers.append("X-Custom-Header", "some_other_value")
475    response = await env.SELF.fetch(
476        "http://example.com/event_decorator", headers=js_headers
477    )
478    text = await response.text()
479    assert text == "success"
480 
481 
482async def response_unit_tests(env):
483    response_json = Response.json([1, 2, 3])
484    assert await response_json.text() == "[1, 2, 3]"
485    assert (
486        repr(response_json)
487        == "Response(status=200, status_text='OK', content_type='application/json')"
488    )
489 
490    response_json = Response.from_json([1, 2, 3])
491    assert await response_json.text() == "[1, 2, 3]"
492 
493    response_json = Response("[1, 2, 3]")
494    assert await response_json.json() == [1, 2, 3]
495 
496    response_json = Response.json("test")
497    assert await response_json.text() == '"test"'
498 
499    response_json = Response.json(["hi", "foo", 42])
500    assert await response_json.text() == '["hi", "foo", 42]'
501 
502    response_json = Response.json({"field": 42})
503    assert await response_json.text() == '{"field": 42}'
504 
505    response_json = Response.json("test")
506    assert response_json.headers.get("content-type") == "application/json"
507 
508    response_json = Response.json("test", headers={"x-other-header": "42"})
509    assert response_json.headers.get("content-type") == "application/json"
510 
511    response_json = Response.json("test", headers={"Content-Type": "42"})
512    assert response_json.headers.get("content-type") == "42"
513 
514    response_none = Response(None, status=204)
515    assert response_none.status == 204
516    assert response_none.body is None
517 
518    response_bytes = Response(b"test")
519    assert response_bytes.status == 200
520    assert await response_bytes.text() == "test"
521 
522    class Test:
523        def __init__(self, x):
524            self.x = x
525 
526    try:
527        response_json = Response.json(Test(42))
528        await response_json.text()
529        raise ValueError("Should have raised")  # noqa: TRY301
530    except Exception as err:
531        assert str(err) == "Object of type Test is not JSON serializable"
532 
533    try:
534        response_json = Response.json({1, 2, 3})
535        await response_json.text()
536        raise ValueError("Should have raised")  # noqa: TRY301
537    except Exception as err:
538        assert str(err) == "Object of type set is not JSON serializable"
539 
540    try:
541        response_json = Response(Test(42))
542        await response_json.text()
543        raise ValueError("Should have raised")  # noqa: TRY301
544    except Exception as err:
545        assert str(err) == "Unsupported type in Response: Test"
546 
547    response_ws = Response(
548        None, status=101, web_socket=js.WebSocket.new("ws://example.com/ignore")
549    )
550    # TODO: it doesn't seem possible to access webSocket even in JS
551    assert response_ws.status == 101
552 
553 
554async def response_buffer_source_unit_tests(env):
555    buffer_source_cases = [
556        # TODO: Float16Array is not supported in Pyodide <= 0.29 (pyodide/pyodide#6005)
557        ("ArrayBuffer", js.Uint8Array.new(to_js([1, 2, 3, 4, 5, 6, 7, 8])).buffer),
558        ("DataView", js.DataView.new(js.Uint8Array.new(to_js([9, 10, 11, 12])).buffer)),
559        ("Uint8Array", js.Uint8Array.new(to_js([1, 2, 3, 4]))),
560        ("Uint8ClampedArray", js.Uint8ClampedArray.new(to_js([1, 2, 3, 4]))),
561        ("Int8Array", js.Int8Array.new(to_js([1, -1, 2, -2]))),
562        ("Uint16Array", js.Uint16Array.new(to_js([1, 2, 3, 4]))),
563        ("Int16Array", js.Int16Array.new(to_js([1, -2, 3, -4]))),
564        ("Uint32Array", js.Uint32Array.new(to_js([1, 2, 3, 4]))),
565        ("Int32Array", js.Int32Array.new(to_js([1, -2, 3, -4]))),
566        ("Float32Array", js.Float32Array.new(to_js([1.5, -2.5, 3.25, -4.75]))),
567        ("Float64Array", js.Float64Array.new(to_js([1.5, -2.5]))),
568        # BigInt64 not supported in Pyodide <= 0.26
569        # ("BigInt64Array", js.BigInt64Array.new(to_js([2**53 + 1, -(2**53 + 1), 2**54 + 2, -(2**54 + 2)]))),
570        # ("BigUint64Array", js.BigUint64Array.new(to_js([2**53 + 1, 2**54 + 2, 2**55 + 3, 2**56 + 4]))),
571        # Test partial views to verify they work correctly when not viewing the whole backing buffer
572        (
573            "Uint8Array.subarray",
574            js.Uint8Array.new(to_js([0, 1, 2, 3, 4, 5])).subarray(1),
575        ),
576        ("Int8Array.subarray", js.Int8Array.new(to_js([0, 1, -1, 2, -2])).subarray(1)),
577    ]
578 
579    for type_name, body in buffer_source_cases:
580        expected_length = int(body.byteLength)
581        try:
582            response = Response(body)
583        except TypeError as exc:
584            raise AssertionError(
585                f"Response rejected BufferSource type {type_name}"
586            ) from exc
587 
588        buffer = await response.buffer()
589        assert int(buffer.byteLength) == expected_length, (
590            f"Response buffer length mismatch for {type_name}"
591        )
592 
593 
594async def can_support_scheduled_cron_trigger(env):
595    result = await env.SELF.scheduled(scheduledTime=1000, cron="* * * * 30")
596    assert result.outcome == "ok"