1
#pragma once
2
 
3
#include <workerd/api/commonjs.h>
4
#include <workerd/api/modules.h>
5
#include <workerd/io/io-context.h>
6
#include <workerd/io/worker.h>
7
#include <workerd/jsg/modules-new.h>
8
#include <workerd/util/strong-bool.h>
9
 
10
#include <pyodide/python-entrypoint.embed.h>
11
 
12
#include <capnp/blob.h>
13
#include <capnp/schema-loader.h>
14
#include <capnp/schema.h>
15
 
16
// This header provides utilities for setting up the ModuleRegistry for a worker.
17
// It is meant to be included in only two places; workerd-api.c++ and the equivalent
18
// file in the internal repo. It is templated on the TypeWrapper and JsgIsolate types.
19
 
20
namespace workerd {
21
namespace api {
22
class ServiceWorkerGlobalScope;
23
}  // namespace api
24
 
25
WD_STRONG_BOOL(IsPythonWorker);
26
 
27
namespace modules::capnp {
28
// Helper to iterate over the nested nodes of a schema for capnp modules, filtering
29
// out the kinds we don't care about.
30
void filterNestedNodes(const auto& schemaLoader, const auto& schema, auto fn) {
31
  for (auto nested: schema.getProto().getNestedNodes()) {
32
    auto child = schemaLoader.get(nested.getId());
33
    switch (child.getProto().which()) {
34
      case ::capnp::schema::Node::FILE:
35
      case ::capnp::schema::Node::STRUCT:
36
      case ::capnp::schema::Node::INTERFACE: {
37
        fn(nested.getName(), child);
38
        break;
39
      }
40
      case ::capnp::schema::Node::ENUM:
41
      case ::capnp::schema::Node::CONST:
42
      case ::capnp::schema::Node::ANNOTATION:
43
        // These kinds are not implemented and cannot contain further nested scopes, so
44
        // don't generate anything at all for now.
45
        break;
46
    }
47
  }
48
}
49
 
50
// This is used only by the original module registry implementation in both workerd
51
// and the internal project. It collects the exports and instantiates the exports of
52
// a capnp module at the same time and returns a ModuleInfo for the original registry.
53
// The new module registry variation uses a different approach where the exports are
54
// collected up front by the exports are instantiated lazily when the module is actually
55
// resolved.
56
template <typename JsgIsolate>
57
jsg::ModuleRegistry::ModuleInfo addCapnpModule(
58
    typename JsgIsolate::Lock& lock, uint64_t typeId, kj::StringPtr name) {
59
  const auto& schemaLoader = lock.template getCapnpSchemaLoader<api::ServiceWorkerGlobalScope>();
60
  auto schema = schemaLoader.get(typeId);
61
  auto fileScope = lock.v8Ref(lock.wrap(lock.v8Context(), schema).template As<v8::Value>());
62
  kj::Vector<kj::StringPtr> exports;
63
  kj::HashMap<kj::StringPtr, jsg::Value> topLevelDecls;
64
 
65
  filterNestedNodes(schemaLoader, schema, [&](auto name, const auto& child) {
66
    // topLevelDecls are the actual exported values...
67
    topLevelDecls.insert(
68
        name, lock.v8Ref(lock.wrap(lock.v8Context(), child).template As<v8::Value>()));
69
    // ... while exports is just the list of names
70
    exports.add(name);
71
  });
72
 
73
  return jsg::ModuleRegistry::ModuleInfo(lock, name, exports.asPtr().asConst(),
74
      jsg::ModuleRegistry::CapnpModuleInfo(kj::mv(fileScope), kj::mv(topLevelDecls)));
75
}
76
}  // namespace modules::capnp
77
 
78
// Creates an instance of the (new) ModuleRegistry. This method provides the
79
// initialization logic that is agnostic to the Worker::Api implementation,
80
// but accepts a callback parameter to handle the Worker::Api-specific details.
81
//
82
// Note: this is a big template but it will only be called from two places in
83
// the codebase, one for workerd and one for the internal project. It depends
84
// on the TypeWrapper specific to each project.
85
template <typename TypeWrapper>
86
static kj::Arc<jsg::modules::ModuleRegistry> newWorkerModuleRegistry(
87
    const jsg::ResolveObserver& resolveObserver,
88
    kj::Maybe<const Worker::Script::ModulesSource&> maybeSource,
89
    const CompatibilityFlags::Reader& featureFlags,
90
    const jsg::Url& bundleBase,
91
    auto setupForApi,
92
    jsg::modules::ModuleRegistry::Builder::Options options =
93
        jsg::modules::ModuleRegistry::Builder::Options::NONE) {
94
  jsg::modules::ModuleRegistry::Builder builder(resolveObserver, bundleBase, options);
95
 
96
  // This callback is used when a module is being loaded to arrange evaluating the
97
  // module outside of the current IoContext.
98
  builder.setEvalCallback(
99
      [](jsg::Lock& js, const auto& module, auto v8Module, const auto& observer) -> jsg::JsPromise {
100
    // Creating the SuppressIoContextScope here ensures that the current IoContext,
101
    // if any, is moved out of the way while we are evaluating.
102
    SuppressIoContextScope suppressIoContextScope;
103
    KJ_DASSERT(!IoContext::hasCurrent(), "Module evaluation must not be in an IoContext");
104
    JSG_TRY(js) {
105
      auto ret = jsg::check(v8Module->Evaluate(js.v8Context()));
106
      KJ_ASSERT(ret->IsPromise());
107
      return jsg::JsPromise(ret.template As<v8::Promise>());
108
    }
109
    JSG_CATCH(exception) {
110
      return js.rejectedJsPromise(jsg::JsValue(exception.getHandle(js)));
111
    }
112
  });
113
 
114
  // Add the module bundles that are built into the runtime.
115
  api::registerBuiltinModules<TypeWrapper>(builder, featureFlags);
116
 
117
  bool hasPythonModules = false;
118
 
119
  // Add the module bundles that are configured by the worker (if any)
120
  // The only case where maybeSource is none is when the worker is using
121
  // the old service worker script format or "inherit", in which case
122
  // we will initialize a module registry with the built-ins, extensions,
123
  // etc but no worker bundle modules will be added.
124
  KJ_IF_SOME(source, maybeSource) {
125
    // Register any capnp schemas contained in the source bundle
126
    auto& schemaLoader = builder.getSchemaLoader();
127
    for (auto schema: source.capnpSchemas) {
128
      schemaLoader.load(schema);
129
    }
130
 
131
    jsg::modules::ModuleBundle::BundleBuilder bundleBuilder(bundleBase);
132
    bool firstEsm = true;
133
    using namespace workerd::api::pyodide;
134
 
135
    for (auto& def: source.modules) {
136
      KJ_SWITCH_ONEOF(def.content) {
137
        KJ_CASE_ONEOF(content, Worker::Script::EsModule) {
138
          jsg::modules::Module::Flags flags = jsg::modules::Module::Flags::ESM;
139
          // Only the first ESM module we encounter is the main module.
140
          // This should also be the first module in the list but we're
141
          // not enforcing that here.
142
          if (firstEsm) {
143
            flags = flags | jsg::modules::Module::Flags::MAIN;
144
            firstEsm = false;
145
          }
146
          if (content.ownBody != kj::none) {
147
            // When the source is owned (e.g. transpiled TypeScript), we must
148
            // copy it into the module registry since the owning rust::String
149
            // may not outlive the registry.
150
            bundleBuilder.addEsmModule(def.name, kj::heapArray<const char>(content.body), flags);
151
          } else {
152
            // The content.body points into process-lifetime capnp message
153
            // buffers. We can safely pass a non-owning reference.
154
            bundleBuilder.addEsmModule(def.name, content.body, flags);
155
          }
156
          break;
157
        }
158
        KJ_CASE_ONEOF(content, Worker::Script::TextModule) {
159
          // The content.body is memory-resident and is expected to outlive the
160
          // module registry. We can safely pass a reference to the module handler.
161
          // It will not be copied into a JS string until the module is actually
162
          // evaluated.
163
          bundleBuilder.addSyntheticModule(def.name,
164
              jsg::modules::Module::newTextModuleHandler(content.body), nullptr,
165
              jsg::modules::Module::ContentType::TEXT);
166
          break;
167
        }
168
        KJ_CASE_ONEOF(content, Worker::Script::DataModule) {
169
          // The content.body is memory-resident and is expected to outlive the
170
          // module registry. We can safely pass a reference to the module handler.
171
          // It will not be copied into a JS string until the module is actually
172
          // evaluated.
173
          bundleBuilder.addSyntheticModule(def.name,
174
              jsg::modules::Module::newDataModuleHandler(content.body), nullptr,
175
              jsg::modules::Module::ContentType::DATA);
176
          break;
177
        }
178
        KJ_CASE_ONEOF(content, Worker::Script::WasmModule) {
179
          // The content.body is memory-resident and is expected to outlive the
180
          // module registry. We can safely pass a reference to the module handler.
181
          // It will not be copied into a JS string until the module is actually
182
          // evaluated.
183
          bundleBuilder.addWasmModule(def.name, content.body);
184
          break;
185
        }
186
        KJ_CASE_ONEOF(content, Worker::Script::JsonModule) {
187
          // The content.body is memory-resident and is expected to outlive the
188
          // module registry. We can safely pass a reference to the module handler.
189
          // It will not be copied into a JS string until the module is actually
190
          // evaluated.
191
          bundleBuilder.addSyntheticModule(def.name,
192
              jsg::modules::Module::newJsonModuleHandler(content.body), nullptr,
193
              jsg::modules::Module::ContentType::JSON);
194
          break;
195
        }
196
        KJ_CASE_ONEOF(content, Worker::Script::CommonJsModule) {
197
          kj::ArrayPtr<const kj::StringPtr> named;
198
          KJ_IF_SOME(n, content.namedExports) {
199
            named = n;
200
          }
201
          bundleBuilder.addSyntheticModule(def.name,
202
              jsg::modules::Module::newCjsStyleModuleHandler<api::CommonJsModuleContext,
203
                  TypeWrapper>(content.body, def.name),
204
              KJ_MAP(name, named) { return kj::str(name); });
205
          break;
206
        }
207
        KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
208
          KJ_FAIL_ASSERT("Python modules are not currently supported with the new module registry");
209
          // KJ_REQUIRE(featureFlags.getPythonWorkers(),
210
          //     "The python_workers compatibility flag is required to use Python.");
211
          // firstEsm = false;
212
          // hasPythonModules = true;
213
          // kj::StringPtr entry = PYTHON_ENTRYPOINT;
214
          // bundleBuilder.addEsmModule(def.name, entry);
215
          // break;
216
        }
217
        KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
218
          // Handled separately
219
          break;
220
        }
221
        KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
222
          // For the new module registry, the implementation is a bit different than
223
          // the original. Up front we collect only the names of the exports since we
224
          // need to know those when we create the synthetic module. The actual exports
225
          // themselves, however, are instantiated lazily when the module is actually
226
          // resolved and evaluated.
227
          auto& schemaLoader = builder.getSchemaLoader();
228
          auto schema = schemaLoader.get(content.typeId);
229
          kj::Vector<kj::String> exports;
230
          modules::capnp::filterNestedNodes(schemaLoader, schema,
231
              [&](auto name, const capnp::Schema& child) { exports.add(kj::str(name)); });
232
 
233
          bundleBuilder.addSyntheticModule(def.name,
234
              [typeId = content.typeId, &schemaLoader](jsg::Lock& js, const jsg::Url&,
235
                  const jsg::modules::Module::ModuleNamespace& ns,
236
                  const jsg::CompilationObserver& observer) {
237
            auto& typeWrapper = TypeWrapper::from(js.v8Isolate);
238
            KJ_IF_SOME(schema, schemaLoader.tryGet(typeId)) {
239
              return js.tryCatch([&] {
240
                // Set the default export...
241
                ns.setDefault(js,
242
                    jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, schema)
243
                                     .template As<v8::Value>()));
244
                // Set each of the named exports...
245
                // The names must match what we collected when the bundle was built.
246
                modules::capnp::filterNestedNodes(
247
                    schemaLoader, schema, [&](auto name, const auto& child) {
248
                  ns.set(js, name,
249
                      jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, child)));
250
                });
251
                return true;
252
              }, [&](jsg::Value exception) {
253
                js.v8Isolate->ThrowException(exception.getHandle(js));
254
                return false;
255
              });
256
            } else {
257
              // The schema should have been loaded when the Worker::Script was created.
258
              // This likely indicates an internal error of some kind.
259
              js.v8Isolate->ThrowException(
260
                  js.typeError("Invalid or unknown capnp module type identifier"));
261
              return false;
262
            }
263
          },
264
              exports.releaseAsArray());
265
        }
266
      }
267
    }
268
 
269
    builder.add(bundleBuilder.finish());
270
  }
271
 
272
  // Now perform any Worker::Api-specific setup.
273
  setupForApi(builder, hasPythonModules ? IsPythonWorker::YES : IsPythonWorker::NO);
274
 
275
  // All done!
276
  return builder.finish();
277
}
278
 
279
// ======================================================================================
280
// Legacy module registry support
281
 
282
namespace modules::legacy {
283
 
284
template <typename JsgIsolate>
285
static v8::Local<v8::String> compileTextGlobal(
286
    typename JsgIsolate::Lock& lock, ::capnp::Text::Reader reader) {
287
  return lock.wrapNoContext(reader);
288
};
289
 
290
template <typename JsgIsolate>
291
static v8::Local<v8::ArrayBuffer> compileDataGlobal(
292
    typename JsgIsolate::Lock& lock, ::capnp::Data::Reader reader) {
293
  return lock.wrapNoContext(kj::heapArray(reader));
294
};
295
 
296
template <typename JsgIsolate>
297
static v8::Local<v8::WasmModuleObject> compileWasmGlobal(typename JsgIsolate::Lock& lock,
298
    ::capnp::Data::Reader reader,
299
    const jsg::CompilationObserver& observer) {
300
  lock.setAllowEval(true);
301
  KJ_DEFER(lock.setAllowEval(false));
302
 
303
  // Allow Wasm compilation to spawn a background thread for tier-up, i.e. recompiling
304
  // Wasm with optimizations in the background. Otherwise Wasm startup is way too slow.
305
  // Until tier-up finishes, requests will be handled using Liftoff-generated code, which
306
  // compiles fast but runs slower.
307
  AllowV8BackgroundThreadsScope scope;
308
 
309
  return jsg::compileWasmModule(lock, reader, observer);
310
};
311
 
312
template <typename JsgIsolate>
313
static v8::Local<v8::Value> compileJsonGlobal(
314
    typename JsgIsolate::Lock& lock, ::capnp::Text::Reader reader) {
315
  return jsg::check(v8::JSON::Parse(lock.v8Context(), lock.wrapNoContext(reader)));
316
};
317
 
318
// Compiles a module for the legacy module registry, returning kj::none if the module
319
// is a Python module or Python requirement, which are handled elsewhere.
320
template <typename JsgIsolate>
321
kj::Maybe<jsg::ModuleRegistry::ModuleInfo> tryCompileLegacyModule(jsg::Lock& js,
322
    kj::StringPtr name,
323
    const Worker::Script::ModuleContent& moduleContent,
324
    const jsg::CompilationObserver& observer,
325
    CompatibilityFlags::Reader featureFlags) {
326
  auto& lock = kj::downcast<typename JsgIsolate::Lock>(js);
327
  KJ_SWITCH_ONEOF(moduleContent) {
328
    KJ_CASE_ONEOF(content, Worker::Script::TextModule) {
329
      return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
330
          jsg::ModuleRegistry::TextModuleInfo(
331
              js, modules::legacy::compileTextGlobal<JsgIsolate>(lock, content.body)));
332
    }
333
    KJ_CASE_ONEOF(content, Worker::Script::DataModule) {
334
      return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
335
          jsg::ModuleRegistry::DataModuleInfo(
336
              js, modules::legacy::compileDataGlobal<JsgIsolate>(lock, content.body)));
337
    }
338
    KJ_CASE_ONEOF(content, Worker::Script::WasmModule) {
339
      auto wasmModule =
340
          modules::legacy::compileWasmGlobal<JsgIsolate>(lock, content.body, observer);
341
      auto moduleInfo = jsg::ModuleRegistry::ModuleInfo(
342
          js, name, kj::none, jsg::ModuleRegistry::WasmModuleInfo(js, wasmModule));
343
      moduleInfo.setModuleSourceObject(lock, wasmModule.template As<v8::Object>());
344
      return moduleInfo;
345
    }
346
    KJ_CASE_ONEOF(content, Worker::Script::JsonModule) {
347
      return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
348
          jsg::ModuleRegistry::JsonModuleInfo(
349
              js, modules::legacy::compileJsonGlobal<JsgIsolate>(lock, content.body)));
350
    }
351
    KJ_CASE_ONEOF(content, Worker::Script::EsModule) {
352
      // TODO(soon): Make sure passing nullptr to compile cache is desired.
353
      return jsg::ModuleRegistry::ModuleInfo(js, name, content.body, nullptr /* compile cache */,
354
          jsg::ModuleInfoCompileOption::BUNDLE, observer);
355
    }
356
    KJ_CASE_ONEOF(content, Worker::Script::CommonJsModule) {
357
      return jsg::ModuleRegistry::ModuleInfo(js, name, content.namedExports,
358
          jsg::ModuleRegistry::CommonJsModuleInfo(lock, name, content.body,
359
              kj::heap<api::CommonJsImpl<typename JsgIsolate::Lock>>(js, kj::Path::parse(name))));
360
    }
361
    KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
362
      // Nothing to do. Handled elsewhere.
363
      return kj::none;
364
    }
365
    KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
366
      // Nothing to do. Handled elsewhere.
367
      return kj::none;
368
    }
369
    KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
370
      return workerd::modules::capnp::addCapnpModule<JsgIsolate>(lock, content.typeId, name);
371
    }
372
  }
373
  KJ_UNREACHABLE;
374
}
375
 
376
template <typename JsgIsolate>
377
kj::Array<Worker::Script::CompiledGlobal> compileServiceWorkerGlobals(jsg::Lock& js,
378
    const Worker::Script::ScriptSource& source,
379
    const Worker::Isolate& isolate,
380
    const jsg::CompilationObserver& observer) {
381
  auto& lock = kj::downcast<typename JsgIsolate::Lock>(js);
382
 
383
  auto globals = source.globals.asPtr();
384
  auto compiledGlobals = kj::heapArrayBuilder<Worker::Script::CompiledGlobal>(globals.size());
385
 
386
  for (auto& global: globals) {
387
    js.withinHandleScope([&] {
388
      // Don't use String's usual TypeHandler here because we want to intern the string.
389
      auto name = jsg::v8StrIntern(js.v8Isolate, global.name);
390
 
391
      v8::Local<v8::Value> value;
392
 
393
      KJ_SWITCH_ONEOF(global.content) {
394
        KJ_CASE_ONEOF(content, Worker::Script::TextModule) {
395
          value =
396
              workerd::modules::legacy::template compileTextGlobal<JsgIsolate>(lock, content.body);
397
        }
398
        KJ_CASE_ONEOF(content, Worker::Script::DataModule) {
399
          value =
400
              workerd::modules::legacy::template compileDataGlobal<JsgIsolate>(lock, content.body);
401
        }
402
        KJ_CASE_ONEOF(content, Worker::Script::WasmModule) {
403
          value = workerd::modules::legacy::template compileWasmGlobal<JsgIsolate>(
404
              lock, content.body, observer);
405
        }
406
        KJ_CASE_ONEOF(content, Worker::Script::JsonModule) {
407
          value =
408
              workerd::modules::legacy::template compileJsonGlobal<JsgIsolate>(lock, content.body);
409
        }
410
        KJ_CASE_ONEOF(content, Worker::Script::EsModule) {
411
          KJ_FAIL_REQUIRE("modules not supported with mainScript");
412
        }
413
        KJ_CASE_ONEOF(content, Worker::Script::CommonJsModule) {
414
          KJ_FAIL_REQUIRE("modules not supported with mainScript");
415
        }
416
        KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
417
          KJ_FAIL_REQUIRE("modules not supported with mainScript");
418
        }
419
        KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
420
          KJ_FAIL_REQUIRE("modules not supported with mainScript");
421
        }
422
        KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
423
          KJ_FAIL_REQUIRE("modules not supported with mainScript");
424
        }
425
      }
426
 
427
      compiledGlobals.add(Worker::Script::CompiledGlobal{
428
        {lock.v8Isolate, name},
429
        {lock.v8Isolate, value},
430
      });
431
    });
432
  }
433
 
434
  return compiledGlobals.finish();
435
}
436
 
437
}  // namespace modules::legacy
438
 
439
// ===========================================================================================
440
// Python module support
441
 
442
namespace modules::python {
443
kj::Own<api::pyodide::PyodideMetadataReader::State> createPyodideMetadataState(
444
    const Worker::Script::ModulesSource& source,
445
    api::pyodide::IsWorkerd isWorkerd,
446
    api::pyodide::IsTracing isTracing,
447
    api::pyodide::SnapshotToDisk snapshotToDisk,
448
    api::pyodide::CreateBaselineSnapshot createBaselineSnapshot,
449
    PythonSnapshotRelease::Reader pythonRelease,
450
    kj::Maybe<kj::Array<kj::byte>> maybeSnapshot,
451
    CompatibilityFlags::Reader featureFlags);
452
 
453
jsg::Bundle::Reader retrievePyodideBundle(
454
    const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version);
455
 
456
// Registers all the modules that are common to both workerd and edgeworker.
457
// Specialised modules like the Jaeger tracing module are registered in edgeworker only, if they
458
// are not specified in the arguments to this function then they get injected as "disabled"
459
// variants.
460
//
461
// This function is used by both workerd and edgeworker.
462
template <typename TracerApi, class Registry>
463
void registerPythonCommonModules(jsg::Lock& lock,
464
    Registry& modules,
465
    CompatibilityFlags::Reader featureFlags,
466
    jsg::Bundle::Reader pyodideBundle,
467
    const workerd::WorkerSource::ModulesSource& source,
468
    kj::Maybe<kj::Array<kj::byte>> maybeSnapshot,
469
    api::pyodide::IsWorkerd isWorkerd,
470
    api::pyodide::IsTracing isTracing,
471
    api::pyodide::SnapshotToDisk snapshotToDisk,
472
    api::pyodide::CreateBaselineSnapshot createBaselineSnapshot,
473
    kj::Maybe<kj::Own<api::pyodide::ArtifactBundler_State>> artifacts,
474
    kj::Maybe<jsg::Ref<api::pyodide::DiskCache>> diskCache,
475
    kj::Maybe<jsg::Ref<TracerApi>> internalJaeger,
476
    kj::Maybe<jsg::ModuleRegistry::ModuleCallback> maybeLimiter) {
477
  KJ_REQUIRE(featureFlags.getPythonWorkers(),
478
      "The python_workers compatibility flag is required to use Python.");
479
 
480
  // We add `pyodide:` packages here including python-entrypoint-helper.js.
481
  modules.addBuiltinBundle(PYODIDE_BUNDLE, kj::none);
482
 
483
  using namespace workerd::api::pyodide;
484
  auto pythonRelease = KJ_REQUIRE_NONNULL(getPythonSnapshotRelease(featureFlags));
485
 
486
  // Inject pyodide bundle.
487
  modules.addBuiltinBundle(pyodideBundle);
488
 
489
  modules.addBuiltinModule("pyodide-internal:runtime-generated/metadata",
490
      lock.alloc<PyodideMetadataReader>(workerd::modules::python::createPyodideMetadataState(source,
491
          isWorkerd, isTracing, snapshotToDisk, createBaselineSnapshot, pythonRelease,
492
          kj::mv(maybeSnapshot), featureFlags)),
493
      jsg::ModuleRegistry::Type::INTERNAL);
494
 
495
  // Inject packages tar file
496
  modules.addBuiltinModule("pyodide-internal:packages_tar_reader", "export default { }"_kj,
497
      workerd::jsg::ModuleRegistry::Type::INTERNAL, {});
498
 
499
  // Inject artifact bundler.
500
  modules.addBuiltinModule("pyodide-internal:artifacts",
501
      lock.alloc<api::pyodide::ArtifactBundler>(kj::mv(artifacts).orDefault(
502
          []() { return api::pyodide::ArtifactBundler::makeDisabledBundler(); })),
503
      jsg::ModuleRegistry::Type::INTERNAL);
504
 
505
  // Inject disk cache module
506
  modules.addBuiltinModule("pyodide-internal:disk_cache",
507
      kj::mv(diskCache).orDefault([&lock]() { return lock.alloc<DiskCache>(); }),
508
      jsg::ModuleRegistry::Type::INTERNAL);
509
 
510
  // Inject the internal jaeger tracer (only implemented in Edgeworker)
511
  KJ_IF_SOME(tracer, internalJaeger) {
512
    modules.addBuiltinModule(
513
        "pyodide-internal:internalJaeger", kj::mv(tracer), jsg::ModuleRegistry::Type::INTERNAL);
514
  } else {
515
    modules.addBuiltinModule("pyodide-internal:internalJaeger",
516
        DisabledInternalJaeger::create(lock), jsg::ModuleRegistry::Type::INTERNAL);
517
  }
518
 
519
  // Inject a WorkerFatalReporter for reporting fatal errors to Runtime Analytics.
520
  modules.addBuiltinModule("pyodide-internal:fatal-reporter",
521
      lock.alloc<api::pyodide::WorkerFatalReporter>(), jsg::ModuleRegistry::Type::INTERNAL);
522
 
523
  // Inject a SimplePythonLimiter
524
  KJ_IF_SOME(limiter, maybeLimiter) {
525
    modules.addBuiltinModule(
526
        "pyodide-internal:limiter", kj::mv(limiter), jsg::ModuleRegistry::Type::INTERNAL);
527
  } else {
528
    modules.addBuiltinModule("pyodide-internal:limiter", SimplePythonLimiter::makeDisabled(lock),
529
        jsg::ModuleRegistry::Type::INTERNAL);
530
  }
531
}
532
 
533
// This function is used to register Python Worker modules in workerd. It uses
534
// registerPythonCommonModules and implements other workerd-specific functionality like the disk
535
// cache.
536
template <typename JsgIsolate, class Registry>
537
void registerPythonWorkerdModules(jsg::Lock& lockParam,
538
    Registry& modules,
539
    CompatibilityFlags::Reader featureFlags,
540
    kj::Maybe<kj::Own<api::pyodide::ArtifactBundler_State>> artifacts,
541
    const api::pyodide::PythonConfig& pythonConfig,
542
    const workerd::WorkerSource::ModulesSource& source) {
543
  KJ_REQUIRE(featureFlags.getPythonWorkers(),
544
      "The python_workers compatibility flag is required to use Python.");
545
 
546
  auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags));
547
  auto version = getPythonBundleName(pythonRelease);
548
  auto bundle = retrievePyodideBundle(pythonConfig, version);
549
 
550
  // Inject pyodide bootstrap module (TODO: load this from the capnproto bundle?)
551
  {
552
    Worker::Script::Module module{
553
      .name = source.mainModule, .content = Worker::Script::EsModule{PYTHON_ENTRYPOINT}};
554
 
555
    auto info = modules::legacy::tryCompileLegacyModule<JsgIsolate>(
556
        lockParam, module.name, module.content, modules.getObserver(), featureFlags);
557
 
558
    auto path = kj::Path::parse(source.mainModule);
559
    modules.add(path, kj::mv(KJ_REQUIRE_NONNULL(info)));
560
  }
561
 
562
  // Determine whether we are creating a baseline snapshot and/or snapshotting to/from disk. This
563
  // functionality is only supported in workerd.
564
  api::pyodide::CreateBaselineSnapshot createBaselineSnapshot(pythonConfig.createBaselineSnapshot);
565
  api::pyodide::SnapshotToDisk snapshotToDisk(
566
      pythonConfig.createSnapshot || createBaselineSnapshot);
567
  kj::Maybe<kj::Array<kj::byte>> snapshot = kj::none;
568
  KJ_IF_SOME(snapshotName, pythonConfig.loadSnapshotFromDisk) {
569
    auto& root = KJ_REQUIRE_NONNULL(pythonConfig.snapshotDirectory);
570
    kj::Path path(snapshotName);
571
    auto maybeFile = root->tryOpenFile(path);
572
    if (maybeFile == kj::none) {
573
      KJ_FAIL_REQUIRE("Expected to find", snapshotName, "in the package cache directory");
574
    }
575
    snapshot = KJ_REQUIRE_NONNULL(maybeFile)->readAllBytes();
576
  }
577
 
578
  // Create disk cache module
579
  auto diskCache = lockParam.alloc<api::pyodide::DiskCache>(
580
      pythonConfig.packageDiskCacheRoot, pythonConfig.snapshotDirectory);
581
 
582
  modules::python::registerPythonCommonModules<api::pyodide::DisabledInternalJaeger>(lockParam,
583
      modules, featureFlags, bundle, source, kj::mv(snapshot), api::pyodide::IsWorkerd::YES,
584
      api::pyodide::IsTracing::NO, snapshotToDisk, createBaselineSnapshot, kj::mv(artifacts),
585
      kj::mv(diskCache), kj::none /* internalJaeger */, kj::none /* limiter */);
586
}
587
}  // namespace modules::python
588
 
589
}  // namespace workerd