1
// Copyright (c) 2026 Cloudflare, Inc.
2
// Licensed under the Apache 2.0 license found in the LICENSE file or at:
3
//     https://opensource.org/licenses/Apache-2.0
4
 
5
/**
6
 * This file contains code that roughly replaces pyodide.loadPackage, with workerd-specific
7
 * optimizations:
8
 * - Wheels are decompressed with a DecompressionStream instead of in Python
9
 * - Wheels are overlaid onto the site-packages dir instead of actually being copied
10
 * - Wheels are fetched from a disk cache if available.
11
 *
12
 * Note that loadPackages is only used in local dev for now, internally we use the full big bundle
13
 * that contains all the packages ready to go.
14
 */
15
 
16
import {
17
  LOCKFILE,
18
  PACKAGES_VERSION,
19
  USING_OLDEST_PACKAGES_VERSION,
20
} from 'pyodide-internal:metadata';
21
import {
22
  VIRTUALIZED_DIR,
23
  STDLIB_PACKAGES,
24
} from 'pyodide-internal:setupPackages';
25
import { parseTarInfo } from 'pyodide-internal:tar';
26
import { createTarFS } from 'pyodide-internal:tarfs';
27
import { default as ArtifactBundler } from 'pyodide-internal:artifacts';
28
import {
29
  PythonUserError,
30
  PythonWorkersInternalError,
31
} from 'pyodide-internal:util';
32
 
33
function getPackageMetadata(requirement: string): PackageDeclaration {
34
  const obj = LOCKFILE['packages'][requirement];
35
  if (!obj) {
36
    throw new PythonUserError(
37
      'Requirement ' + requirement + ' not found in lockfile'
38
    );
39
  }
40
 
41
  return obj;
42
}
43
 
44
function loadBundleFromArtifactBundler(requirement: string): Reader {
45
  const filename = getPackageMetadata(requirement).file_name;
46
  const fullPath = `python-package-bucket/${PACKAGES_VERSION}/${filename}`;
47
  const reader = ArtifactBundler.getPackage(fullPath);
48
  if (!reader) {
49
    throw new PythonWorkersInternalError(
50
      'Failed to get package ' + fullPath + ' from ArtifactBundler'
51
    );
52
  }
53
  return reader;
54
}
55
 
56
/**
57
 * Downloads the requirements specified and loads them into Pyodide. Note that this does not
58
 * do any dependency resolution, it just installs the requirements that are specified. See
59
 * `getTransitiveRequirements` for the code that deals with this.
60
 */
61
export function loadPackages(Module: Module, requirements: Set<string>): void {
62
  let pkgsToLoad = requirements;
63
  // TODO: Package snapshot created with '20240829.4' needs the stdlib packages to be added here.
64
  // We should remove this check once the next Python and packages versions are rolled
65
  // out.
66
  if (USING_OLDEST_PACKAGES_VERSION) {
67
    pkgsToLoad = pkgsToLoad.union(new Set(STDLIB_PACKAGES));
68
  }
69
 
70
  for (const req of pkgsToLoad) {
71
    if (req === 'test') {
72
      continue; // Skip the test package, it is only useful for internal Python regression testing.
73
    }
74
    if (VIRTUALIZED_DIR.hasRequirementLoaded(req)) {
75
      continue;
76
    }
77
 
78
    const reader = loadBundleFromArtifactBundler(req);
79
    const [tarInfo, soFiles] = parseTarInfo(reader);
80
    const pkg = getPackageMetadata(req);
81
    VIRTUALIZED_DIR.addSmallBundle(tarInfo, soFiles, req, pkg.install_dir);
82
  }
83
 
84
  const tarFS = createTarFS(Module);
85
  VIRTUALIZED_DIR.mount(Module, tarFS);
86
}