1
// Copyright (c) 2026 Cloudflare, Inc.
2
// Licensed under the Apache 2.0 license found in the LICENSE file or at:
3
//     https://opensource.org/licenses/Apache-2.0
4
 
5
import { parseTarInfo } from 'pyodide-internal:tar';
6
import { createMetadataFS } from 'pyodide-internal:metadatafs';
7
import { LOCKFILE } from 'pyodide-internal:metadata';
8
import {
9
  invalidateCaches,
10
  PythonWorkersInternalError,
11
  PythonUserError,
12
  simpleRunPython,
13
} from 'pyodide-internal:util';
14
import { default as EmbeddedPackagesTarReader } from 'pyodide-internal:packages_tar_reader';
15
 
16
const canonicalizeNameRegex = /[-_.]+/g;
17
const DYNLIB_PATH = '/usr/lib';
18
 
19
/**
20
 * Canonicalize a package name. Port of Python's packaging.utils.canonicalize_name.
21
 * @param name The package name to canonicalize.
22
 * @returns The canonicalize package name.
23
 * @private
24
 */
25
function canonicalizePackageName(name: string): string {
26
  return name.replace(canonicalizeNameRegex, '-').toLowerCase();
27
}
28
 
29
// The "name" field in the lockfile is not canonicalized
30
export const STDLIB_PACKAGES: string[] = Object.values(LOCKFILE.packages)
31
  .filter(({ package_type }) => package_type === 'cpython_module')
32
  .map(({ name }) => canonicalizePackageName(name));
33
 
34
// Each item in the list is an element of the file path, for example
35
// `folder/file.txt` -> `["folder", "file.txt"]
36
export type FilePath = string[];
37
 
38
function createTarFsInfo(): TarFSInfo {
39
  return {
40
    children: new Map(),
41
    mode: 0o777,
42
    type: '5',
43
    modtime: 0,
44
    size: 0,
45
    path: '',
46
    name: '',
47
    parts: [],
48
    reader: null,
49
  };
50
}
51
 
52
/**
53
 * VirtualizedDir keeps track of the virtualized view of the site-packages
54
 * directory generated for each worker as well as a virtualized view of the dynamic libraries stored
55
 * in /usr/lib.
56
 */
57
class VirtualizedDir {
58
  // TODO(soon): Can we use the # syntax here?
59
  // eslint-disable-next-line no-restricted-syntax
60
  private rootInfo: TarFSInfo; // site-packages directory
61
  // TODO(soon): Can we use the # syntax here?
62
  // eslint-disable-next-line no-restricted-syntax
63
  private dynlibTarFs: TarFSInfo; // /usr/lib directory
64
  // TODO(soon): Can we use the # syntax here?
65
  // eslint-disable-next-line no-restricted-syntax
66
  private soFiles: FilePath[];
67
  // TODO(soon): Can we use the # syntax here?
68
  // eslint-disable-next-line no-restricted-syntax
69
  private loadedRequirements: Set<string>;
70
  constructor() {
71
    this.rootInfo = createTarFsInfo();
72
    this.dynlibTarFs = createTarFsInfo();
73
    this.soFiles = [];
74
    this.loadedRequirements = new Set();
75
  }
76
 
77
  /**
78
   * mountOverlay "overlays" a directory onto the site-packages root directory.
79
   * All files and subdirectories in the overlay will be accessible at site-packages by the worker.
80
   * If a file or directory already exists, an error is thrown.
81
   * @param {TarInfo} overlayInfo The directory that is to be "copied" into site-packages
82
   */
83
  mountOverlay(overlayInfo: TarFSInfo, dir: InstallDir): void {
84
    const dest = dir == 'dynlib' ? this.dynlibTarFs : this.rootInfo;
85
    overlayInfo.children!.forEach((val, key) => {
86
      if (dest.children!.has(key)) {
87
        throw new PythonWorkersInternalError(
88
          `File/folder ${key} being written by multiple packages`
89
        );
90
      }
91
      dest.children!.set(key, val);
92
    });
93
  }
94
 
95
  /**
96
   * A small bundle contains just a single package, it can be thought of as a wheel.
97
   *
98
   * The entire bundle will be overlaid onto site-packages or /usr/lib depending on its install_dir.
99
   *
100
   * @param {TarInfo} tarInfo The root tarInfo for the small bundle (See tar.js)
101
   * @param {List<String>} soFiles A list of .so files contained in the small bundle
102
   * @param {String} requirement The canonicalized package name this small bundle corresponds to
103
   * @param {InstallDir} installDir The `install_dir` field from the metadata about the package taken from the lockfile
104
   */
105
  addSmallBundle(
106
    tarInfo: TarFSInfo,
107
    soFiles: string[],
108
    requirement: string,
109
    installDir: InstallDir
110
  ): void {
111
    for (const soFile of soFiles) {
112
      this.soFiles.push(soFile.split('/'));
113
    }
114
    this.mountOverlay(tarInfo, installDir);
115
    this.loadedRequirements.add(requirement);
116
  }
117
 
118
  /**
119
   * A big bundle contains multiple packages, each package contained in a folder whose name is the canonicalized package name.
120
   * This function overlays the requested packages onto the site-packages directory.
121
   * @param {TarInfo} tarInfo The root tarInfo for the big bundle (See tar.js)
122
   * @param {List<String>} soFiles A list of .so files contained in the big bundle
123
   * @param {List<String>} requirements canonicalized list of packages to pick from the big bundle
124
   */
125
  addBigBundle(
126
    tarInfo: TarFSInfo,
127
    soFiles: string[],
128
    requirements: Set<string>
129
  ): void {
130
    // add all the .so files we will need to preload from the big bundle
131
    for (const soFile of soFiles) {
132
      // If folder is in list of requirements include .so file in list to preload.
133
      const [pkg, ...rest] = soFile.split('/');
134
      if (requirements.has(pkg!)) {
135
        this.soFiles.push(rest);
136
      }
137
    }
138
 
139
    for (const req of requirements) {
140
      const child = tarInfo.children!.get(req);
141
      if (!child) {
142
        throw new PythonUserError(
143
          `Requirement ${req} not found in pyodide packages tar`
144
        );
145
      }
146
      this.mountOverlay(child, 'site');
147
      this.loadedRequirements.add(req);
148
    }
149
  }
150
 
151
  getSitePackagesRoot(): TarFSInfo {
152
    return this.rootInfo;
153
  }
154
 
155
  getDynlibRoot(): TarFSInfo {
156
    return this.dynlibTarFs;
157
  }
158
 
159
  /** Only used for Pyodide 0.26.0a2 */
160
  getSoFilesToLoad(): FilePath[] {
161
    return this.soFiles;
162
  }
163
 
164
  hasRequirementLoaded(req: string): boolean {
165
    return this.loadedRequirements.has(req);
166
  }
167
 
168
  mount(Module: Module, tarFS: EmscriptenFS<TarFSInfo>): void {
169
    Module.FS.mkdirTree(Module.FS.sessionSitePackages);
170
    Module.FS.mount(
171
      tarFS,
172
      { info: this.rootInfo },
173
      Module.FS.sessionSitePackages
174
    );
175
    Module.FS.mkdirTree(DYNLIB_PATH);
176
    Module.FS.mount(tarFS, { info: this.dynlibTarFs }, DYNLIB_PATH);
177
  }
178
}
179
 
180
/**
181
 * This stitches together the view of the site packages directory. Each
182
 * requirement corresponds to a folder in the original tar file. For each
183
 * requirement in the list we grab the corresponding folder and stitch them
184
 * together into a combined folder.
185
 *
186
 * This also returns the list of soFiles in the resulting site-packages
187
 * directory so we can preload them.
188
 *
189
 * TODO(later): This needs to be removed when external package loading is enabled.
190
 */
191
export function buildVirtualizedDir(): VirtualizedDir {
192
  if (EmbeddedPackagesTarReader.read === undefined) {
193
    // Package retrieval is enabled, so the embedded tar reader isn't initialized.
194
    // All packages, including STDLIB_PACKAGES, are loaded in `loadPackages`.
195
    return new VirtualizedDir();
196
  }
197
 
198
  const [bigTarInfo, bigTarSoFiles] = parseTarInfo(EmbeddedPackagesTarReader);
199
 
200
  const requirementsInBigBundle = new Set(STDLIB_PACKAGES);
201
  const res = new VirtualizedDir();
202
  res.addBigBundle(bigTarInfo, bigTarSoFiles, requirementsInBigBundle);
203
 
204
  return res;
205
}
206
 
207
/**
208
 * Patch loadPackage:
209
 *  - in workerd, disable integrity checks
210
 *  - otherwise, disable it entirely
211
 *
212
 * TODO: stop using loadPackage in workerd.
213
 */
214
export function patchLoadPackage(pyodide: Pyodide): void {
215
  pyodide.loadPackage = disabledLoadPackage;
216
  return;
217
}
218
 
219
function disabledLoadPackage(): never {
220
  throw new PythonWorkersInternalError(
221
    'pyodide.loadPackage is disabled because packages are encoded in the binary'
222
  );
223
}
224
 
225
/**
226
 * This mounts the metadataFS (which contains user code).
227
 */
228
export function mountWorkerFiles(Module: Module): void {
229
  Module.FS.mkdirTree('/session/metadata');
230
  const mdFS = createMetadataFS(Module);
231
  Module.FS.mount(mdFS, {}, '/session/metadata');
232
  invalidateCaches(Module);
233
}
234
 
235
/**
236
 * Add the directories created by mountLib to sys.path.
237
 * Has to run after the runtime is initialized but before memory snapshot is collected.
238
 */
239
export function adjustSysPath(Module: Module): void {
240
  const site_packages = Module.FS.sessionSitePackages;
241
  simpleRunPython(
242
    Module,
243
    `import sys; sys.path.append("/session/metadata"); sys.path.append("${site_packages}"); del sys`
244
  );
245
}
246
 
247
export const VIRTUALIZED_DIR = buildVirtualizedDir();