1
// Adapted from github.com/evanw/esbuild
2
// Original copyright and license:
3
//     Copyright (c) 2020 Evan Wallace
4
//     MIT License
5
//
6
//     Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
7
//     The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
8
//     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
9
import {
10
  downloadedBinPath,
11
  pkgAndSubpathForCurrentPlatform,
12
} from "./node-platform";
13
 
14
import fs from "fs";
15
import os from "os";
16
import path from "path";
17
import zlib from "zlib";
18
import https from "https";
19
import child_process from "child_process";
20
 
21
declare const LATEST_COMPATIBILITY_DATE: string;
22
declare const WORKERD_VERSION: string;
23
 
24
const toPath = path.join(__dirname, "bin", "workerd");
25
let isToPathJS = true;
26
 
27
function validateBinaryVersion(...command: string[]): void {
28
  command.push("--version");
29
  let stdout: string;
30
  try {
31
    stdout = child_process
32
      .execFileSync(command.shift()!, command, {
33
        // Without this, this install script strangely crashes with the error
34
        // "EACCES: permission denied, write" but only on Ubuntu Linux when node is
35
        // installed from the Snap Store. This is not a problem when you download
36
        // the official version of node. The problem appears to be that stderr
37
        // (i.e. file descriptor 2) isn't writable?
38
        //
39
        // More info:
40
        // - https://snapcraft.io/ (what the Snap Store is)
41
        // - https://nodejs.org/dist/ (download the official version of node)
42
        // - https://github.com/evanw/esbuild/issues/1711#issuecomment-1027554035
43
        //
44
        stdio: [/* stdin */ "pipe", /* stdout */ "pipe", /* stderr */ "inherit"],
45
      })
46
      .toString()
47
      .trim();
48
  } catch (e) {
49
    let msg = `[workerd] Failed to validate workerd binary
50
51
Local development will not work. This usually means you're on an unsupported
52
operating system, or missing some shared libraries.`;
53
    if (process.platform === "linux") {
54
      msg += " On Debian-based systems,\nmake sure you've installed the `libc++1` package."
55
    }
56
    console.error(msg);
57
    return;
58
  }
59
  if (stdout !== `workerd ${LATEST_COMPATIBILITY_DATE}`) {
60
    throw new Error(
61
      `Expected ${JSON.stringify(
62
        LATEST_COMPATIBILITY_DATE
63
      )} but got ${JSON.stringify(stdout)}`
64
    );
65
  }
66
}
67
 
68
function isYarn(): boolean {
69
  const { npm_config_user_agent } = process.env;
70
  if (npm_config_user_agent) {
71
    return /\byarn\//.test(npm_config_user_agent);
72
  }
73
  return false;
74
}
75
 
76
function fetch(url: string): Promise<Buffer> {
77
  return new Promise((resolve, reject) => {
78
    https
79
      .get(url, (res) => {
80
        if (
81
          (res.statusCode === 301 || res.statusCode === 302) &&
82
          res.headers.location
83
        )
84
          return fetch(res.headers.location).then(resolve, reject);
85
        if (res.statusCode !== 200)
86
          return reject(new Error(`Server responded with ${res.statusCode}`));
87
        let chunks: Buffer[] = [];
88
        res.on("data", (chunk) => chunks.push(chunk));
89
        res.on("end", () => resolve(Buffer.concat(chunks)));
90
      })
91
      .on("error", reject);
92
  });
93
}
94
 
95
function extractFileFromTarGzip(buffer: Buffer, subpath: string): Buffer {
96
  try {
97
    buffer = zlib.unzipSync(buffer);
98
  } catch (err: any) {
99
    throw new Error(
100
      `Invalid gzip data in archive: ${(err && err.message) || err}`
101
    );
102
  }
103
  let str = (i: number, n: number) =>
104
    String.fromCharCode(...buffer.subarray(i, i + n)).replace(/\0.*$/, "");
105
  let offset = 0;
106
  subpath = `package/${subpath}`;
107
  while (offset < buffer.length) {
108
    let name = str(offset, 100);
109
    let size = parseInt(str(offset + 124, 12), 8);
110
    offset += 512;
111
    if (!isNaN(size)) {
112
      if (name === subpath) return buffer.subarray(offset, offset + size);
113
      offset += (size + 511) & ~511;
114
    }
115
  }
116
  throw new Error(`Could not find ${JSON.stringify(subpath)} in archive`);
117
}
118
 
119
function installUsingNPM(pkg: string, subpath: string, binPath: string): void {
120
  // Erase "npm_config_global" so that "npm install --global workerd" works.
121
  // Otherwise this nested "npm install" will also be global, and the install
122
  // will deadlock waiting for the global installation lock.
123
  const env = { ...process.env, npm_config_global: undefined };
124
 
125
  // Create a temporary directory inside the "workerd" package with an empty
126
  // "package.json" file. We'll use this to run "npm install" in.
127
  const libDir = path.dirname(require.resolve("workerd"));
128
  const installDir = path.join(libDir, "npm-install");
129
  fs.mkdirSync(installDir);
130
  try {
131
    fs.writeFileSync(path.join(installDir, "package.json"), "{}");
132
 
133
    // Run "npm install" in the temporary directory which should download the
134
    // desired package. Try to avoid unnecessary log output. This uses the "npm"
135
    // command instead of a HTTP request so that it hopefully works in situations
136
    // where HTTP requests are blocked but the "npm" command still works due to,
137
    // for example, a custom configured npm registry and special firewall rules.
138
    child_process.execSync(
139
      `npm install --loglevel=error --prefer-offline --no-audit --progress=false ${pkg}@${WORKERD_VERSION}`,
140
      { cwd: installDir, stdio: "pipe", env }
141
    );
142
 
143
    // Move the downloaded binary executable into place. The destination path
144
    // is the same one that the JavaScript API code uses so it will be able to
145
    // find the binary executable here later.
146
    const installedBinPath = path.join(
147
      installDir,
148
      "node_modules",
149
      pkg,
150
      subpath
151
    );
152
    fs.renameSync(installedBinPath, binPath);
153
  } finally {
154
    // Try to clean up afterward so we don't unnecessarily waste file system
155
    // space. Leaving nested "node_modules" directories can also be problematic
156
    // for certain tools that scan over the file tree and expect it to have a
157
    // certain structure.
158
    try {
159
      removeRecursive(installDir);
160
    } catch {
161
      // Removing a file or directory can randomly break on Windows, returning
162
      // EBUSY for an arbitrary length of time. I think this happens when some
163
      // other program has that file or directory open (e.g. an anti-virus
164
      // program). This is fine on Unix because the OS just unlinks the entry
165
      // but keeps the reference around until it's unused. There's nothing we
166
      // can do in this case so we just leave the directory there.
167
    }
168
  }
169
}
170
 
171
function removeRecursive(dir: string): void {
172
  for (const entry of fs.readdirSync(dir)) {
173
    const entryPath = path.join(dir, entry);
174
    let stats;
175
    try {
176
      stats = fs.lstatSync(entryPath);
177
    } catch {
178
      continue; // Guard against https://github.com/nodejs/node/issues/4760
179
    }
180
    if (stats.isDirectory()) removeRecursive(entryPath);
181
    else fs.unlinkSync(entryPath);
182
  }
183
  fs.rmdirSync(dir);
184
}
185
 
186
function maybeOptimizePackage(binPath: string): void {
187
  // This package contains a "bin/workerd" JavaScript file that finds and runs
188
  // the appropriate binary executable. However, this means that running the
189
  // "workerd" command runs another instance of "node" which is way slower than
190
  // just running the binary executable directly.
191
  //
192
  // Here we optimize for this by replacing the JavaScript file with the binary
193
  // executable at install time. This optimization does not work on Windows
194
  // because on Windows the binary executable must be called "workerd.exe"
195
  // instead of "workerd".
196
  //
197
  // This doesn't work with Yarn both because of lack of support for binary
198
  // files in Yarn 2+ (see https://github.com/yarnpkg/berry/issues/882) and
199
  // because Yarn (even Yarn 1?) may run the same install scripts in the same
200
  // place multiple times from different platforms, especially when people use
201
  // Docker. Avoid idempotency issues by just not optimizing when using Yarn.
202
  //
203
  // This optimization also doesn't apply when npm's "--ignore-scripts" flag is
204
  // used since in that case this install script will not be run.
205
  if (os.platform() !== "win32" && !isYarn()) {
206
    const tempPath = path.join(__dirname, "bin-workerd");
207
    try {
208
      // First link the binary with a temporary file. If this fails and throws an
209
      // error, then we'll just end up doing nothing. This uses a hard link to
210
      // avoid taking up additional space on the file system.
211
      fs.linkSync(binPath, tempPath);
212
 
213
      // Then use rename to atomically replace the target file with the temporary
214
      // file. If this fails and throws an error, then we'll just end up leaving
215
      // the temporary file there, which is harmless.
216
      fs.renameSync(tempPath, toPath);
217
 
218
      // If we get here, then we know that the target location is now a binary
219
      // executable instead of a JavaScript file.
220
      isToPathJS = false;
221
 
222
      // If this install script is being re-run, then "renameSync" will fail
223
      // since the underlying inode is the same (it just returns without doing
224
      // anything, and without throwing an error). In that case we should remove
225
      // the file manually.
226
      fs.unlinkSync(tempPath);
227
    } catch {
228
      // Ignore errors here since this optimization is optional
229
    }
230
  }
231
}
232
 
233
async function downloadDirectlyFromNPM(
234
  pkg: string,
235
  subpath: string,
236
  binPath: string
237
): Promise<void> {
238
  // If that fails, the user could have npm configured incorrectly or could not
239
  // have npm installed. Try downloading directly from npm as a last resort.
240
  const unscopedPkg = pkg.substring(pkg.indexOf("/") + 1);
241
  const url = `https://registry.npmjs.org/${pkg}/-/${unscopedPkg}-${WORKERD_VERSION}.tgz`;
242
  console.error(`[workerd] Trying to download ${JSON.stringify(url)}`);
243
  try {
244
    fs.writeFileSync(
245
      binPath,
246
      extractFileFromTarGzip(await fetch(url), subpath)
247
    );
248
    fs.chmodSync(binPath, 0o755);
249
  } catch (e: any) {
250
    console.error(
251
      `[workerd] Failed to download ${JSON.stringify(url)}: ${
252
        (e && e.message) || e
253
      }`
254
    );
255
    throw e;
256
  }
257
}
258
 
259
async function checkAndPreparePackage(): Promise<void> {
260
  const { pkg, subpath } = pkgAndSubpathForCurrentPlatform();
261
 
262
  let binPath: string;
263
  try {
264
    // First check for the binary package from our "optionalDependencies". This
265
    // package should have been installed alongside this package at install time.
266
    binPath = require.resolve(`${pkg}/${subpath}`);
267
  } catch (e) {
268
    console.error(`[workerd] Failed to find package "${pkg}" on the file system
269
270
This can happen if you use the "--no-optional" flag. The "optionalDependencies"
271
package.json feature is used by workerd to install the correct binary executable
272
for your current platform. This install script will now attempt to work around
273
this. If that fails, you need to remove the "--no-optional" flag to use workerd.
274
`);
275
 
276
    // If that didn't work, then someone probably installed workerd with the
277
    // "--no-optional" flag. Attempt to compensate for this by downloading the
278
    // package using a nested call to "npm" instead.
279
    //
280
    // THIS MAY NOT WORK. Package installation uses "optionalDependencies" for
281
    // a reason: manually downloading the package has a lot of obscure edge
282
    // cases that fail because people have customized their environment in
283
    // some strange way that breaks downloading. This code path is just here
284
    // to be helpful but it's not the supported way of installing workerd.
285
    binPath = downloadedBinPath(pkg, subpath);
286
    try {
287
      console.error(`[workerd] Trying to install package "${pkg}" using npm`);
288
      installUsingNPM(pkg, subpath, binPath);
289
    } catch (e2: any) {
290
      console.error(
291
        `[workerd] Failed to install package "${pkg}" using npm: ${
292
          (e2 && e2.message) || e2
293
        }`
294
      );
295
 
296
      // If that didn't also work, then something is likely wrong with the "npm"
297
      // command. Attempt to compensate for this by manually downloading the
298
      // package from the npm registry over HTTP as a last resort.
299
      try {
300
        await downloadDirectlyFromNPM(pkg, subpath, binPath);
301
      } catch (e3: any) {
302
        throw new Error(`Failed to install package "${pkg}"`);
303
      }
304
    }
305
  }
306
 
307
  maybeOptimizePackage(binPath);
308
}
309
 
310
checkAndPreparePackage().then(() => {
311
  if (isToPathJS) {
312
    // We need "node" before this command since it's a JavaScript file
313
    validateBinaryVersion(process.execPath, toPath);
314
  } else {
315
    // This is no longer a JavaScript file so don't run it using "node"
316
    validateBinaryVersion(toPath);
317
  }
318
});