Skip to content
rachel/anvil
File

Blob: tests/integration/queue-runner/harness.ts

typescript624 lines
1import { spawn, execFile as execFileCallback, type ChildProcess } from "node:child_process";
2import { createWriteStream, type WriteStream } from "node:fs";
3import { readFile, rm, writeFile } from "node:fs/promises";
4import net from "node:net";
5import { join, resolve } from "node:path";
6import process from "node:process";
7import crypto from "node:crypto";
8import { setTimeout as sleep } from "node:timers/promises";
9import { promisify } from "node:util";
10import { fileURLToPath } from "node:url";
11 
12import type {
13  CreateProjectRequest,
14  CreateWebhookRequest,
15  DispatchMode,
16  GetMeResponse,
17  GetProjectRunsResponse,
18  GetProjectWebhooksResponse,
19  ProjectDetail,
20  ProjectResponse,
21  RunDetail,
22  RunStatus,
23  TriggerRunRequest,
24  TriggerRunAcceptedResponse,
25  UpsertWebhookResponse,
26} from "@/contracts";
27import { BranchName, CommitSha, OwnerSlug, ProjectSlug } from "@/contracts";
28import {
29  E2E_BASELINE_EMAIL,
30  E2E_BASELINE_NAME,
31  E2E_BASELINE_SLUG,
32  E2E_BASELINE_TESSERA_SUB,
33  setMockOidcIdentity,
34  TEST_OIDC_CLIENT_ID,
35  TEST_OIDC_CLIENT_SECRET,
36  type MockIdentity,
37} from "../../helpers/oidc-mock";
38 
39const execFile = promisify(execFileCallback);
40 
41const REPO_ROOT = resolve(fileURLToPath(new URL("../../..", import.meta.url)));
42export const FIXTURE_REPO_URL = "https://github.com/miragespace/ci-test";
43export const FIXTURE_DEFAULT_BRANCH = BranchName.assertDecode("main");
44export const FIXTURE_CONFIG_PATH = ".anvil.yml";
45export const GITHUB_WEBHOOK_PROVIDER = "github";
46const WEBHOOK_BEFORE_SHA = CommitSha.assertDecode("1111111111111111111111111111111111111111");
47export const EXPECTED_STEP_NAMES = ["install", "test", "build"] as const;
48export const EXPECTED_LOG_MARKERS = ["CI Runner Probe", "Registry probe succeeded"] as const;
49const SERVER_READY_TIMEOUT_MS = 120_000;
50const RUN_TIMEOUT_MS = 240_000;
51const PROJECT_SETTLE_TIMEOUT_MS = 120_000;
52const POLL_INTERVAL_MS = 1_000;
53const NPX_COMMAND = process.platform === "win32" ? "npx.cmd" : "npx";
54const VITE_BIN_PATH = resolve(REPO_ROOT, "node_modules/vite/bin/vite.js");
55const SESSION_COOKIE_NAME = "__Host-anvil_session";
56const DEV_VARS_EXAMPLE_PATH = resolve(REPO_ROOT, ".dev.vars.example");
57 
58export interface OperatorIdentity extends MockIdentity {
59  displayName: string;
60  slug: OwnerSlug;
61}
62 
63export type SessionId = string;
64export type ProjectRecord = ProjectResponse["project"];
65export type ProjectId = ProjectRecord["id"];
66export type IndexedRun = GetProjectRunsResponse["runs"][number];
67export type RunId = TriggerRunAcceptedResponse["runId"];
68export type WebhookSummary = GetProjectWebhooksResponse["webhooks"][number];
69export type WebhookDelivery = WebhookSummary["recentDeliveries"][number];
70 
71interface CreateProjectOptions {
72  dispatchMode?: DispatchMode;
73}
74 
75export interface IntegrationContext {
76  tempDir: string;
77  baseUrl: string;
78  port: number;
79  serverProcess: ChildProcess;
80  stdoutLogPath: string;
81  stderrLogPath: string;
82  stdoutStream: WriteStream;
83  stderrStream: WriteStream;
84  cloudflareEnv: string;
85  devVarsPath: string;
86  oidcIssuer: string;
87}
88 
89class AssertionError extends Error {}
90 
91export const assert: (condition: unknown, message: string) => asserts condition = (condition, message) => {
92  if (!condition) {
93    throw new AssertionError(message);
94  }
95};
96 
97const delayUntil = async <T>(description: string, timeoutMs: number, action: () => Promise<T | null>): Promise<T> => {
98  const deadline = Date.now() + timeoutMs;
99  let lastError: unknown = null;
100 
101  while (Date.now() < deadline) {
102    try {
103      const result = await action();
104      if (result !== null) {
105        return result;
106      }
107    } catch (error) {
108      lastError = error;
109    }
110 
111    await sleep(POLL_INTERVAL_MS);
112  }
113 
114  if (lastError instanceof Error) {
115    throw new Error(`${description} timed out: ${lastError.message}`);
116  }
117 
118  throw new Error(`${description} timed out.`);
119};
120 
121const slugFragment = (): string => crypto.randomUUID().replace(/-/gu, "").slice(0, 10);
122 
123const closeWriteStream = async (stream: WriteStream): Promise<void> =>
124  await new Promise<void>((resolveClose, reject) => {
125    if (stream.destroyed || stream.closed || stream.writableFinished) {
126      resolveClose();
127      return;
128    }
129 
130    try {
131      stream.end((error?: Error | null) => {
132        if (error) {
133          reject(error);
134          return;
135        }
136 
137        resolveClose();
138      });
139    } catch (error) {
140      reject(error);
141    }
142  });
143 
144const getFreePort = async (): Promise<number> =>
145  await new Promise<number>((resolvePort, reject) => {
146    const server = net.createServer();
147    server.unref();
148    server.once("error", reject);
149    server.listen(0, "127.0.0.1", () => {
150      const address = server.address();
151      if (!address || typeof address === "string") {
152        server.close(() => reject(new Error("Failed to allocate a localhost port.")));
153        return;
154      }
155 
156      server.close((error?: Error | null) => {
157        if (error) {
158          reject(error);
159          return;
160        }
161 
162        resolvePort(address.port);
163      });
164    });
165  });
166 
167const apiFetch = async <T>(baseUrl: string, path: string, init: RequestInit = {}): Promise<T> => {
168  const response = await fetch(new URL(path, baseUrl), init);
169  const text = await response.text();
170  const body: unknown = text.length > 0 ? JSON.parse(text) : null;
171 
172  if (!response.ok) {
173    throw new Error(`HTTP ${response.status} for ${path}: ${text}`);
174  }
175 
176  return body as T;
177};
178 
179const apiFetchStatus = async (baseUrl: string, path: string): Promise<number> => {
180  const response = await fetch(new URL(path, baseUrl), {
181    signal: AbortSignal.timeout(5_000),
182  });
183  await response.arrayBuffer();
184  return response.status;
185};
186 
187const execCommand = async (
188  command: string,
189  args: string[],
190  env: NodeJS.ProcessEnv,
191): Promise<{ stdout: string; stderr: string }> =>
192  await execFile(command, args, {
193    cwd: REPO_ROOT,
194    env,
195    maxBuffer: 10 * 1024 * 1024,
196  });
197 
198const waitForServerReady = async (baseUrl: string): Promise<void> => {
199  await delayUntil("dev server readiness", SERVER_READY_TIMEOUT_MS, async () => {
200    const rootStatus = await apiFetchStatus(baseUrl, "/");
201    const privateStatus = await apiFetchStatus(baseUrl, "/api/private/me");
202    return rootStatus === 200 && privateStatus === 403 ? true : null;
203  });
204};
205 
206export const applyMigrations = async (persistTo: string): Promise<void> => {
207  await execCommand(
208    NPX_COMMAND,
209    ["wrangler", "d1", "migrations", "apply", "anvil-db", "--local", "--persist-to", persistTo],
210    {
211      ...process.env,
212      CI: "1",
213      NO_D1_WARNING: "true",
214    },
215  );
216};
217 
218const createDevVarsFile = async (oidcIssuer: string): Promise<{ cloudflareEnv: string; devVarsPath: string }> => {
219  const cloudflareEnv = `e2e-${process.pid}-${Date.now()}`;
220  const devVarsPath = resolve(REPO_ROOT, `.dev.vars.${cloudflareEnv}`);
221  const template = await readFile(DEV_VARS_EXAMPLE_PATH, "utf8");
222  const stripped = template
223    .replace(/^TESSERA_OIDC_ISSUER=.*$/gmu, "")
224    .replace(/^TESSERA_OIDC_CLIENT_ID=.*$/gmu, "")
225    .replace(/^TESSERA_OIDC_CLIENT_SECRET=.*$/gmu, "")
226    .trimEnd();
227  const content =
228    `${stripped}\n` +
229    `TESSERA_OIDC_ISSUER=${oidcIssuer}\n` +
230    `TESSERA_OIDC_CLIENT_ID=${TEST_OIDC_CLIENT_ID}\n` +
231    `TESSERA_OIDC_CLIENT_SECRET=${TEST_OIDC_CLIENT_SECRET}\n`;
232 
233  await writeFile(devVarsPath, content);
234  return { cloudflareEnv, devVarsPath };
235};
236 
237export const startDevServer = async (persistTo: string, oidcIssuer: string): Promise<IntegrationContext> => {
238  const port = await getFreePort();
239  const baseUrl = `http://127.0.0.1:${port}`;
240  const { cloudflareEnv, devVarsPath } = await createDevVarsFile(oidcIssuer);
241  const stdoutLogPath = join(persistTo, "dev.stdout.log");
242  const stderrLogPath = join(persistTo, "dev.stderr.log");
243  const stdoutStream = createWriteStream(stdoutLogPath, { flags: "a" });
244  const stderrStream = createWriteStream(stderrLogPath, { flags: "a" });
245  const serverProcess = spawn(process.execPath, [VITE_BIN_PATH, "dev", "--host", "127.0.0.1", "--port", String(port)], {
246    cwd: REPO_ROOT,
247    env: {
248      ...process.env,
249      ANVIL_PERSIST_STATE_PATH: persistTo,
250      CLOUDFLARE_ENV: cloudflareEnv,
251    },
252    stdio: ["ignore", "pipe", "pipe"],
253    detached: process.platform !== "win32",
254  });
255 
256  serverProcess.stdout?.pipe(stdoutStream);
257  serverProcess.stderr?.pipe(stderrStream);
258 
259  try {
260    await Promise.race([
261      waitForServerReady(baseUrl),
262      new Promise<never>((_, reject) => {
263        serverProcess.once("exit", (code, signal) => {
264          reject(new Error(`dev server exited before readiness (code=${code}, signal=${signal})`));
265        });
266      }),
267    ]);
268  } catch (error) {
269    await stopDevServer(serverProcess);
270    await closeWriteStream(stdoutStream);
271    await closeWriteStream(stderrStream);
272    await rm(devVarsPath, { force: true });
273    throw error;
274  }
275 
276  return {
277    tempDir: persistTo,
278    baseUrl,
279    port,
280    serverProcess,
281    stdoutLogPath,
282    stderrLogPath,
283    stdoutStream,
284    stderrStream,
285    cloudflareEnv,
286    devVarsPath,
287    oidcIssuer,
288  };
289};
290 
291export const stopDevServer = async (serverProcess: ChildProcess): Promise<void> => {
292  if (serverProcess.exitCode !== null || serverProcess.signalCode !== null) {
293    return;
294  }
295 
296  const killServer = (signal: NodeJS.Signals): void => {
297    if (serverProcess.pid === undefined) {
298      return;
299    }
300 
301    if (process.platform === "win32") {
302      serverProcess.kill(signal);
303      return;
304    }
305 
306    process.kill(-serverProcess.pid, signal);
307  };
308 
309  killServer("SIGTERM");
310  const exited = await Promise.race([
311    new Promise<boolean>((resolveExit) => {
312      serverProcess.once("exit", () => resolveExit(true));
313    }),
314    sleep(10_000).then(() => false),
315  ]);
316 
317  if (!exited && serverProcess.exitCode === null && serverProcess.signalCode === null) {
318    killServer("SIGKILL");
319    await new Promise<void>((resolveExit) => {
320      serverProcess.once("exit", () => resolveExit());
321    });
322  }
323};
324 
325export const closeContextLogs = async (context: IntegrationContext): Promise<void> => {
326  context.serverProcess.stdout?.unpipe(context.stdoutStream);
327  context.serverProcess.stderr?.unpipe(context.stderrStream);
328  await closeWriteStream(context.stdoutStream);
329  await closeWriteStream(context.stderrStream);
330};
331 
332const authHeaders = (baseUrl: string, sessionId: SessionId, headers?: HeadersInit): Headers => {
333  const result = new Headers(headers);
334  result.set("cookie", `${SESSION_COOKIE_NAME}=${sessionId}`);
335  result.set("origin", new URL(baseUrl).origin);
336  return result;
337};
338 
339export const createOperatorIdentity = (): OperatorIdentity => {
340  const slug = OwnerSlug.assertDecode(E2E_BASELINE_SLUG);
341  return {
342    sub: E2E_BASELINE_TESSERA_SUB,
343    email: E2E_BASELINE_EMAIL,
344    email_verified: true,
345    name: E2E_BASELINE_NAME,
346    displayName: E2E_BASELINE_NAME,
347    slug,
348  };
349};
350 
351const toGitHubRepositoryFullName = (repositoryUrl: string): string => {
352  const url = new URL(repositoryUrl);
353  return url.pathname.replace(/^\/+/u, "");
354};
355 
356const buildGitHubRepository = (repositoryUrl: string, defaultBranch: string) => ({
357  full_name: toGitHubRepositoryFullName(repositoryUrl),
358  html_url: repositoryUrl,
359  clone_url: `${repositoryUrl}.git`,
360  default_branch: defaultBranch,
361});
362 
363const signGitHubPayload = (secret: string, body: string): string =>
364  `sha256=${crypto.createHmac("sha256", secret).update(body).digest("hex")}`;
365 
366const extractCookieValue = (setCookie: string | null, name: string): string => {
367  const cookie = setCookie
368    ?.split(/,(?=\s*[^;=]+=[^;]+)/u)
369    .map((part) => part.trim())
370    .find((part) => part.startsWith(`${name}=`));
371 
372  if (!cookie) {
373    throw new Error(`Missing ${name} cookie in Set-Cookie header.`);
374  }
375 
376  return cookie.slice(name.length + 1).split(";", 1)[0] ?? "";
377};
378 
379export const oidcSignInOnce = async (
380  context: Pick<IntegrationContext, "baseUrl" | "oidcIssuer">,
381  identity: OperatorIdentity,
382): Promise<SessionId> => {
383  await setMockOidcIdentity(context.oidcIssuer, identity);
384 
385  const start = await fetch(new URL("/api/public/oidc/start?return_to=%2Fapp%2Fprojects", context.baseUrl), {
386    redirect: "manual",
387    headers: { "cf-connecting-ip": "127.0.0.1" },
388  });
389  assert(start.status === 302, `Expected OIDC start redirect, got ${start.status}.`);
390  const txCookie = extractCookieValue(start.headers.get("set-cookie"), "__Host-anvil_oidc_tx");
391  const authorizationUrl = start.headers.get("location");
392  assert(authorizationUrl, "Expected OIDC start to return authorization Location.");
393 
394  const authorization = await fetch(authorizationUrl, { redirect: "manual" });
395  assert(
396    authorization.status === 307 || authorization.status === 302,
397    `Expected authorize redirect, got ${authorization.status}.`,
398  );
399  const callbackUrl = authorization.headers.get("location");
400  assert(callbackUrl, "Expected mock provider to redirect to callback.");
401 
402  const callback = await fetch(callbackUrl, {
403    redirect: "manual",
404    headers: {
405      cookie: `__Host-anvil_oidc_tx=${txCookie}`,
406      "cf-connecting-ip": "127.0.0.1",
407    },
408  });
409  assert(callback.status === 302, `Expected OIDC callback redirect, got ${callback.status} ${await callback.text()}.`);
410 
411  const sessionId = extractCookieValue(callback.headers.get("set-cookie"), SESSION_COOKIE_NAME);
412  const me = await apiFetch<GetMeResponse>(context.baseUrl, "/api/private/me", {
413    headers: authHeaders(context.baseUrl, sessionId),
414  });
415  assert(me.user.slug === identity.slug, `Expected signed-in slug ${identity.slug}, got ${me.user.slug}.`);
416  assert(me.user.email === identity.email, `Expected signed-in email ${identity.email}, got ${me.user.email}.`);
417  assert(me.user.displayName === identity.displayName, `Expected signed-in displayName ${identity.displayName}.`);
418 
419  return sessionId;
420};
421 
422export const createProject = async (
423  baseUrl: string,
424  sessionId: SessionId,
425  name: string,
426  options: CreateProjectOptions = {},
427): Promise<ProjectRecord> => {
428  const body = {
429    projectSlug: ProjectSlug.assertDecode(`queue-${slugFragment()}`),
430    name,
431    repoUrl: FIXTURE_REPO_URL,
432    defaultBranch: FIXTURE_DEFAULT_BRANCH,
433    configPath: FIXTURE_CONFIG_PATH,
434    dispatchMode: options.dispatchMode,
435  } satisfies CreateProjectRequest;
436 
437  const response = await apiFetch<ProjectResponse>(baseUrl, "/api/private/projects", {
438    method: "POST",
439    headers: authHeaders(baseUrl, sessionId, { "content-type": "application/json" }),
440    body: JSON.stringify(body),
441  });
442 
443  return response.project;
444};
445 
446export const resolveFixtureHeadCommitSha = async (): Promise<string> => {
447  const { stdout } = await execCommand("git", ["ls-remote", FIXTURE_REPO_URL, `refs/heads/${FIXTURE_DEFAULT_BRANCH}`], {
448    ...process.env,
449  });
450  const commitSha = stdout.trim().split(/\s+/u)[0];
451  assert(typeof commitSha === "string" && commitSha.length > 0, "Expected git ls-remote to return a commit SHA.");
452  return CommitSha.assertDecode(commitSha);
453};
454 
455export const triggerRun = async (baseUrl: string, sessionId: SessionId, projectId: ProjectId): Promise<RunId> => {
456  const body = {} satisfies TriggerRunRequest;
457  const response = await apiFetch<TriggerRunAcceptedResponse>(baseUrl, `/api/private/projects/${projectId}/runs`, {
458    method: "POST",
459    headers: authHeaders(baseUrl, sessionId, { "content-type": "application/json" }),
460    body: JSON.stringify(body),
461  });
462 
463  return response.runId;
464};
465 
466export const putGitHubWebhook = async (
467  baseUrl: string,
468  sessionId: SessionId,
469  projectId: ProjectId,
470  secret: string,
471): Promise<WebhookSummary> => {
472  const body = {
473    enabled: true,
474    secret,
475  } satisfies CreateWebhookRequest;
476  const response = await apiFetch<UpsertWebhookResponse>(
477    baseUrl,
478    `/api/private/projects/${projectId}/webhooks/${GITHUB_WEBHOOK_PROVIDER}`,
479    {
480      method: "PUT",
481      headers: authHeaders(baseUrl, sessionId, { "content-type": "application/json" }),
482      body: JSON.stringify(body),
483    },
484  );
485 
486  return response.webhook;
487};
488 
489export const getProjectWebhooks = async (
490  baseUrl: string,
491  sessionId: SessionId,
492  projectId: ProjectId,
493): Promise<GetProjectWebhooksResponse> =>
494  await apiFetch<GetProjectWebhooksResponse>(baseUrl, `/api/private/projects/${projectId}/webhooks`, {
495    headers: authHeaders(baseUrl, sessionId),
496  });
497 
498export const postGitHubPushWebhook = async (
499  baseUrl: string,
500  project: Pick<ProjectRecord, "ownerSlug" | "projectSlug" | "repoUrl" | "defaultBranch">,
501  secret: string,
502  deliveryId: string,
503  commitSha: string,
504): Promise<{ status: number; text: string }> => {
505  const body = JSON.stringify({
506    ref: `refs/heads/${project.defaultBranch}`,
507    before: WEBHOOK_BEFORE_SHA,
508    after: commitSha,
509    head_commit: {
510      id: commitSha,
511    },
512    repository: buildGitHubRepository(project.repoUrl, project.defaultBranch),
513  });
514  const response = await fetch(
515    new URL(`/api/public/hooks/${GITHUB_WEBHOOK_PROVIDER}/${project.ownerSlug}/${project.projectSlug}`, baseUrl),
516    {
517      method: "POST",
518      headers: {
519        "content-type": "application/json; charset=utf-8",
520        "x-github-event": "push",
521        "x-github-delivery": deliveryId,
522        "x-hub-signature-256": signGitHubPayload(secret, body),
523      },
524      body,
525    },
526  );
527 
528  return {
529    status: response.status,
530    text: await response.text(),
531  };
532};
533 
534export const getRunDetail = async (baseUrl: string, sessionId: SessionId, runId: RunId): Promise<RunDetail> =>
535  await apiFetch<RunDetail>(baseUrl, `/api/private/runs/${runId}`, {
536    headers: authHeaders(baseUrl, sessionId),
537  });
538 
539export const getProjectDetail = async (
540  baseUrl: string,
541  sessionId: SessionId,
542  projectId: ProjectId,
543): Promise<ProjectDetail> =>
544  await apiFetch<ProjectDetail>(baseUrl, `/api/private/projects/${projectId}`, {
545    headers: authHeaders(baseUrl, sessionId),
546  });
547 
548export const getProjectRuns = async (
549  baseUrl: string,
550  sessionId: SessionId,
551  projectId: ProjectId,
552): Promise<GetProjectRunsResponse> =>
553  await apiFetch<GetProjectRunsResponse>(baseUrl, `/api/private/projects/${projectId}/runs`, {
554    headers: authHeaders(baseUrl, sessionId),
555  });
556 
557export const waitForTerminalRun = async (baseUrl: string, sessionId: SessionId, runId: RunId): Promise<RunDetail> =>
558  await delayUntil(`run ${runId} terminalization`, RUN_TIMEOUT_MS, async () => {
559    const detail = await getRunDetail(baseUrl, sessionId, runId);
560    return detail.run.status === "passed" || detail.run.status === "failed" || detail.run.status === "canceled"
561      ? detail
562      : null;
563  });
564 
565export const waitForProjectSettled = async (
566  baseUrl: string,
567  sessionId: SessionId,
568  projectId: ProjectId,
569): Promise<ProjectDetail> =>
570  await delayUntil(`project ${projectId} reconciliation`, PROJECT_SETTLE_TIMEOUT_MS, async () => {
571    const detail = await getProjectDetail(baseUrl, sessionId, projectId);
572    return detail.activeRun === null && detail.pendingRuns.length === 0 ? detail : null;
573  });
574 
575export const waitForIndexedRun = async (
576  baseUrl: string,
577  sessionId: SessionId,
578  projectId: ProjectId,
579  runId: RunId,
580  expectedStatus: RunStatus,
581): Promise<IndexedRun> =>
582  await delayUntil(`run ${runId} D1 sync`, PROJECT_SETTLE_TIMEOUT_MS, async () => {
583    const response = await getProjectRuns(baseUrl, sessionId, projectId);
584    const found = response.runs.find((run) => run.id === runId);
585    return found && found.status === expectedStatus ? found : null;
586  });
587 
588export const waitForAcceptedWebhookDelivery = async (
589  baseUrl: string,
590  sessionId: SessionId,
591  projectId: ProjectId,
592  deliveryId: string,
593): Promise<WebhookDelivery> =>
594  await delayUntil(`webhook delivery ${deliveryId} acceptance`, PROJECT_SETTLE_TIMEOUT_MS, async () => {
595    const response = await getProjectWebhooks(baseUrl, sessionId, projectId);
596    const webhook = response.webhooks.find((candidate) => candidate.provider === GITHUB_WEBHOOK_PROVIDER);
597    const delivery = webhook?.recentDeliveries.find((candidate) => candidate.deliveryId === deliveryId);
598    return delivery && delivery.outcome === "accepted" && delivery.runId !== null ? delivery : null;
599  });
600 
601export const printFailureContext = (context: IntegrationContext): void => {
602  console.error(`Preserved temp state: ${context.tempDir}`);
603  console.error(`Dev server stdout: ${context.stdoutLogPath}`);
604  console.error(`Dev server stderr: ${context.stderrLogPath}`);
605  console.error(
606    `Reopen preserved state: CLOUDFLARE_ENV=${context.cloudflareEnv} ANVIL_PERSIST_STATE_PATH=${context.tempDir} npm run dev -- --host 127.0.0.1 --port ${context.port}`,
607  );
608};
609 
610export const printLogTails = async (context: IntegrationContext): Promise<void> => {
611  for (const [label, filePath] of [
612    ["stdout", context.stdoutLogPath],
613    ["stderr", context.stderrLogPath],
614  ] as const) {
615    try {
616      const content = await readFile(filePath, "utf8");
617      const tail = content.trim().split("\n").slice(-20).join("\n");
618      if (tail.length > 0) {
619        console.error(`Last ${label} log lines:\n${tail}`);
620      }
621    } catch {}
622  }
623};