Skip to content
rachel/bland
File

Blob: tests/e2e/harness.ts

typescript521 lines
1import { spawn, execFile as execFileCallback, type ChildProcess } from "node:child_process";
2import { createWriteStream, type WriteStream } from "node:fs";
3import { readFile, rm, writeFile } from "node:fs/promises";
4import net from "node:net";
5import { join, resolve } from "node:path";
6import process from "node:process";
7import { setTimeout as sleep } from "node:timers/promises";
8import { promisify } from "node:util";
9import { fileURLToPath } from "node:url";
10 
11const execFile = promisify(execFileCallback);
12 
13import {
14  E2E_BASELINE_EMAIL,
15  E2E_BASELINE_NAME,
16  E2E_BASELINE_TESSERA_SUB,
17  TEST_OIDC_CLIENT_ID,
18  TEST_OIDC_CLIENT_SECRET,
19} from "./oidc-mock";
20 
21const REPO_ROOT = resolve(fileURLToPath(new URL("../..", import.meta.url)));
22const VITE_BIN_PATH = resolve(REPO_ROOT, "node_modules/vite/bin/vite.js");
23const WRANGLER_BIN_PATH = resolve(REPO_ROOT, "node_modules/wrangler/bin/wrangler.js");
24const EMOJI_GENERATOR_PATH = resolve(REPO_ROOT, "scripts/generate-emoji-data.ts");
25const SERVER_READY_TIMEOUT_MS = 120_000;
26const SERVER_START_ATTEMPTS = 5;
27const POLL_INTERVAL_MS = 1_000;
28const COMMAND_STDIO_MAX_BYTES = 10 * 1024 * 1024;
29const COMMAND_TAIL_BYTES = 8_000;
30const LOG_TAIL_LINES = 40;
31const PROCESS_TERM_TIMEOUT_MS = 10_000;
32const PROCESS_KILL_TIMEOUT_MS = 5_000;
33const PROCESS_CLOSE_TIMEOUT_MS = 2_000;
34const D1_DATABASE = "bland-prod";
35const DEV_VARS_PATH = resolve(REPO_ROOT, ".dev.vars");
36const DEV_VARS_EXAMPLE_PATH = resolve(REPO_ROOT, ".dev.vars.example");
37 
38const WORKSPACE_NAME = "bland";
39const WORKSPACE_SLUG = "bland";
40 
41const COMMAND_TIMEOUTS = {
42  emoji: 60_000,
43  migrations: 180_000,
44  seed: 60_000,
45} as const;
46 
47export interface E2eContext {
48  tempDir: string;
49  baseUrl: string;
50  port: number;
51  cloudflareEnv: string;
52  devVarsPath: string;
53  serverProcess: ChildProcess;
54  stdoutLogPath: string;
55  stderrLogPath: string;
56  stdoutStream: WriteStream;
57  stderrStream: WriteStream;
58}
59 
60export interface E2eContextFile {
61  baseUrl: string;
62  tempDir: string;
63  oidcIssuer: string;
64}
65 
66export const TEST_CREDENTIALS = {
67  email: E2E_BASELINE_EMAIL,
68  name: E2E_BASELINE_NAME,
69  tesseraSub: E2E_BASELINE_TESSERA_SUB,
70  workspaceSlug: WORKSPACE_SLUG,
71} as const;
72 
73function formatElapsed(startedAt: number): string {
74  return `${((Date.now() - startedAt) / 1000).toFixed(1)}s`;
75}
76 
77function stringifyOutput(value: unknown): string {
78  if (typeof value === "string") return value;
79  if (Buffer.isBuffer(value)) return value.toString("utf8");
80  return "";
81}
82 
83function tailText(value: string, maxBytes = COMMAND_TAIL_BYTES): string {
84  const trimmed = value.trimEnd();
85  if (trimmed.length <= maxBytes) return trimmed;
86  return `... truncated ...\n${trimmed.slice(-maxBytes)}`;
87}
88 
89async function readLogTail(filePath: string, lineCount = LOG_TAIL_LINES): Promise<string> {
90  try {
91    const content = await readFile(filePath, "utf8");
92    return content.trim().split("\n").slice(-lineCount).join("\n");
93  } catch {
94    return "";
95  }
96}
97 
98async function formatLogTails(stdoutLogPath: string, stderrLogPath: string): Promise<string> {
99  const [stdoutTail, stderrTail] = await Promise.all([readLogTail(stdoutLogPath), readLogTail(stderrLogPath)]);
100  const sections: string[] = [];
101  if (stdoutTail) sections.push(`Last stdout log lines:\n${stdoutTail}`);
102  if (stderrTail) sections.push(`Last stderr log lines:\n${stderrTail}`);
103  return sections.join("\n\n");
104}
105 
106const closeWriteStream = async (stream: WriteStream): Promise<void> =>
107  await new Promise<void>((resolveClose, reject) => {
108    if (stream.destroyed || stream.closed || stream.writableFinished) {
109      resolveClose();
110      return;
111    }
112    try {
113      stream.end((error?: Error | null) => {
114        if (error) {
115          reject(error);
116          return;
117        }
118        resolveClose();
119      });
120    } catch (error) {
121      reject(error);
122    }
123  });
124 
125export const getFreePort = async (): Promise<number> =>
126  await new Promise<number>((resolvePort, reject) => {
127    const server = net.createServer();
128    server.unref();
129    server.once("error", reject);
130    server.listen(0, "127.0.0.1", () => {
131      const address = server.address();
132      if (!address || typeof address === "string") {
133        server.close(() => reject(new Error("Failed to allocate a localhost port.")));
134        return;
135      }
136      server.close((error?: Error | null) => {
137        if (error) {
138          reject(error);
139          return;
140        }
141        resolvePort(address.port);
142      });
143    });
144  });
145 
146const execCommand = async (
147  label: string,
148  command: string,
149  args: string[],
150  env: NodeJS.ProcessEnv,
151  timeoutMs: number,
152): Promise<{ stdout: string; stderr: string }> => {
153  const startedAt = Date.now();
154  console.log(`[e2e] ${label} started`);
155 
156  try {
157    const result = await execFile(command, args, {
158      cwd: REPO_ROOT,
159      env,
160      killSignal: "SIGTERM",
161      maxBuffer: COMMAND_STDIO_MAX_BYTES,
162      timeout: timeoutMs,
163    });
164    console.log(`[e2e] ${label} completed in ${formatElapsed(startedAt)}`);
165    return result;
166  } catch (error) {
167    const commandError = error as Error & {
168      code?: number | string | null;
169      killed?: boolean;
170      signal?: NodeJS.Signals | null;
171      stderr?: string | Buffer;
172      stdout?: string | Buffer;
173      timedOut?: boolean;
174    };
175    const stdoutTail = tailText(stringifyOutput(commandError.stdout));
176    const stderrTail = tailText(stringifyOutput(commandError.stderr));
177    const details = [
178      `${label} failed after ${formatElapsed(startedAt)}`,
179      `command: ${command} ${args.join(" ")}`,
180      `code: ${String(commandError.code ?? "unknown")}`,
181      `signal: ${String(commandError.signal ?? "none")}`,
182      `killed: ${String(commandError.killed ?? false)}`,
183      `message: ${commandError.message}`,
184      stdoutTail ? `stdout tail:\n${stdoutTail}` : null,
185      stderrTail ? `stderr tail:\n${stderrTail}` : null,
186    ].filter(Boolean);
187    throw new Error(details.join("\n"), { cause: error });
188  }
189};
190 
191const execWrangler = async (
192  label: string,
193  args: string[],
194  persistTo: string,
195  timeoutMs: number,
196): Promise<{ stdout: string; stderr: string }> =>
197  await execCommand(
198    label,
199    process.execPath,
200    [WRANGLER_BIN_PATH, ...args],
201    { ...process.env, BLAND_PERSIST_STATE_PATH: persistTo, CI: "1", NO_D1_WARNING: "true" },
202    timeoutMs,
203  );
204 
205export const generateEmojiData = async (): Promise<void> => {
206  await execCommand(
207    "emoji generation",
208    process.execPath,
209    ["--import", "tsx", EMOJI_GENERATOR_PATH],
210    { ...process.env, CI: "1" },
211    COMMAND_TIMEOUTS.emoji,
212  );
213};
214 
215const waitForServerReady = async (
216  baseUrl: string,
217  logs: { stdoutLogPath: string; stderrLogPath: string },
218): Promise<void> => {
219  const startedAt = Date.now();
220  const deadline = Date.now() + SERVER_READY_TIMEOUT_MS;
221  let lastStatus: number | null = null;
222  let lastError: string | null = null;
223 
224  while (Date.now() < deadline) {
225    try {
226      const status = await fetch(new URL("/api/v1/health", baseUrl), {
227        signal: AbortSignal.timeout(5_000),
228      }).then((r) => r.status);
229      lastStatus = status;
230      lastError = null;
231      if (status === 200) {
232        console.log(`[e2e] dev server ready in ${formatElapsed(startedAt)}`);
233        return;
234      }
235    } catch (error) {
236      lastError = error instanceof Error ? error.message : String(error);
237    }
238    await sleep(POLL_INTERVAL_MS);
239  }
240 
241  const logTails = await formatLogTails(logs.stdoutLogPath, logs.stderrLogPath);
242  const details = [
243    `dev server readiness timed out after ${formatElapsed(startedAt)}`,
244    `health URL: ${new URL("/api/v1/health", baseUrl).toString()}`,
245    `last status: ${lastStatus ?? "none"}`,
246    `last error: ${lastError ?? "none"}`,
247    logTails || null,
248  ].filter(Boolean);
249  throw new Error(details.join("\n\n"));
250};
251 
252async function withStartupLogTails(error: unknown, logs: { stdoutLogPath: string; stderrLogPath: string }) {
253  const baseMessage = error instanceof Error ? error.message : String(error);
254  const logTails = await formatLogTails(logs.stdoutLogPath, logs.stderrLogPath);
255  return new Error(logTails ? `${baseMessage}\n\n${logTails}` : baseMessage, { cause: error });
256}
257 
258async function waitForProcessExit(serverProcess: ChildProcess, timeoutMs: number): Promise<boolean> {
259  if (serverProcess.exitCode !== null || serverProcess.signalCode !== null) return true;
260 
261  return await new Promise<boolean>((resolveExit) => {
262    function onExit() {
263      clearTimeout(timer);
264      resolveExit(true);
265    }
266 
267    const timer = setTimeout(() => {
268      serverProcess.off("exit", onExit);
269      resolveExit(false);
270    }, timeoutMs);
271 
272    serverProcess.once("exit", onExit);
273  });
274}
275 
276function waitForProcessClose(serverProcess: ChildProcess): Promise<void> {
277  return new Promise<void>((resolveClose) => {
278    serverProcess.once("close", () => resolveClose());
279  });
280}
281 
282async function waitForProcessCloseWithTimeout(closePromise: Promise<void>, timeoutMs: number): Promise<boolean> {
283  return await Promise.race([closePromise.then(() => true), sleep(timeoutMs).then(() => false)]);
284}
285 
286async function loadDevVarsTemplate(): Promise<string> {
287  try {
288    return await readFile(DEV_VARS_EXAMPLE_PATH, "utf8");
289  } catch (error) {
290    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
291  }
292 
293  throw new Error("Missing .dev.vars.example");
294}
295 
296async function createE2eDevVarsFile(
297  baseUrl: string,
298  oidcIssuer: string,
299): Promise<{ cloudflareEnv: string; devVarsPath: string }> {
300  const cloudflareEnv = `e2e-${process.pid}-${Date.now()}`;
301  const devVarsPath = `${DEV_VARS_PATH}.${cloudflareEnv}`;
302  const template = await loadDevVarsTemplate();
303  const allowedOrigins = new URL(baseUrl).origin;
304  const stripped = template
305    .replace(/^ALLOWED_ORIGINS=.*$/gm, "")
306    .replace(/^BLAND_AI_MODE=.*$/gm, "")
307    .replace(/^TESSERA_OIDC_ISSUER=.*$/gm, "")
308    .replace(/^TESSERA_OIDC_CLIENT_ID=.*$/gm, "")
309    .replace(/^TESSERA_OIDC_CLIENT_SECRET=.*$/gm, "")
310    .trimEnd();
311  const content =
312    `${stripped}\n` +
313    `ALLOWED_ORIGINS=${allowedOrigins}\n` +
314    `BLAND_AI_MODE=mock\n` +
315    `TESSERA_OIDC_ISSUER=${oidcIssuer}\n` +
316    `TESSERA_OIDC_CLIENT_ID=${TEST_OIDC_CLIENT_ID}\n` +
317    `TESSERA_OIDC_CLIENT_SECRET=${TEST_OIDC_CLIENT_SECRET}\n`;
318 
319  await writeFile(devVarsPath, content);
320 
321  return { cloudflareEnv, devVarsPath };
322}
323 
324function escapeSql(value: string): string {
325  return value.replace(/'/g, "''");
326}
327 
328function generateUlid(): string {
329  const timestamp = Date.now();
330  const encodingChars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
331  let result = "";
332  let t = timestamp;
333  for (let i = 9; i >= 0; i--) {
334    result = encodingChars[t % 32] + result;
335    t = Math.floor(t / 32);
336  }
337  const random = new Uint8Array(10);
338  crypto.getRandomValues(random);
339  for (let i = 0; i < 10; i++) {
340    result += encodingChars[random[i] % 32];
341  }
342  return result;
343}
344 
345export const applyMigrations = async (persistTo: string): Promise<void> => {
346  await execWrangler(
347    "D1 migrations",
348    ["d1", "migrations", "apply", D1_DATABASE, "--local", "--persist-to", persistTo],
349    persistTo,
350    COMMAND_TIMEOUTS.migrations,
351  );
352};
353 
354export const seedTestUser = async (persistTo: string): Promise<void> => {
355  const userId = generateUlid();
356  const workspaceId = generateUlid();
357  const now = new Date().toISOString();
358 
359  const sql = [
360    `INSERT INTO users (id, email, name, created_at, updated_at) VALUES ('${escapeSql(userId)}', '${escapeSql(E2E_BASELINE_EMAIL)}', '${escapeSql(E2E_BASELINE_NAME)}', '${escapeSql(now)}', '${escapeSql(now)}');`,
361    `INSERT INTO tessera_identities (sub, user_id, created_at, last_seen_at) VALUES ('${escapeSql(E2E_BASELINE_TESSERA_SUB)}', '${escapeSql(userId)}', '${escapeSql(now)}', '${escapeSql(now)}');`,
362    `INSERT INTO workspaces (id, name, slug, owner_id, created_at) VALUES ('${escapeSql(workspaceId)}', '${escapeSql(WORKSPACE_NAME)}', '${escapeSql(WORKSPACE_SLUG)}', '${escapeSql(userId)}', '${escapeSql(now)}');`,
363    `INSERT INTO memberships (user_id, workspace_id, role, joined_at) VALUES ('${escapeSql(userId)}', '${escapeSql(workspaceId)}', 'owner', '${escapeSql(now)}');`,
364  ].join("\n");
365 
366  await execWrangler(
367    "D1 seed user",
368    ["d1", "execute", D1_DATABASE, "--local", "--persist-to", persistTo, "--yes", "--command", sql],
369    persistTo,
370    COMMAND_TIMEOUTS.seed,
371  );
372};
373 
374function isPortInUseStartupError(error: unknown): boolean {
375  const message = error instanceof Error ? error.message : String(error);
376  return message.includes("is already in use") || message.includes("EADDRINUSE");
377}
378 
379export const startDevServer = async (persistTo: string, oidcIssuer: string): Promise<E2eContext> => {
380  let lastError: unknown = null;
381 
382  for (let attempt = 1; attempt <= SERVER_START_ATTEMPTS; attempt += 1) {
383    try {
384      return await startDevServerOnce(persistTo, oidcIssuer, attempt);
385    } catch (error) {
386      lastError = error;
387      if (!isPortInUseStartupError(error) || attempt === SERVER_START_ATTEMPTS) {
388        throw error;
389      }
390      console.warn(`[e2e] dev server port was busy on attempt ${attempt}; retrying with a fresh port`);
391      await sleep(250);
392    }
393  }
394 
395  throw lastError instanceof Error ? lastError : new Error(String(lastError));
396};
397 
398const startDevServerOnce = async (persistTo: string, oidcIssuer: string, attempt: number): Promise<E2eContext> => {
399  const port = await getFreePort();
400  const baseUrl = `http://127.0.0.1:${port}`;
401  const { cloudflareEnv, devVarsPath } = await createE2eDevVarsFile(baseUrl, oidcIssuer);
402  const stdoutLogPath = join(persistTo, "dev.stdout.log");
403  const stderrLogPath = join(persistTo, "dev.stderr.log");
404  const stdoutStream = createWriteStream(stdoutLogPath, { flags: "a" });
405  const stderrStream = createWriteStream(stderrLogPath, { flags: "a" });
406 
407  console.log(`[e2e] dev server startup attempt ${attempt}/${SERVER_START_ATTEMPTS} on ${baseUrl}`);
408 
409  const serverProcess = spawn(
410    process.execPath,
411    [VITE_BIN_PATH, "dev", "--host", "127.0.0.1", "--port", String(port), "--strictPort"],
412    {
413      cwd: REPO_ROOT,
414      env: { ...process.env, BLAND_PERSIST_STATE_PATH: persistTo, CLOUDFLARE_ENV: cloudflareEnv },
415      stdio: ["ignore", "pipe", "pipe"],
416      detached: process.platform !== "win32",
417    },
418  );
419 
420  serverProcess.stdout?.pipe(stdoutStream);
421  serverProcess.stderr?.pipe(stderrStream);
422  const closePromise = waitForProcessClose(serverProcess);
423 
424  let removeExitListener = () => {};
425  try {
426    const exitedEarly = new Promise<never>((_, reject) => {
427      const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
428        reject(new Error(`dev server exited before readiness (code=${code}, signal=${signal})`));
429      };
430      serverProcess.once("exit", onExit);
431      removeExitListener = () => serverProcess.off("exit", onExit);
432    });
433 
434    await Promise.race([waitForServerReady(baseUrl, { stdoutLogPath, stderrLogPath }), exitedEarly]);
435    removeExitListener();
436  } catch (error) {
437    removeExitListener();
438    await stopDevServer(serverProcess);
439    await waitForProcessCloseWithTimeout(closePromise, PROCESS_CLOSE_TIMEOUT_MS);
440    serverProcess.stdout?.unpipe(stdoutStream);
441    serverProcess.stderr?.unpipe(stderrStream);
442    await closeWriteStream(stdoutStream);
443    await closeWriteStream(stderrStream);
444    const startupError = await withStartupLogTails(error, { stdoutLogPath, stderrLogPath });
445    await rm(devVarsPath, { force: true });
446    throw startupError;
447  }
448 
449  return {
450    tempDir: persistTo,
451    baseUrl,
452    port,
453    cloudflareEnv,
454    devVarsPath,
455    serverProcess,
456    stdoutLogPath,
457    stderrLogPath,
458    stdoutStream,
459    stderrStream,
460  };
461};
462 
463export const stopDevServer = async (serverProcess: ChildProcess): Promise<void> => {
464  if (serverProcess.exitCode !== null || serverProcess.signalCode !== null) return;
465 
466  const killServer = (signal: NodeJS.Signals): boolean => {
467    if (serverProcess.pid === undefined) return false;
468    try {
469      if (process.platform === "win32") {
470        return serverProcess.kill(signal);
471      }
472      process.kill(-serverProcess.pid, signal);
473      return true;
474    } catch (error) {
475      if ((error as NodeJS.ErrnoException).code === "ESRCH") return false;
476      throw error;
477    }
478  };
479 
480  const termWait = waitForProcessExit(serverProcess, PROCESS_TERM_TIMEOUT_MS);
481  const termSent = killServer("SIGTERM");
482  if (!termSent || (await termWait)) return;
483 
484  if (serverProcess.exitCode === null && serverProcess.signalCode === null) {
485    const killWait = waitForProcessExit(serverProcess, PROCESS_KILL_TIMEOUT_MS);
486    const killSent = killServer("SIGKILL");
487    if (killSent && !(await killWait)) {
488      console.error(`[e2e] dev server did not exit after SIGKILL within ${PROCESS_KILL_TIMEOUT_MS}ms`);
489    }
490  }
491};
492 
493export const closeContextLogs = async (ctx: E2eContext): Promise<void> => {
494  ctx.serverProcess.stdout?.unpipe(ctx.stdoutStream);
495  ctx.serverProcess.stderr?.unpipe(ctx.stderrStream);
496  await closeWriteStream(ctx.stdoutStream);
497  await closeWriteStream(ctx.stderrStream);
498};
499 
500export const printFailureContext = (ctx: E2eContext): void => {
501  console.error(`Preserved temp state: ${ctx.tempDir}`);
502  console.error(`Cloudflare env vars: ${ctx.devVarsPath}`);
503  console.error(`Dev server stdout: ${ctx.stdoutLogPath}`);
504  console.error(`Dev server stderr: ${ctx.stderrLogPath}`);
505  console.error(
506    `Reopen preserved state: CLOUDFLARE_ENV=${ctx.cloudflareEnv} BLAND_PERSIST_STATE_PATH=${ctx.tempDir} node ${VITE_BIN_PATH} dev --host 127.0.0.1 --port ${ctx.port} --strictPort`,
507  );
508};
509 
510export const printLogTails = async (ctx: E2eContext): Promise<void> => {
511  for (const [label, filePath] of [
512    ["stdout", ctx.stdoutLogPath],
513    ["stderr", ctx.stderrLogPath],
514  ] as const) {
515    const tail = await readLogTail(filePath);
516    if (tail.length > 0) {
517      console.error(`Last ${label} log lines:\n${tail}`);
518    }
519  }
520};