1
import { env } from "cloudflare:workers";
2
import { describe, expect, it, vi } from "vitest";
3
 
4
import { BranchName } from "@/contracts";
5
import * as sidecarState from "@/worker/durable/project-do/sidecar-state";
6
 
7
import {
8
  acceptManualRunWithoutAlarm,
9
  claimRunWorkWithoutAlarm,
10
  expectAcceptedManualRun,
11
  finalizeRunExecutionWithoutAlarm,
12
  requestRunCancelWithoutAlarm,
13
} from "../../helpers/project-do";
14
import { registerWorkerRuntimeHooks } from "../../helpers/worker-hooks";
15
import { readProjectDoRows, seedProject, seedUser } from "../../helpers/runtime";
16
 
17
describe("ProjectDO transition invariants", () => {
18
  registerWorkerRuntimeHooks();
19
 
20
  describe("queue ordering and capacity", () => {
21
    it("keeps FIFO order and a single active run per project", async () => {
22
      const user = await seedUser();
23
      const project = await seedProject(user, {
24
        dispatchMode: "queue",
25
      });
26
      const stub = env.PROJECT_DO.getByName(project.id);
27
 
28
      const firstAccepted = await acceptManualRunWithoutAlarm(stub, {
29
        projectId: project.id,
30
        triggeredByUserId: user.id,
31
        branch: project.defaultBranch,
32
      });
33
      const secondAccepted = await acceptManualRunWithoutAlarm(stub, {
34
        projectId: project.id,
35
        triggeredByUserId: user.id,
36
        branch: BranchName.assertDecode("release"),
37
      });
38
 
39
      expect(firstAccepted.executable).toBe(true);
40
      expect(secondAccepted.executable).toBe(false);
41
 
42
      const initialState = await stub.getProjectDetailState(project.id);
43
      expect(initialState.activeRunId).toBeNull();
44
      expect(initialState.pendingRuns.map((run) => run.runId)).toEqual([firstAccepted.runId, secondAccepted.runId]);
45
 
46
      const firstClaim = await claimRunWorkWithoutAlarm(stub, {
47
        projectId: project.id,
48
        runId: firstAccepted.runId,
49
      });
50
      expect(firstClaim.kind).toBe("execute");
51
 
52
      const duplicateClaim = await claimRunWorkWithoutAlarm(stub, {
53
        projectId: project.id,
54
        runId: firstAccepted.runId,
55
      });
56
      expect(duplicateClaim).toEqual({
57
        kind: "stale",
58
        reason: "run_active",
59
      });
60
 
61
      const supersededClaim = await claimRunWorkWithoutAlarm(stub, {
62
        projectId: project.id,
63
        runId: secondAccepted.runId,
64
      });
65
      expect(supersededClaim).toEqual({
66
        kind: "stale",
67
        reason: "superseded",
68
      });
69
 
70
      await finalizeRunExecutionWithoutAlarm(stub, {
71
        projectId: project.id,
72
        runId: firstAccepted.runId,
73
        terminalStatus: "passed",
74
        lastError: null,
75
      });
76
 
77
      const afterFinalize = await stub.getProjectDetailState(project.id);
78
      expect(afterFinalize.activeRunId).toBeNull();
79
      expect(afterFinalize.pendingRuns.map((run) => run.runId)).toEqual([secondAccepted.runId]);
80
 
81
      const secondClaim = await claimRunWorkWithoutAlarm(stub, {
82
        projectId: project.id,
83
        runId: secondAccepted.runId,
84
      });
85
      expect(secondClaim.kind).toBe("execute");
86
 
87
      const rows = await readProjectDoRows(project.id);
88
      expect(rows.state?.activeRunId).toBe(secondAccepted.runId);
89
      expect(rows.runs.map((row) => ({ runId: row.runId, status: row.status }))).toEqual([
90
        { runId: firstAccepted.runId, status: "passed" },
91
        { runId: secondAccepted.runId, status: "active" },
92
      ]);
93
    });
94
 
95
    it("returns a rejected result when the per-project queue is full", async () => {
96
      const user = await seedUser({
97
        email: "queue-full@example.com",
98
        slug: "queue-full-user",
99
      });
100
      const project = await seedProject(user, {
101
        projectSlug: "queue-full-project",
102
        dispatchMode: "queue",
103
      });
104
      const stub = env.PROJECT_DO.getByName(project.id);
105
 
106
      for (let index = 0; index < 20; index += 1) {
107
        await acceptManualRunWithoutAlarm(stub, {
108
          projectId: project.id,
109
          triggeredByUserId: user.id,
110
          branch: project.defaultBranch,
111
        });
112
      }
113
 
114
      const overflow = await stub.acceptManualRun({
115
        projectId: project.id,
116
        triggeredByUserId: user.id,
117
        branch: BranchName.assertDecode("overflow"),
118
      });
119
 
120
      expect(overflow).toEqual({
121
        kind: "rejected",
122
        reason: "queue_full",
123
      });
124
 
125
      const rows = await readProjectDoRows(project.id);
126
      expect(rows.runs).toHaveLength(20);
127
    });
128
  });
129
 
130
  describe("terminal invariants", () => {
131
    it("rejects active to canceled finalization", async () => {
132
      const user = await seedUser({
133
        email: "active-canceled@example.com",
134
        slug: "active-canceled-user",
135
      });
136
      const project = await seedProject(user, {
137
        projectSlug: "active-canceled-project",
138
      });
139
      const projectStub = env.PROJECT_DO.getByName(project.id);
140
 
141
      const accepted = await acceptManualRunWithoutAlarm(projectStub, {
142
        projectId: project.id,
143
        triggeredByUserId: user.id,
144
        branch: project.defaultBranch,
145
      });
146
      const claim = await claimRunWorkWithoutAlarm(projectStub, {
147
        projectId: project.id,
148
        runId: accepted.runId,
149
      });
150
      expect(claim.kind).toBe("execute");
151
 
152
      await expect(
153
        finalizeRunExecutionWithoutAlarm(projectStub, {
154
          projectId: project.id,
155
          runId: accepted.runId,
156
          terminalStatus: "canceled",
157
          lastError: null,
158
        }),
159
      ).rejects.toThrow(`Run ${accepted.runId} cannot transition from active to canceled.`);
160
 
161
      const rows = await readProjectDoRows(project.id);
162
      expect(rows.state?.activeRunId).toBe(accepted.runId);
163
      expect(rows.runs[0]?.status).toBe("active");
164
    });
165
 
166
    it("rejects cancel_requested to passed finalization", async () => {
167
      const user = await seedUser({
168
        email: "cancel-requested-passed@example.com",
169
        slug: "cancel-requested-passed-user",
170
      });
171
      const project = await seedProject(user, {
172
        projectSlug: "cancel-requested-passed-project",
173
      });
174
      const projectStub = env.PROJECT_DO.getByName(project.id);
175
 
176
      const accepted = await acceptManualRunWithoutAlarm(projectStub, {
177
        projectId: project.id,
178
        triggeredByUserId: user.id,
179
        branch: project.defaultBranch,
180
      });
181
      const claim = await claimRunWorkWithoutAlarm(projectStub, {
182
        projectId: project.id,
183
        runId: accepted.runId,
184
      });
185
      expect(claim.kind).toBe("execute");
186
 
187
      const cancelResult = await requestRunCancelWithoutAlarm(projectStub, {
188
        projectId: project.id,
189
        runId: accepted.runId,
190
      });
191
      expect(cancelResult.status).toBe("cancel_requested");
192
 
193
      await expect(
194
        finalizeRunExecutionWithoutAlarm(projectStub, {
195
          projectId: project.id,
196
          runId: accepted.runId,
197
          terminalStatus: "passed",
198
          lastError: null,
199
        }),
200
      ).rejects.toThrow(`Run ${accepted.runId} cannot transition from cancel_requested to passed.`);
201
 
202
      const rows = await readProjectDoRows(project.id);
203
      expect(rows.state?.activeRunId).toBe(accepted.runId);
204
      expect(rows.runs[0]?.status).toBe("cancel_requested");
205
    });
206
 
207
    it("treats duplicate terminal finalization with the same status as idempotent", async () => {
208
      const user = await seedUser({
209
        email: "duplicate-terminal@example.com",
210
        slug: "duplicate-terminal-user",
211
      });
212
      const project = await seedProject(user, {
213
        projectSlug: "duplicate-terminal-project",
214
      });
215
      const projectStub = env.PROJECT_DO.getByName(project.id);
216
 
217
      const accepted = await acceptManualRunWithoutAlarm(projectStub, {
218
        projectId: project.id,
219
        triggeredByUserId: user.id,
220
        branch: project.defaultBranch,
221
      });
222
      const claim = await claimRunWorkWithoutAlarm(projectStub, {
223
        projectId: project.id,
224
        runId: accepted.runId,
225
      });
226
      expect(claim.kind).toBe("execute");
227
 
228
      await finalizeRunExecutionWithoutAlarm(projectStub, {
229
        projectId: project.id,
230
        runId: accepted.runId,
231
        terminalStatus: "failed",
232
        lastError: "checkout_failed",
233
      });
234
 
235
      await expect(
236
        finalizeRunExecutionWithoutAlarm(projectStub, {
237
          projectId: project.id,
238
          runId: accepted.runId,
239
          terminalStatus: "failed",
240
          lastError: "checkout_failed",
241
        }),
242
      ).resolves.toMatchObject({
243
        snapshot: {
244
          runId: accepted.runId,
245
        },
246
      });
247
 
248
      const rows = await readProjectDoRows(project.id);
249
      expect(rows.state?.activeRunId).toBeNull();
250
      expect(rows.runs[0]?.status).toBe("failed");
251
    });
252
 
253
    it("rejects duplicate terminal finalization when the status changes", async () => {
254
      const user = await seedUser({
255
        email: "mismatched-terminal@example.com",
256
        slug: "mismatched-terminal-user",
257
      });
258
      const project = await seedProject(user, {
259
        projectSlug: "mismatched-terminal-project",
260
      });
261
      const projectStub = env.PROJECT_DO.getByName(project.id);
262
 
263
      const accepted = await acceptManualRunWithoutAlarm(projectStub, {
264
        projectId: project.id,
265
        triggeredByUserId: user.id,
266
        branch: project.defaultBranch,
267
      });
268
      const claim = await claimRunWorkWithoutAlarm(projectStub, {
269
        projectId: project.id,
270
        runId: accepted.runId,
271
      });
272
      expect(claim.kind).toBe("execute");
273
 
274
      await finalizeRunExecutionWithoutAlarm(projectStub, {
275
        projectId: project.id,
276
        runId: accepted.runId,
277
        terminalStatus: "failed",
278
        lastError: "checkout_failed",
279
      });
280
 
281
      await expect(
282
        finalizeRunExecutionWithoutAlarm(projectStub, {
283
          projectId: project.id,
284
          runId: accepted.runId,
285
          terminalStatus: "passed",
286
          lastError: null,
287
        }),
288
      ).rejects.toThrow(`Run ${accepted.runId} is already terminal in status failed, cannot finalize as passed.`);
289
 
290
      const rows = await readProjectDoRows(project.id);
291
      expect(rows.runs[0]?.status).toBe("failed");
292
    });
293
  });
294
 
295
  describe("cancellation and sidecar invariants", () => {
296
    it("cancels executable runs immediately and promotes the next pending run", async () => {
297
      const user = await seedUser({
298
        email: "cancel@example.com",
299
        slug: "cancel-user",
300
      });
301
      const project = await seedProject(user, {
302
        projectSlug: "cancel-project",
303
        dispatchMode: "queue",
304
      });
305
      const stub = env.PROJECT_DO.getByName(project.id);
306
 
307
      const firstAccepted = await acceptManualRunWithoutAlarm(stub, {
308
        projectId: project.id,
309
        triggeredByUserId: user.id,
310
        branch: project.defaultBranch,
311
      });
312
      const secondAccepted = await acceptManualRunWithoutAlarm(stub, {
313
        projectId: project.id,
314
        triggeredByUserId: user.id,
315
        branch: BranchName.assertDecode("feature-x"),
316
      });
317
 
318
      const cancelResult = await requestRunCancelWithoutAlarm(stub, {
319
        projectId: project.id,
320
        runId: firstAccepted.runId,
321
      });
322
      expect(cancelResult.status).toBe("canceled");
323
 
324
      const nextClaim = await claimRunWorkWithoutAlarm(stub, {
325
        projectId: project.id,
326
        runId: secondAccepted.runId,
327
      });
328
      expect(nextClaim.kind).toBe("execute");
329
 
330
      const canceledClaim = await claimRunWorkWithoutAlarm(stub, {
331
        projectId: project.id,
332
        runId: firstAccepted.runId,
333
      });
334
      expect(canceledClaim).toEqual({
335
        kind: "stale",
336
        reason: "canceled",
337
      });
338
    });
339
 
340
    it("does not fail claimRunWork after the SQLite commit when heartbeat sidecar writes fail", async () => {
341
      const user = await seedUser({
342
        email: "sidecar-heartbeat@example.com",
343
        slug: "sidecar-heartbeat-user",
344
      });
345
      const project = await seedProject(user, {
346
        projectSlug: "sidecar-heartbeat-project",
347
      });
348
      const projectStub = env.PROJECT_DO.getByName(project.id);
349
 
350
      const accepted = expectAcceptedManualRun(
351
        await projectStub.acceptManualRun({
352
          projectId: project.id,
353
          triggeredByUserId: user.id,
354
          branch: project.defaultBranch,
355
        }),
356
      );
357
      const heartbeatSpy = vi
358
        .spyOn(sidecarState, "setHeartbeatAt")
359
        .mockRejectedValueOnce(new Error("heartbeat failed"));
360
 
361
      try {
362
        const claim = await projectStub.claimRunWork({
363
          projectId: project.id,
364
          runId: accepted.runId,
365
        });
366
 
367
        expect(claim.kind).toBe("execute");
368
 
369
        const rows = await readProjectDoRows(project.id);
370
        expect(rows.state?.activeRunId).toBe(accepted.runId);
371
      } finally {
372
        heartbeatSpy.mockRestore();
373
      }
374
    });
375
  });
376
});