1
/**
2
 * Fixture-based validation for the streaming pack indexer.
3
 *
4
 * Validates the indexer against the real 42 MiB fixture pack in
5
 * uncommitted-fixture/ when present. Skipped when the fixture is not available.
6
 *
7
 * Fixture files are read from disk via the FIXTURE_READER service binding
8
 * (defined in vitest.config.ts), which runs in Node.js and returns binary data.
9
 * This avoids the Vite SSR externalization issue with node:fs.
10
 *
11
 * The fixture pack is uploaded to R2 via FixedLengthStream because the local
12
 * harness needs a known-length stream for R2.put(). Real client uploads may
13
 * still arrive chunked without Content-Length.
14
 */
15
 
16
import { describe, expect, it } from "vitest";
17
import { env } from "cloudflare:workers";
18
import { scanPack, resolveDeltasAndWriteIdx } from "@/worker/git/pack/indexer";
19
import { SubrequestLimiter, DEFAULT_SUBREQUEST_BUDGET } from "@/worker/git/operations/limits";
20
import { createLogger } from "@/worker/common/logger";
21
import { packIndexKey } from "@/worker/keys";
22
import { buildPack, buildAppendOnlyDelta, makeCommit, makeTree } from "./util/git-pack";
23
import { computeOid } from "@/worker/git/core/objects";
24
import { bytesToHex } from "@/worker/common/hex";
25
 
26
const log = createLogger("debug", { service: "PackIndexerFixture" });
27
const fixtureEnv = env as unknown as {
28
  FIXTURE_READER: { fetch(req: Request): Promise<Response> };
29
  PACK_INDEXER_FIXTURE?: string;
30
};
31
 
32
const FIXTURE_PACK_NAME = "pack-395a180893e59dad8ef9d7fa135ecd8b1b399bb1";
33
const FIXTURE_PACK_KEY = `test/fixture/${FIXTURE_PACK_NAME}.pack`;
34
 
35
/**
36
 * Read a fixture file from disk via the FIXTURE_READER service binding.
37
 * Returns null if the file doesn't exist.
38
 */
39
async function readFixtureFile(relativePath: string): Promise<Uint8Array | null> {
40
  const resp = await fixtureEnv.FIXTURE_READER.fetch(
41
    new Request(`http://fixture/${encodeURIComponent(relativePath)}`)
42
  );
43
  if (!resp.ok) return null;
44
  return new Uint8Array(await resp.arrayBuffer());
45
}
46
 
47
describe("pack indexer fixture validation", () => {
48
  it("indexes a multi-object pack with mixed types and deltas", { timeout: 30_000 }, async () => {
49
    const blob1 = new TextEncoder().encode("file one content version 1\n");
50
    const blob2 = new TextEncoder().encode("file two content version 1\n");
51
    const blob1v2Suffix = new TextEncoder().encode("modified line\n");
52
    const delta1 = buildAppendOnlyDelta(blob1, blob1v2Suffix);
53
 
54
    const tree = await makeTree();
55
    const commit1 = await makeCommit(tree.oid, "first commit");
56
    const commit2Payload = new TextEncoder().encode(
57
      `tree ${tree.oid}\nparent ${commit1.oid}\nauthor You <you@example.com> 0 +0000\ncommitter You <you@example.com> 0 +0000\n\nsecond commit\n`
58
    );
59
 
60
    const packBytes = await buildPack([
61
      { type: "blob", payload: blob1 },
62
      { type: "blob", payload: blob2 },
63
      { type: "ofs-delta", baseIndex: 0, delta: delta1 },
64
      { type: "tree", payload: tree.payload },
65
      { type: "commit", payload: commit1.payload },
66
      { type: "commit", payload: commit2Payload },
67
    ]);
68
 
69
    const packKey = "test/multi-object.pack";
70
    await env.REPO_BUCKET.put(packKey, packBytes);
71
    const head = await env.REPO_BUCKET.head(packKey);
72
 
73
    const counter = { count: 0 };
74
    const limiter = new SubrequestLimiter(6);
75
 
76
    const scanResult = await scanPack({
77
      env,
78
      packKey,
79
      packSize: head!.size,
80
      limiter,
81
      countSubrequest: (n = 1) => {
82
        counter.count += n;
83
      },
84
      log,
85
    });
86
 
87
    expect(scanResult.objectCount).toBe(6);
88
    expect(scanResult.table.resolved.reduce((a: number, b: number) => a + b, 0)).toBe(5);
89
 
90
    const resolveResult = await resolveDeltasAndWriteIdx({
91
      env,
92
      packKey,
93
      packSize: head!.size,
94
      limiter,
95
      countSubrequest: (n = 1) => {
96
        counter.count += n;
97
      },
98
      log,
99
      scanResult,
100
      repoId: "test/multi",
101
    });
102
 
103
    expect(scanResult.table.resolved.reduce((a: number, b: number) => a + b, 0)).toBe(6);
104
    expect(resolveResult.objectCount).toBe(6);
105
 
106
    const expectedPayload = new Uint8Array(blob1.length + blob1v2Suffix.length);
107
    expectedPayload.set(blob1, 0);
108
    expectedPayload.set(blob1v2Suffix, blob1.length);
109
    const expectedOid = await computeOid("blob", expectedPayload);
110
    const deltaOid = bytesToHex(scanResult.table.oids.subarray(2 * 20, 3 * 20));
111
    expect(deltaOid).toBe(expectedOid);
112
 
113
    const idxObj = await env.REPO_BUCKET.get(packIndexKey(packKey));
114
    expect(idxObj).not.toBeNull();
115
    expect(counter.count).toBeLessThan(DEFAULT_SUBREQUEST_BUDGET);
116
  });
117
 
118
  it("generates idx matching fixture byte-for-byte", { timeout: 120_000 }, async () => {
119
    if (fixtureEnv.PACK_INDEXER_FIXTURE !== "1") {
120
      console.log("SKIP: set PACK_INDEXER_FIXTURE=1 to run the 42 MiB fixture validation");
121
      return;
122
    }
123
 
124
    // Read fixture files via the FIXTURE_READER service binding.
125
    const fixturePack = await readFixtureFile(`uncommitted-fixture/${FIXTURE_PACK_NAME}.pack`);
126
    const fixtureIdx = await readFixtureFile(`uncommitted-fixture/${FIXTURE_PACK_NAME}.idx`);
127
    if (!fixturePack || !fixtureIdx) {
128
      console.log("SKIP: fixture not found in uncommitted-fixture/");
129
      return;
130
    }
131
 
132
    log.info("fixture:loaded", {
133
      packBytes: fixturePack.byteLength,
134
      idxBytes: fixtureIdx.byteLength,
135
    });
136
 
137
    // Upload the fixture through FixedLengthStream because the local harness
138
    // needs a known-length stream for R2.put(). The integrated receive path
139
    // still needs to handle real chunked client uploads separately.
140
    const fls = new FixedLengthStream(fixturePack.byteLength);
141
    const flsWriter = fls.writable.getWriter();
142
    const UPLOAD_CHUNK = 512 * 1024;
143
    const writePromise = (async () => {
144
      for (let offset = 0; offset < fixturePack.byteLength; offset += UPLOAD_CHUNK) {
145
        const end = Math.min(offset + UPLOAD_CHUNK, fixturePack.byteLength);
146
        await flsWriter.write(fixturePack.subarray(offset, end));
147
      }
148
      await flsWriter.close();
149
    })();
150
    await Promise.all([env.REPO_BUCKET.put(FIXTURE_PACK_KEY, fls.readable), writePromise]);
151
 
152
    const head = await env.REPO_BUCKET.head(FIXTURE_PACK_KEY);
153
    expect(head).not.toBeNull();
154
    expect(head!.size).toBe(fixturePack.byteLength);
155
 
156
    // Run the indexer.
157
    const counter = { count: 0 };
158
    const limiter = new SubrequestLimiter(6);
159
    const countSub = (n = 1) => {
160
      counter.count += n;
161
    };
162
 
163
    const scanStart = Date.now();
164
    const scanResult = await scanPack({
165
      env,
166
      packKey: FIXTURE_PACK_KEY,
167
      packSize: head!.size,
168
      limiter,
169
      countSubrequest: countSub,
170
      log,
171
    });
172
    const scanMs = Date.now() - scanStart;
173
 
174
    const resolveStart = Date.now();
175
    const resolveResult = await resolveDeltasAndWriteIdx({
176
      env,
177
      packKey: FIXTURE_PACK_KEY,
178
      packSize: head!.size,
179
      limiter,
180
      countSubrequest: countSub,
181
      log,
182
      scanResult,
183
      repoId: "fixture/test",
184
      // 48 MiB: larger packs need more LRU headroom to avoid excessive R2
185
      // re-reads. The array-backed payload cache keeps that extra headroom
186
      // within the worker memory budget for the representative fixture.
187
      lruBudget: 48 * 1024 * 1024,
188
    });
189
    const resolveMs = Date.now() - resolveStart;
190
 
191
    // Compare idx byte-for-byte.
192
    const idxObj = await env.REPO_BUCKET.get(packIndexKey(FIXTURE_PACK_KEY));
193
    expect(idxObj).not.toBeNull();
194
    const generatedIdx = new Uint8Array(await idxObj!.arrayBuffer());
195
 
196
    expect(generatedIdx.byteLength).toBe(fixtureIdx.byteLength);
197
 
198
    let firstMismatch = -1;
199
    for (let i = 0; i < fixtureIdx.byteLength; i++) {
200
      if (generatedIdx[i] !== fixtureIdx[i]) {
201
        firstMismatch = i;
202
        break;
203
      }
204
    }
205
    expect(firstMismatch).toBe(-1);
206
 
207
    // The platform hard cap is 10,000 subrequests. Budget 5,000 for the
208
    // indexer, leaving headroom for connectivity checks and other ops.
209
    expect(counter.count).toBeLessThan(5_000);
210
 
211
    log.info("fixture:stats", {
212
      objectCount: scanResult.objectCount,
213
      scanMs,
214
      resolveMs,
215
      totalMs: scanMs + resolveMs,
216
      subrequests: counter.count,
217
      idxBytes: resolveResult.idxBytes,
218
    });
219
  });
220
});