1
import { describe, expect, it } from "vitest";
2
import { env } from "cloudflare:workers";
3
import { listCommitChangedFiles } from "@/worker/git";
4
import {
5
  hasObjectsBatch,
6
  loadIdxView,
7
  parseIdxView,
8
  readObject,
9
  getNextOffsetByIndex,
10
} from "@/worker/git/object-store";
11
import { computeNeededFast } from "@/worker/git/operations/fetch/neededFast";
12
import { hexToBytes } from "@/worker/common/hex";
13
import {
14
  callStubWithRetry,
15
  deleteLooseObjectCopies,
16
  seedLegacyPackedRepo,
17
  uniqueRepoId,
18
} from "./util/test-helpers";
19
import { setupRepoForTests } from "./util/repoSeed";
20
import { buildPack } from "./util/git-pack";
21
import { buildTreePayload } from "./util/packed-repo";
22
import { createTestCacheContext, seedPackFirstRepo } from "./util/pack-first";
23
import { encodeGitObject } from "@/worker/git/core";
24
 
25
const UINT32_SPAN = 0x1_0000_0000;
26
 
27
function buildSingleEntryIdx(
28
  oidHex: string,
29
  offset: number,
30
  includeLargeOffset = true
31
): Uint8Array {
32
  const count = 1;
33
  const totalSize =
34
    8 + 256 * 4 + count * 20 + count * 4 + count * 4 + (includeLargeOffset ? 8 : 0) + 20 + 20;
35
  const buf = new Uint8Array(totalSize);
36
  const dv = new DataView(buf.buffer);
37
  let pos = 0;
38
 
39
  buf[pos++] = 0xff;
40
  buf[pos++] = 0x74;
41
  buf[pos++] = 0x4f;
42
  buf[pos++] = 0x63;
43
  dv.setUint32(pos, 2, false);
44
  pos += 4;
45
 
46
  const oidBytes = hexToBytes(oidHex);
47
  for (let bucket = 0; bucket < 256; bucket++) {
48
    dv.setUint32(pos, bucket >= oidBytes[0] ? count : 0, false);
49
    pos += 4;
50
  }
51
 
52
  buf.set(oidBytes, pos);
53
  pos += 20;
54
 
55
  dv.setUint32(pos, 0, false);
56
  pos += 4;
57
 
58
  if (includeLargeOffset) {
59
    dv.setUint32(pos, 0x80000000, false);
60
  } else {
61
    dv.setUint32(pos, 0, false);
62
  }
63
  pos += 4;
64
 
65
  if (includeLargeOffset) {
66
    const hi = Math.floor(offset / UINT32_SPAN);
67
    const lo = offset % UINT32_SPAN;
68
    dv.setUint32(pos, hi, false);
69
    dv.setUint32(pos + 4, lo, false);
70
  }
71
 
72
  return buf;
73
}
74
 
75
function makeTracingLimiter(labels: string[]) {
76
  return {
77
    async run<T>(label: string, fn: () => Promise<T>): Promise<T> {
78
      labels.push(label);
79
      return await fn();
80
    },
81
  };
82
}
83
 
84
describe("pack-first read path closure", () => {
85
  it("treats empty batch ref results as resolved without per-object fallback reads", async () => {
86
    const repo = uniqueRepoId("pack-needed-fast-empty-refs");
87
    await setupRepoForTests(env, "o", repo);
88
    const repoId = `o/${repo}`;
89
    const seeded = await seedPackFirstRepo(repoId);
90
    await deleteLooseObjectCopies(env, seeded.getStub, seeded.objectOids);
91
 
92
    const cacheCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`);
93
    const needed = await computeNeededFast(env, repoId, [seeded.nextTree.oid], [], cacheCtx);
94
 
95
    expect(new Set(needed)).toEqual(new Set([seeded.nextTree.oid, seeded.nextBlob.oid]));
96
    expect(cacheCtx.memo?.refs?.get(seeded.nextBlob.oid)).toEqual([]);
97
    expect(cacheCtx.memo?.objects?.has(seeded.nextBlob.oid)).not.toBe(true);
98
  });
99
 
100
  it("keeps loose-only wants as partial results instead of inventing compatibility refs", async () => {
101
    const repo = uniqueRepoId("pack-needed-fast-pack-only");
102
    await setupRepoForTests(env, "o", repo);
103
    const repoId = `o/${repo}`;
104
    const getStub = () => env.REPO_DO.get(env.REPO_DO.idFromName(repoId));
105
    const author = "You <you@example.com> 0 +0000";
106
 
107
    const blobPayload = new TextEncoder().encode("pack only\n");
108
    const blob = await encodeGitObject("blob", blobPayload);
109
    const treePayload = buildTreePayload([{ mode: "100644", name: "README.md", oid: blob.oid }]);
110
    const tree = await encodeGitObject("tree", treePayload);
111
    const commitPayload = new TextEncoder().encode(
112
      `tree ${tree.oid}\n` + `author ${author}\n` + `committer ${author}\n\n` + `pack only commit\n`
113
    );
114
    const commit = await encodeGitObject("commit", commitPayload);
115
 
116
    const packBytes = await buildPack([
117
      { type: "blob", payload: blobPayload },
118
      { type: "tree", payload: treePayload },
119
    ]);
120
 
121
    await seedLegacyPackedRepo({
122
      env,
123
      repoId,
124
      getStub,
125
      packs: [{ name: "pack-needed-fast-pack-only.pack", packBytes }],
126
      refs: [{ name: "refs/heads/main", oid: commit.oid }],
127
      head: { target: "refs/heads/main", oid: commit.oid },
128
      looseObjects: [commit],
129
    });
130
    const cacheCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`);
131
    const needed = await computeNeededFast(env, repoId, [commit.oid], [], cacheCtx);
132
 
133
    expect(needed).toEqual([commit.oid]);
134
    expect(cacheCtx.memo?.refs?.has(commit.oid)).not.toBe(true);
135
  });
136
 
137
  it("coalesces concurrent idx loads and only decrements the request budget once per pack", async () => {
138
    const repo = uniqueRepoId("pack-idx-coalesce");
139
    await setupRepoForTests(env, "o", repo);
140
    const repoId = `o/${repo}`;
141
    const seeded = await seedPackFirstRepo(repoId);
142
    const activeCatalog = await callStubWithRetry(seeded.getStub, (stub) =>
143
      stub.getActivePackCatalog()
144
    );
145
 
146
    const cacheCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`, 10);
147
    cacheCtx.memo = {
148
      ...(cacheCtx.memo || {}),
149
      packCatalog: activeCatalog,
150
    };
151
 
152
    const found = await hasObjectsBatch(
153
      env,
154
      repoId,
155
      [
156
        seeded.baseBlob.oid,
157
        seeded.baseTree.oid,
158
        seeded.baseCommit.oid,
159
        seeded.nextBlob.oid,
160
        seeded.nextTree.oid,
161
        seeded.nextCommit.oid,
162
      ],
163
      cacheCtx
164
    );
165
 
166
    expect(found).toEqual([true, true, true, true, true, true]);
167
    // The idx loader now uses packCatalog.packBytes as the pack-size hint,
168
    // so a cold load for this single seeded pack only spends the idx fetch budget.
169
    expect(cacheCtx.memo?.subreqBudget).toBe(9);
170
  });
171
 
172
  it("reuses hinted idx loads across requests without another R2 fetch", async () => {
173
    const repo = uniqueRepoId("pack-idx-cross-request");
174
    await setupRepoForTests(env, "o", repo);
175
    const repoId = `o/${repo}`;
176
    const seeded = await seedPackFirstRepo(repoId);
177
    const activeCatalog = await callStubWithRetry(seeded.getStub, (stub) =>
178
      stub.getActivePackCatalog()
179
    );
180
    const pack = activeCatalog[0]!;
181
 
182
    const warmLabels: string[] = [];
183
    const warmCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`, 10);
184
    warmCtx.memo = {
185
      ...(warmCtx.memo || {}),
186
      limiter: makeTracingLimiter(warmLabels),
187
    };
188
 
189
    const warmed = await loadIdxView(env, pack.packKey, warmCtx, pack.packBytes);
190
 
191
    expect(warmed).toBeDefined();
192
    expect(warmLabels).toEqual(["r2:get-pack-idx"]);
193
    expect(warmCtx.memo?.subreqBudget).toBe(9);
194
 
195
    const reuseLabels: string[] = [];
196
    const reuseCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`, 10);
197
    reuseCtx.memo = {
198
      ...(reuseCtx.memo || {}),
199
      limiter: makeTracingLimiter(reuseLabels),
200
    };
201
 
202
    const reused = await loadIdxView(env, pack.packKey, reuseCtx, pack.packBytes);
203
 
204
    expect(reused).toBeDefined();
205
    expect(reuseLabels).toEqual([]);
206
    expect(reuseCtx.memo?.subreqBudget).toBe(10);
207
  });
208
 
209
  it("reads a packed base object with one coalesced range read", async () => {
210
    const repo = uniqueRepoId("pack-read-single-range");
211
    await setupRepoForTests(env, "o", repo);
212
    const repoId = `o/${repo}`;
213
    const seeded = await seedPackFirstRepo(repoId);
214
    await deleteLooseObjectCopies(env, seeded.getStub, seeded.objectOids);
215
    const activeCatalog = await callStubWithRetry(seeded.getStub, (stub) =>
216
      stub.getActivePackCatalog()
217
    );
218
 
219
    const labels: string[] = [];
220
    const cacheCtx = createTestCacheContext(
221
      `https://example.com/${repoId}/blob/${seeded.nextBlob.oid}`,
222
      20
223
    );
224
    cacheCtx.memo = {
225
      ...(cacheCtx.memo || {}),
226
      packCatalog: activeCatalog,
227
      limiter: makeTracingLimiter(labels),
228
    };
229
 
230
    const obj = await readObject(env, repoId, seeded.nextBlob.oid, cacheCtx);
231
 
232
    expect(obj?.type).toBe("blob");
233
    expect(labels.filter((label) => label === "r2:get-range")).toHaveLength(1);
234
    expect(cacheCtx.memo?.subreqBudget).toBe(18);
235
  });
236
 
237
  it("does not let a bad pack-size hint poison later idx loads", async () => {
238
    const repo = uniqueRepoId("pack-idx-hint-poison");
239
    await setupRepoForTests(env, "o", repo);
240
    const repoId = `o/${repo}`;
241
    const seeded = await seedPackFirstRepo(repoId);
242
    const activeCatalog = await callStubWithRetry(seeded.getStub, (stub) =>
243
      stub.getActivePackCatalog()
244
    );
245
    const pack = activeCatalog[0]!;
246
 
247
    const poisonCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`);
248
    const poisoned = await loadIdxView(env, pack.packKey, poisonCtx, pack.packBytes - 7);
249
    expect(poisoned).toBeDefined();
250
    const poisonedLastPackIndex = poisoned!.sortedOffsetIndices[poisoned!.count - 1]!;
251
    expect(getNextOffsetByIndex(poisoned!, poisonedLastPackIndex)).toBe(pack.packBytes - 27);
252
 
253
    const cleanCtx = createTestCacheContext(`https://example.com/${repoId}/git-upload-pack`);
254
    const healed = await loadIdxView(env, pack.packKey, cleanCtx, pack.packBytes);
255
 
256
    expect(healed).toBeDefined();
257
    const healedLastPackIndex = healed!.sortedOffsetIndices[healed!.count - 1]!;
258
    expect(getNextOffsetByIndex(healed!, healedLastPackIndex)).toBe(pack.packBytes - 20);
259
  });
260
 
261
  it("parses 64-bit idx offsets without truncating them to 32 bits", () => {
262
    const oidHex = "12".repeat(20);
263
    const offset = UINT32_SPAN + 123;
264
    const packSize = offset + 4096;
265
    const idxBuf = buildSingleEntryIdx(oidHex, offset);
266
 
267
    const view = parseIdxView("test/large-offset.pack", idxBuf, packSize);
268
 
269
    expect(view).toBeDefined();
270
    expect(view!.offsets[0]).toBe(offset);
271
    expect(view!.sortedOffsets[0]).toBe(offset);
272
    expect(getNextOffsetByIndex(view!, 0)).toBe(packSize - 20);
273
  });
274
 
275
  it("rejects truncated 64-bit idx offset tables cleanly", () => {
276
    const oidHex = "34".repeat(20);
277
    const offset = UINT32_SPAN + 456;
278
    const packSize = offset + 1024;
279
    const idxBuf = buildSingleEntryIdx(oidHex, offset, false);
280
    const dv = new DataView(idxBuf.buffer);
281
    const offsetTablePos = 8 + 256 * 4 + 20 + 4;
282
    dv.setUint32(offsetTablePos, 0x80000000, false);
283
 
284
    expect(parseIdxView("test/truncated-large-offset.pack", idxBuf, packSize)).toBeUndefined();
285
  });
286
 
287
  it("rejects 64-bit idx offsets above the safe integer range", () => {
288
    const oidHex = "56".repeat(20);
289
    const idxBuf = buildSingleEntryIdx(oidHex, UINT32_SPAN + 1);
290
    const dv = new DataView(idxBuf.buffer);
291
    const largeOffsetPos = 8 + 256 * 4 + 20 + 4 + 4;
292
    dv.setUint32(largeOffsetPos, 0x00200000, false);
293
    dv.setUint32(largeOffsetPos + 4, 1, false);
294
 
295
    expect(() =>
296
      parseIdxView("test/unsafe-large-offset.pack", idxBuf, Number.MAX_SAFE_INTEGER)
297
    ).toThrow(/safe integer support/);
298
  });
299
 
300
  it("marks packed diff reads as soft-budget truncated once packed subrequests are counted", async () => {
301
    const repo = uniqueRepoId("pack-soft-budget");
302
    await setupRepoForTests(env, "o", repo);
303
    const repoId = `o/${repo}`;
304
    const seeded = await seedPackFirstRepo(repoId);
305
    await deleteLooseObjectCopies(env, seeded.getStub, seeded.objectOids);
306
 
307
    const cacheCtx = createTestCacheContext(
308
      `https://example.com/${repoId}/commit/${seeded.nextCommit.oid}`,
309
      0
310
    );
311
    const diff = await listCommitChangedFiles(env, repoId, seeded.nextCommit.oid, cacheCtx, {
312
      timeBudgetMs: 2000,
313
    });
314
 
315
    expect(diff.truncated).toBe(true);
316
    expect(diff.truncateReason).toBe("soft_budget");
317
  });
318
});