Skip to content
rachel/git-on-cloudflare
File

Blob: src/worker/git/pack/indexer/resolve/index.ts

typescript788 lines
1/**
2 * Delta resolution and idx writing (Pass 2).
3 *
4 * Processes entries from the scan result in pack-offset order, resolves delta
5 * chains, computes OIDs, and writes standard Git idx v2 plus logical-reference
6 * sidecar artifacts to R2.
7 *
8 * Processing in offset order naturally satisfies the OFS_DELTA dependency
9 * ordering (bases are always at lower offsets). REF_DELTA entries are first
10 * given every chance to resolve against later in-pack deltas; only the
11 * remaining unresolved roots fall back to the active pack snapshot.
12 *
13 * Base payloads are held in a byte-budgeted LRU cache. On eviction, bases can
14 * be recomputed from the pack via buffered pack reads. The typical case
15 * requires zero extra reads beyond the sequential pass.
16 */
17 
18import type { CacheContext } from "@/worker/cache";
19import { exports as workerExports } from "cloudflare:workers";
20import { bytesEqual } from "@/worker/common/bytes";
21import { bytesToHex } from "@/worker/common/hex";
22import { computeOidBytes, objTypeCode } from "@/worker/git/core/objects";
23import { applyGitDelta } from "@/worker/git/object-store/delta";
24import { findOidIndexFromBytes, parseIdxView } from "@/worker/git/object-store/idxView";
25import type { IdxView } from "@/worker/git/object-store/types";
26import { ensureMemo, typeCodeToObjectType } from "@/worker/git/object-store/support";
27import { packIndexKey, packRefsKey } from "@/worker/keys";
28 
29import { searchOffsetIndex, getRefBaseOidAt } from "../types";
30import type { ResolveOptions, ResolveResult } from "../types";
31import { writeIdxV2 } from "../writeIdx";
32 
33import { drainReadyDeferredQueue } from "./deferred";
34import {
35  createInPackDependencyQueue,
36  enqueueReadyDeferred,
37  promoteReadyInPackDependents,
38  registerInPackDependency,
39} from "./dependencies";
40import { throwIfAborted } from "./errors";
41import { readExternalBaseObject } from "./externalBase";
42import { resolveDeltaEntry, storeOid } from "./helpers";
43import { PayloadLRU } from "./payloadCache";
44import { inflateFromReader, SequentialReader } from "./reader";
45import {
46  createRefBaseLookup,
47  enqueueWaitingRefDelta,
48  getResolvedBaseEntry,
49  noteResolvedEntry,
50  promoteWaitingRefDeltas,
51  type RefBaseLookup,
52} from "./refLookup";
53import type { InPackDependencyQueue } from "./dependencies";
54 
55const DEFAULT_LRU_BUDGET = 32 * 1024 * 1024; // 32 MiB
56const DEFAULT_CHUNK_SIZE = 4_194_304; // 4 MiB — larger chunks reduce R2 reads during resolve
57const RESOLVE_PROGRESS_STEPS = 20;
58 
59function emitResolveProgress(
60  onProgress: ResolveOptions["onProgress"],
61  resolved: number,
62  total: number
63): void {
64  if (!onProgress || total <= 0) return;
65  const percent = Math.round((resolved / total) * 100);
66  if (resolved >= total) {
67    onProgress(`Resolving deltas: 100% (${total}/${total}), done.\n`);
68    return;
69  }
70  onProgress(`Resolving deltas: ${percent}% (${resolved}/${total})\r`);
71}
72 
73export async function resolveDeltasAndWriteIdx(opts: ResolveOptions): Promise<ResolveResult> {
74  const { env, packKey, packSize, log, scanResult, repoId, lruBudget } = opts;
75  const { table, objectCount, packChecksum } = scanResult;
76  const initialResolvedCount = scanResult.resolvedCount;
77  const unresolvedCount = objectCount - initialResolvedCount;
78 
79  if (opts.existingIdxView) {
80    validateExistingIdxView(opts.existingIdxView, objectCount, packSize, packChecksum);
81    seedEntryOidsFromExistingIdx(scanResult, opts.existingIdxView);
82  }
83 
84  const resolveCacheCtx = ensureResolveCacheContext(
85    opts.cacheCtx,
86    repoId,
87    opts.activeCatalog,
88    opts.limiter
89  );
90  const resolveOpts =
91    opts.cacheCtx === resolveCacheCtx ? opts : { ...opts, cacheCtx: resolveCacheCtx };
92 
93  log.info("resolve:start", { objectCount, unresolvedCount });
94  throwIfAborted(opts.signal, log, "resolve:start");
95  if (unresolvedCount === 0) {
96    return await writeAndParseIdx(resolveOpts, packKey, packSize, table, objectCount, packChecksum);
97  }
98  opts.onProgress?.(`Resolving deltas: 0% (0/${unresolvedCount})\r`);
99 
100  const lru = new PayloadLRU(lruBudget ?? DEFAULT_LRU_BUDGET, objectCount);
101  const chunkSize = opts.chunkSize ?? DEFAULT_CHUNK_SIZE;
102  const progressInterval = Math.max(1, Math.floor(unresolvedCount / RESOLVE_PROGRESS_STEPS));
103  // Two readers: one for the main sequential pass, one for rematerialization
104  // and deferred work so cache misses do not trash the main pass window.
105  const seqReader = new SequentialReader(
106    env,
107    packKey,
108    packSize,
109    chunkSize,
110    opts.limiter,
111    opts.countSubrequest,
112    log,
113    opts.signal
114  );
115  const auxReader = new SequentialReader(
116    env,
117    packKey,
118    packSize,
119    chunkSize,
120    opts.limiter,
121    opts.countSubrequest,
122    log,
123    opts.signal
124  );
125 
126  const isBase = new Uint8Array(objectCount);
127  const baseIndex = new Int32Array(objectCount).fill(-1);
128  const deadlines = new Uint32Array(objectCount);
129  const resolvedTypeCodes = new Uint8Array(objectCount);
130 
131  for (let i = 0; i < objectCount; i++) {
132    if (table.resolved[i]) {
133      resolvedTypeCodes[i] = table.types[i];
134    }
135  }
136 
137  if (scanResult.refDeltaCount === 0) {
138    buildOfsDependencies(table, objectCount, isBase, baseIndex, deadlines);
139    propagateDeadlines(table, objectCount, baseIndex, deadlines);
140    lru.setDeadlines(deadlines);
141    await seqReader.preload(table.offsets[0]);
142 
143    let resolved = initialResolvedCount;
144    for (let i = 0; i < objectCount; i++) {
145      throwIfAborted(opts.signal, log, "resolve:ofs-main");
146      lru.setCurrentOffset(table.offsets[i]);
147      if (table.resolved[i]) {
148        await cacheResolvedBaseIfNeeded(
149          table,
150          i,
151          isBase,
152          resolvedTypeCodes,
153          lru,
154          seqReader,
155          !!opts.existingIdxView
156        );
157        continue;
158      }
159      const bi = baseIndex[i];
160      if (bi < 0 || !table.resolved[bi]) {
161        throw new Error(`resolve: OFS_DELTA base is not ready for entry ${i}`);
162      }
163      await resolveDeltaEntry({
164        index: i,
165        resolveOpts,
166        table,
167        lru,
168        deltaReader: seqReader,
169        baseReader: auxReader,
170        baseIndex,
171        resolvedTypeCodes,
172        isBase,
173        deadlines,
174        refLookup: null,
175      });
176      resolved++;
177      logResolveProgress(
178        log,
179        opts.onProgress,
180        resolved,
181        initialResolvedCount,
182        unresolvedCount,
183        progressInterval
184      );
185    }
186 
187    if (resolved !== objectCount) {
188      throw new Error(`resolve: ${objectCount - resolved} objects could not be resolved`);
189    }
190    return await writeAndParseIdx(resolveOpts, packKey, packSize, table, objectCount, packChecksum);
191  }
192 
193  const refLookup = createRefBaseLookup(scanResult);
194  if (!refLookup) {
195    throw new Error("resolve: expected REF_DELTA lookup state for mixed pack");
196  }
197 
198  for (let i = 0; i < objectCount; i++) {
199    if (table.resolved[i]) {
200      noteResolvedEntry(refLookup, table.oids, i);
201    }
202  }
203 
204  // Mixed packs need two kinds of dependency bookkeeping:
205  // 1) REF_DELTA entries whose base OID is not mapped to an in-pack entry yet.
206  //    Those stay in `refLookup` until some later resolution publishes the OID.
207  // 2) Deltas whose in-pack base entry is already known but not resolved yet.
208  //    Those register in the dependency queue so a late base can wake them in
209  //    O(children) time without rescanning the whole deferred set.
210  const dependencyQueue = createInPackDependencyQueue(objectCount);
211  buildMixedDependencies(
212    scanResult,
213    isBase,
214    baseIndex,
215    deadlines,
216    refLookup,
217    dependencyQueue,
218    opts.existingIdxView
219  );
220  propagateDeadlines(table, objectCount, baseIndex, deadlines);
221  lru.setDeadlines(deadlines);
222  await seqReader.preload(table.offsets[0]);
223 
224  const deferred: number[] = [];
225  const deferredQueued = new Uint8Array(objectCount);
226  const readyDeferred: number[] = [];
227  let resolved = initialResolvedCount;
228 
229  for (let i = 0; i < objectCount; i++) {
230    throwIfAborted(opts.signal, log, "resolve:mixed-main");
231    lru.setCurrentOffset(table.offsets[i]);
232 
233    if (table.resolved[i]) {
234      await cacheResolvedBaseIfNeeded(
235        table,
236        i,
237        isBase,
238        resolvedTypeCodes,
239        lru,
240        seqReader,
241        !!opts.existingIdxView
242      );
243      continue;
244    }
245 
246    const bi = baseIndex[i];
247    if (table.types[i] === 7 && bi < 0) {
248      // A missing base OID at first sight is ambiguous: it may be a true thin
249      // pack external, or a later in-pack delta/base whose OID is not known
250      // yet. Defer external fallback until the in-pack promotion path is done.
251      deferred.push(i);
252      continue;
253    }
254    if (bi < 0 || !table.resolved[bi]) {
255      deferred.push(i);
256      continue;
257    }
258 
259    await resolveDeltaEntry({
260      index: i,
261      resolveOpts,
262      table,
263      lru,
264      deltaReader: seqReader,
265      baseReader: auxReader,
266      baseIndex,
267      resolvedTypeCodes,
268      isBase,
269      deadlines,
270      refLookup,
271      dependencyQueue,
272      readyDeferred,
273      deferredQueued,
274    });
275    resolved++;
276    logResolveProgress(
277      log,
278      opts.onProgress,
279      resolved,
280      initialResolvedCount,
281      unresolvedCount,
282      progressInterval
283    );
284  }
285 
286  for (const index of deferred) {
287    enqueueReadyDeferred(readyDeferred, deferredQueued, table, baseIndex, index);
288  }
289 
290  resolved = await drainReadyDeferredQueue({
291    readyDeferred,
292    deferredQueued,
293    resolved,
294    initialResolvedCount,
295    totalUnresolved: unresolvedCount,
296    log,
297    resolveOpts,
298    table,
299    lru,
300    reader: auxReader,
301    resolvedTypeCodes,
302    refLookup,
303    dependencyQueue,
304    baseIndex,
305    isBase,
306    deadlines,
307  });
308 
309  for (const index of deferred) {
310    throwIfAborted(opts.signal, log, "resolve:external-fallback");
311    if (table.resolved[index]) continue;
312 
313    const bi = baseIndex[index];
314    if (bi >= 0) {
315      // This entry now has an in-pack base and just needs the shared deferred
316      // drain to pick it up. Re-enqueue it instead of treating it as a thin
317      // pack external-base case.
318      enqueueReadyDeferred(readyDeferred, deferredQueued, table, baseIndex, index);
319      continue;
320    }
321 
322    // Still no in-pack base after all promotions. At this point a REF_DELTA is
323    // a true thin-pack external lookup, so fall back to the active catalog.
324    const baseOid = bytesToHex(getRefBaseOidAt(scanResult.refBaseOids, index));
325    const baseObj = await readExternalBaseObject(resolveOpts, baseOid);
326    if (!baseObj) continue;
327 
328    lru.setCurrentOffset(table.offsets[index]);
329    const deltaPayload = await inflateFromReader(auxReader, table, index);
330    throwIfAborted(opts.signal, log, "resolve:external-fallback");
331    const result = applyGitDelta(baseObj.payload, deltaPayload);
332    if (result.length !== table.decompressedSizes[index]) {
333      throw new Error(
334        `resolve: deferred delta result size mismatch at offset ${table.offsets[index]} (expected ${table.decompressedSizes[index]}, got ${result.length})`
335      );
336    }
337 
338    if (opts.existingIdxView) {
339      table.resolved[index] = 1;
340    } else {
341      storeOid(table, index, await computeOidBytes(baseObj.type, result));
342    }
343    resolvedTypeCodes[index] = objTypeCode(baseObj.type);
344    table.objectTypes[index] = resolvedTypeCodes[index];
345    scanResult.refsBuilder?.recordObject(index, baseObj.type, result);
346    promoteWaitingRefDeltas(
347      refLookup,
348      index,
349      table,
350      baseIndex,
351      isBase,
352      deadlines,
353      readyDeferred,
354      deferredQueued
355    );
356    promoteReadyInPackDependents(
357      dependencyQueue,
358      index,
359      readyDeferred,
360      deferredQueued,
361      table,
362      baseIndex
363    );
364    if (isBase[index]) {
365      lru.set(index, { type: baseObj.type, payload: result });
366    }
367    resolved++;
368    logResolveProgress(
369      log,
370      opts.onProgress,
371      resolved,
372      initialResolvedCount,
373      unresolvedCount,
374      progressInterval
375    );
376    resolved = await drainReadyDeferredQueue({
377      readyDeferred,
378      deferredQueued,
379      resolved,
380      initialResolvedCount,
381      totalUnresolved: unresolvedCount,
382      log,
383      resolveOpts,
384      table,
385      lru,
386      reader: auxReader,
387      resolvedTypeCodes,
388      refLookup,
389      dependencyQueue,
390      baseIndex,
391      isBase,
392      deadlines,
393    });
394  }
395 
396  resolved = await drainReadyDeferredQueue({
397    readyDeferred,
398    deferredQueued,
399    resolved,
400    initialResolvedCount,
401    totalUnresolved: unresolvedCount,
402    log,
403    resolveOpts,
404    table,
405    lru,
406    reader: auxReader,
407    resolvedTypeCodes,
408    refLookup,
409    dependencyQueue,
410    baseIndex,
411    isBase,
412    deadlines,
413  });
414 
415  if (resolved !== objectCount) {
416    throw new Error(`resolve: ${objectCount - resolved} objects could not be resolved`);
417  }
418 
419  return await writeAndParseIdx(resolveOpts, packKey, packSize, table, objectCount, packChecksum);
420}
421 
422function buildOfsDependencies(
423  table: ResolveOptions["scanResult"]["table"],
424  objectCount: number,
425  isBase: Uint8Array,
426  baseIndex: Int32Array,
427  deadlines: Uint32Array
428): void {
429  for (let i = 0; i < objectCount; i++) {
430    if (table.resolved[i]) continue;
431    const baseOff = table.ofsBaseOffsets[i];
432    const bi = searchOffsetIndex(table.offsets, baseOff);
433    if (bi < 0) {
434      throw new Error(
435        `resolve: OFS_DELTA at offset ${table.offsets[i]} references unknown base offset ${baseOff}`
436      );
437    }
438    baseIndex[i] = bi;
439    isBase[bi] = 1;
440    deadlines[bi] = Math.max(deadlines[bi], table.offsets[i]);
441  }
442}
443 
444function buildMixedDependencies(
445  scanResult: ResolveOptions["scanResult"],
446  isBase: Uint8Array,
447  baseIndex: Int32Array,
448  deadlines: Uint32Array,
449  refLookup: RefBaseLookup,
450  dependencyQueue: InPackDependencyQueue,
451  existingIdxView?: IdxView
452): void {
453  const { table, objectCount } = scanResult;
454  for (let i = 0; i < objectCount; i++) {
455    if (table.resolved[i]) continue;
456 
457    if (table.types[i] === 6) {
458      const baseOff = table.ofsBaseOffsets[i];
459      const bi = searchOffsetIndex(table.offsets, baseOff);
460      if (bi < 0) {
461        throw new Error(
462          `resolve: OFS_DELTA at offset ${table.offsets[i]} references unknown base offset ${baseOff}`
463        );
464      }
465      baseIndex[i] = bi;
466      isBase[bi] = 1;
467      deadlines[bi] = Math.max(deadlines[bi], table.offsets[i]);
468      if (!table.resolved[bi]) {
469        // OFS_DELTA entries know their base slot up front, so record the edge
470        // now when that base is still unresolved. Once the base resolves later
471        // in the pass, the dependency queue can wake this child immediately.
472        registerInPackDependency(dependencyQueue, baseIndex, i);
473      }
474      continue;
475    }
476 
477    if (table.types[i] !== 7) continue;
478    const idxBaseEntry = existingIdxView
479      ? findSamePackRefBaseEntry(scanResult, existingIdxView, i)
480      : -1;
481    if (idxBaseEntry >= 0 && idxBaseEntry !== i) {
482      // Backfill starts from a trusted `.idx`, so it already knows every
483      // same-pack REF_DELTA base OID before the scan result has resolved those
484      // entries. The idx is OID-sorted, so convert through the stored pack
485      // offset before assigning the resolver's pack-entry index.
486      baseIndex[i] = idxBaseEntry;
487      isBase[idxBaseEntry] = 1;
488      deadlines[idxBaseEntry] = Math.max(deadlines[idxBaseEntry], table.offsets[i]);
489      if (!table.resolved[idxBaseEntry]) {
490        registerInPackDependency(dependencyQueue, baseIndex, i);
491      }
492      continue;
493    }
494 
495    const bi = getResolvedBaseEntry(refLookup, i);
496    if (bi >= 0) {
497      // This REF_DELTA already points at an in-pack entry whose OID was known
498      // during scan or was published by an earlier resolved base.
499      baseIndex[i] = bi;
500      isBase[bi] = 1;
501      deadlines[bi] = Math.max(deadlines[bi], table.offsets[i]);
502      continue;
503    }
504    // The base OID is not mapped to an in-pack entry yet. Keep it in the
505    // REF lookup so a later resolution can promote it, or the caller can treat
506    // it as a thin-pack external base once in-pack promotion is exhausted.
507    enqueueWaitingRefDelta(refLookup, i);
508  }
509}
510 
511function findSamePackRefBaseEntry(
512  scanResult: ResolveOptions["scanResult"],
513  existingIdxView: IdxView,
514  entryIndex: number
515): number {
516  const oidIndex = findOidIndexFromBytes(existingIdxView, scanResult.refBaseOids, entryIndex * 20);
517  if (oidIndex < 0) return -1;
518 
519  let runStart = oidIndex;
520  while (runStart > 0) {
521    if (!idxOidMatchesRefBase(existingIdxView, scanResult.refBaseOids, entryIndex, runStart - 1)) {
522      break;
523    }
524    runStart--;
525  }
526 
527  let runEnd = oidIndex;
528  while (runEnd + 1 < existingIdxView.count) {
529    if (!idxOidMatchesRefBase(existingIdxView, scanResult.refBaseOids, entryIndex, runEnd + 1)) {
530      break;
531    }
532    runEnd++;
533  }
534 
535  let firstDeltaBase = -1;
536  let sawCurrentEntry = false;
537  for (let scanIndex = runStart; scanIndex <= runEnd; scanIndex++) {
538    const baseOffset = existingIdxView.offsets[scanIndex];
539    const baseEntry = searchOffsetIndex(scanResult.table.offsets, baseOffset);
540    if (baseEntry < 0) continue;
541    if (baseEntry === entryIndex) {
542      sawCurrentEntry = true;
543      continue;
544    }
545    if (scanResult.table.types[baseEntry] < 6) return baseEntry;
546    if (firstDeltaBase < 0) firstDeltaBase = baseEntry;
547  }
548 
549  // If the current REF_DELTA is itself in this duplicate-OID run, then the
550  // base OID equals the entry's final OID. Without a full-object duplicate in
551  // the target pack, another delta in the same run may still depend on the true
552  // external base. Let external fallback resolve that acyclic base instead of
553  // manufacturing a same-pack dependency cycle.
554  return sawCurrentEntry ? -1 : firstDeltaBase;
555}
556 
557function idxOidMatchesRefBase(
558  existingIdxView: IdxView,
559  refBaseOids: ResolveOptions["scanResult"]["refBaseOids"],
560  entryIndex: number,
561  oidIndex: number
562): boolean {
563  const refStart = entryIndex * 20;
564  const idxStart = oidIndex * 20;
565  for (let offset = 0; offset < 20; offset++) {
566    if (existingIdxView.rawNames[idxStart + offset] !== refBaseOids[refStart + offset]) {
567      return false;
568    }
569  }
570  return true;
571}
572 
573function seedEntryOidsFromExistingIdx(
574  scanResult: ResolveOptions["scanResult"],
575  existingIdxView: IdxView
576): void {
577  const { table, objectCount } = scanResult;
578  for (let oidIndex = 0; oidIndex < objectCount; oidIndex++) {
579    const baseOffset = existingIdxView.offsets[oidIndex];
580    const entryIndex = searchOffsetIndex(table.offsets, baseOffset);
581    if (entryIndex < 0) {
582      throw new Error("resolve: existing idx view mismatch with scanned pack offsets");
583    }
584    table.oids.set(
585      existingIdxView.rawNames.subarray(oidIndex * 20, oidIndex * 20 + 20),
586      entryIndex * 20
587    );
588  }
589}
590 
591function propagateDeadlines(
592  table: ResolveOptions["scanResult"]["table"],
593  objectCount: number,
594  baseIndex: Int32Array,
595  deadlines: Uint32Array
596): void {
597  // The initial dependency graph only knows about OFS_DELTA edges and REF_DELTA
598  // edges whose base already resolved during the scan pass. A reverse sweep is
599  // enough to push each entry's last-needed offset into its in-pack base chain
600  // without rewalking ancestors for every node.
601  for (let i = objectCount - 1; i >= 0; i--) {
602    if (table.resolved[i]) continue;
603    const bi = baseIndex[i];
604    if (bi < 0) continue;
605    const neededUntil = Math.max(table.offsets[i], deadlines[i]);
606    if (deadlines[bi] < neededUntil) deadlines[bi] = neededUntil;
607  }
608}
609 
610async function cacheResolvedBaseIfNeeded(
611  table: ResolveOptions["scanResult"]["table"],
612  index: number,
613  isBase: Uint8Array,
614  resolvedTypeCodes: Uint8Array,
615  lru: PayloadLRU,
616  reader: SequentialReader,
617  refsOnlyBackfill: boolean
618): Promise<void> {
619  if (!isBase[index] || lru.get(index)) return;
620  const t = typeCodeToObjectType(resolvedTypeCodes[index]);
621  if (!t) return;
622  if (refsOnlyBackfill && t === "blob") return;
623  const payload = await inflateFromReader(reader, table, index);
624  lru.set(index, { type: t, payload });
625}
626 
627function logResolveProgress(
628  log: ResolveOptions["log"],
629  onProgress: ResolveOptions["onProgress"],
630  resolved: number,
631  initialResolvedCount: number,
632  unresolvedCount: number,
633  progressInterval: number
634): void {
635  const newlyResolved = resolved - initialResolvedCount;
636  if (newlyResolved > 0 && newlyResolved % 10000 === 0) {
637    log.debug("resolve:progress", { resolved: newlyResolved, total: unresolvedCount });
638  }
639  if (
640    newlyResolved > 0 &&
641    (newlyResolved % progressInterval === 0 || newlyResolved === unresolvedCount)
642  ) {
643    emitResolveProgress(onProgress, newlyResolved, unresolvedCount);
644  }
645}
646 
647function ensureResolveCacheContext(
648  cacheCtx: CacheContext | undefined,
649  repoId: string,
650  activeCatalog: ResolveOptions["activeCatalog"],
651  limiter: ResolveOptions["limiter"]
652): CacheContext {
653  const resolvedCacheCtx =
654    cacheCtx ??
655    ({
656      req: new Request("http://localhost"),
657      ctx: {
658        waitUntil() {},
659        passThroughOnException() {},
660        props: undefined,
661        exports: workerExports,
662      },
663      memo: {},
664    } satisfies CacheContext);
665 
666  ensureMemo(resolvedCacheCtx, repoId);
667  resolvedCacheCtx.memo = resolvedCacheCtx.memo || {};
668  if (!resolvedCacheCtx.memo.limiter) {
669    // External-base reads must share the caller's limiter/budget. If they
670    // silently allocate their own memo state here, the receive path can exceed
671    // platform limits without the main request noticing.
672    resolvedCacheCtx.memo.limiter = limiter;
673  }
674  if (activeCatalog) {
675    // The caller-supplied snapshot is the authority for this resolve pass. If
676    // the memo already holds older pack-catalog state, thin-pack validation
677    // must not drift back to it when external bases are resolved.
678    resolvedCacheCtx.memo.packCatalog = activeCatalog;
679  }
680  return resolvedCacheCtx;
681}
682 
683function validateExistingIdxView(
684  idxView: IdxView,
685  objectCount: number,
686  packSize: number,
687  packChecksum: Uint8Array
688): void {
689  if (
690    idxView.count !== objectCount ||
691    idxView.packSize !== packSize ||
692    !bytesEqual(idxView.packChecksum, packChecksum)
693  ) {
694    throw new Error("resolve: existing idx view mismatch with resolved pack");
695  }
696}
697 
698async function putPackIdx(opts: ResolveOptions, idxBuf: Uint8Array): Promise<void> {
699  const idxKey = packIndexKey(opts.packKey);
700  throwIfAborted(opts.signal, opts.log, "resolve:put-pack-idx");
701  // Count the idx write under the same request budget as the pack reads. R2
702  // writes are subject to the same platform request limits in practice, so
703  // skipping them here makes the validation numbers too optimistic.
704  opts.countSubrequest();
705  await opts.limiter.run("r2:put-pack-idx", async () => {
706    await opts.env.REPO_BUCKET.put(idxKey, idxBuf);
707  });
708}
709 
710async function putPackRefs(opts: ResolveOptions, refsBuf: Uint8Array): Promise<void> {
711  const refsKey = packRefsKey(opts.packKey);
712  throwIfAborted(opts.signal, opts.log, "resolve:put-pack-refs");
713  opts.log.info("ref-index:write-start", {
714    packKey: opts.packKey,
715    refsKey,
716    bytes: refsBuf.byteLength,
717  });
718  opts.countSubrequest();
719  try {
720    await opts.limiter.run("r2:put-pack-refs", async () => {
721      await opts.env.REPO_BUCKET.put(refsKey, refsBuf);
722    });
723  } catch (error) {
724    opts.log.error("ref-index:write-error", {
725      packKey: opts.packKey,
726      refsKey,
727      error: String(error),
728    });
729    throw error;
730  }
731  opts.log.info("ref-index:write-complete", {
732    packKey: opts.packKey,
733    refsKey,
734    bytes: refsBuf.byteLength,
735  });
736}
737 
738async function writeAndParseIdx(
739  opts: ResolveOptions,
740  packKey: string,
741  packSize: number,
742  table: ResolveOptions["scanResult"]["table"],
743  objectCount: number,
744  packChecksum: Uint8Array
745): Promise<ResolveResult> {
746  throwIfAborted(opts.signal, opts.log, "resolve:write-idx");
747  opts.onProgress?.("Writing pack index\n");
748  let idxBytes = 0;
749  let idxView = opts.existingIdxView;
750 
751  if (opts.writeIdx !== false) {
752    const idxBuf = await writeIdxV2(table, objectCount, packChecksum);
753    idxBytes = idxBuf.byteLength;
754    throwIfAborted(opts.signal, opts.log, "resolve:write-idx");
755    await putPackIdx(opts, idxBuf);
756    idxView = parseIdxView(packKey, idxBuf, packSize);
757    if (!idxView) throw new Error("resolve: failed to parse generated idx");
758  }
759 
760  if (!idxView) {
761    throw new Error("resolve: existing idx view is required when idx writing is disabled");
762  }
763  validateExistingIdxView(idxView, objectCount, packSize, packChecksum);
764 
765  opts.onProgress?.("Writing pack reference index\n");
766  const refsBuilder = opts.scanResult.refsBuilder;
767  if (!refsBuilder) {
768    throw new Error("resolve: pack reference builder is required to write .refs");
769  }
770 
771  const refsResult = refsBuilder.build({
772    table,
773    objectCount,
774    packBytes: packSize,
775    packChecksum: idxView.packChecksum,
776    idxChecksum: idxView.idxChecksum,
777  });
778  throwIfAborted(opts.signal, opts.log, "resolve:write-pack-refs");
779  await putPackRefs(opts, refsResult.bytes);
780 
781  opts.log.info("resolve:done", {
782    objectCount,
783    idxBytes,
784    refIndexBytes: refsResult.refIndexBytes,
785  });
786  return { objectCount, idxBytes, refIndexBytes: refsResult.refIndexBytes, idxView };
787}