1
/**
2
 * Types for the streaming pack indexer.
3
 *
4
 * The entry table uses typed arrays to keep per-object metadata compact.
5
 * OIDs are stored as raw 20-byte values in a flat Uint8Array; hex strings
6
 * are materialized lazily when needed.
7
 */
8
 
9
import type { CacheContext } from "@/worker/cache";
10
import type { Logger } from "@/worker/common/logger";
11
import type { IdxView, PackCatalogRow } from "@/worker/git/object-store/types";
12
import type { Limiter } from "@/worker/git/operations/limits";
13
import type { PackRefsBuilder } from "@/worker/git/pack/refIndex";
14
 
15
// ---------------------------------------------------------------------------
16
// Entry table – struct-of-arrays backed by typed arrays
17
// ---------------------------------------------------------------------------
18
 
19
export interface PackEntryTable {
20
  count: number;
21
 
22
  /** Byte offset in pack where each entry starts. */
23
  offsets: Uint32Array;
24
 
25
  /** Pack type code: 1=commit, 2=tree, 3=blob, 4=tag, 6=OFS_DELTA, 7=REF_DELTA. */
26
  types: Uint8Array;
27
 
28
  /**
29
   * Final logical object type code: 1=commit, 2=tree, 3=blob, 4=tag.
30
   * Delta entries retain their pack type in `types`; this array records the
31
   * resolved object type needed by closure sidecars and rematerialization.
32
   */
33
  objectTypes: Uint8Array;
34
 
35
  /** Header length in bytes (type varint + optional delta metadata). */
36
  headerLens: Uint16Array;
37
 
38
  /** Byte offset where the raw entry data (header + compressed) ends. */
39
  spanEnds: Uint32Array;
40
 
41
  /** CRC-32 of the raw entry bytes (header + compressed payload). */
42
  crc32s: Uint32Array;
43
 
44
  /** Raw 20-byte OIDs in a flat buffer (count * 20 bytes). Zeroed for unresolved deltas. */
45
  oids: Uint8Array;
46
 
47
  /**
48
   * Decompressed payload size for non-delta objects.
49
   * For delta objects this holds the delta *result* size (from the delta header).
50
   */
51
  decompressedSizes: Uint32Array;
52
 
53
  /** For OFS_DELTA: absolute byte offset of the base entry; 0 otherwise. */
54
  ofsBaseOffsets: Uint32Array;
55
 
56
  /** 1 when the OID has been computed, 0 when the entry is still pending resolution. */
57
  resolved: Uint8Array;
58
}
59
 
60
/**
61
 * Raw 20-byte REF_DELTA base OIDs stored in entry order.
62
 *
63
 * Only entries whose type is REF_DELTA read meaningful bytes from this buffer.
64
 * The flat layout avoids a sparse Map allocation per delta entry.
65
 */
66
export type RefBaseOids = Uint8Array;
67
 
68
// ---------------------------------------------------------------------------
69
// Allocator
70
// ---------------------------------------------------------------------------
71
 
72
export function allocateEntryTable(count: number): PackEntryTable {
73
  return {
74
    count,
75
    offsets: new Uint32Array(count),
76
    types: new Uint8Array(count),
77
    objectTypes: new Uint8Array(count),
78
    headerLens: new Uint16Array(count),
79
    spanEnds: new Uint32Array(count),
80
    crc32s: new Uint32Array(count),
81
    oids: new Uint8Array(count * 20),
82
    decompressedSizes: new Uint32Array(count),
83
    ofsBaseOffsets: new Uint32Array(count),
84
    resolved: new Uint8Array(count),
85
  };
86
}
87
 
88
// ---------------------------------------------------------------------------
89
// Result types
90
// ---------------------------------------------------------------------------
91
 
92
export interface ScanResult {
93
  table: PackEntryTable;
94
  refBaseOids: RefBaseOids;
95
  refDeltaCount: number;
96
  resolvedCount: number;
97
  objectCount: number;
98
  /** Trailing 20-byte SHA-1 from the pack file. */
99
  packChecksum: Uint8Array;
100
  /** Per-entry logical object reference builder populated as payloads resolve. */
101
  refsBuilder?: PackRefsBuilder;
102
}
103
 
104
// ---------------------------------------------------------------------------
105
// Binary search helpers for typed-array lookups (replaces Map<number, number>)
106
// ---------------------------------------------------------------------------
107
 
108
/**
109
 * Binary search for a pack byte offset in a sorted Uint32Array.
110
 * Returns the matching slot index within that sorted array, or -1 if not found.
111
 */
112
export function searchOffsetIndex(sortedOffsets: Uint32Array, target: number): number {
113
  let lo = 0;
114
  let hi = sortedOffsets.length - 1;
115
  while (lo <= hi) {
116
    const mid = (lo + hi) >>> 1;
117
    const v = sortedOffsets[mid];
118
    if (v === target) return mid;
119
    if (v < target) lo = mid + 1;
120
    else hi = mid - 1;
121
  }
122
  return -1;
123
}
124
 
125
export interface ResolveResult {
126
  objectCount: number;
127
  idxBytes: number;
128
  refIndexBytes: number;
129
  idxView: IdxView;
130
}
131
 
132
// ---------------------------------------------------------------------------
133
// Options
134
// ---------------------------------------------------------------------------
135
 
136
export interface IndexerOptions {
137
  env: Env;
138
  /** Full R2 key of the .pack file. */
139
  packKey: string;
140
  /** Total byte size of the .pack file in R2. */
141
  packSize: number;
142
  /** Bytes per sequential R2 range read (default 1 MiB). */
143
  chunkSize?: number;
144
  limiter: Limiter;
145
  countSubrequest: (n?: number) => void;
146
  log: Logger;
147
  signal?: AbortSignal;
148
  onProgress?: (message: string) => void;
149
}
150
 
151
export interface ResolveOptions extends IndexerOptions {
152
  scanResult: ScanResult;
153
  /** Active pack catalog snapshot for external (thin-pack) base resolution. */
154
  activeCatalog?: PackCatalogRow[];
155
  cacheCtx?: CacheContext;
156
  /** Repository identifier used by the object store. */
157
  repoId: string;
158
  /** Hard byte budget for the base-payload LRU cache (default 32 MiB). */
159
  lruBudget?: number;
160
  /**
161
   * Backfill can rebuild the derived `.refs` artifact from an already-active
162
   * pack without rewriting the deterministic `.idx` beside it.
163
   */
164
  writeIdx?: boolean;
165
  /** Existing idx view required when `writeIdx` is false. */
166
  existingIdxView?: IdxView;
167
}
168
 
169
export interface ConnectivityCheckOptions {
170
  env: Env;
171
  repoId: string;
172
  /** Full R2 key of the newly indexed .pack. */
173
  newPackKey: string;
174
  /** Pre-built IdxView for the new pack (avoids an extra R2 read). */
175
  newIdxView: IdxView;
176
  /** Byte size of the new .pack in R2. */
177
  newPackSize: number;
178
  /** Active pack catalog snapshot (existing packs). */
179
  activeCatalog: PackCatalogRow[];
180
  commands: { oldOid: string; newOid: string; ref: string }[];
181
  statuses: { ref: string; ok: boolean; msg?: string }[];
182
  log: Logger;
183
  cacheCtx: CacheContext;
184
}
185
 
186
export function getRefBaseOidAt(refBaseOids: RefBaseOids, index: number): Uint8Array {
187
  const start = index * 20;
188
  return refBaseOids.subarray(start, start + 20);
189
}