diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 31452734..13d7658e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -# This workflow is used to release a new version of @devrev/ts-adaas package to +# This workflow is used to release a new version of @devrev/airsync-sdk package to # npm registry and generate release notes using softprops/action-gh-release # action. It consists of two jobs: # @@ -93,7 +93,7 @@ jobs: id: version run: | # Get the latest version including prereleases - LATEST_VERSION=$(npm view @devrev/ts-adaas versions --json 2>/dev/null | jq -r '.[-1]' || echo "0.0.0") + LATEST_VERSION=$(npm view @devrev/airsync-sdk versions --json 2>/dev/null | jq -r '.[-1]' || echo "0.0.0") echo "Latest published version: $LATEST_VERSION" echo "LATEST_PUBLISHED_VERSION=$LATEST_VERSION" >> $GITHUB_ENV diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..a1183527 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,785 @@ +# Migrating connectors from v1 (`@devrev/ts-adaas`) to v2 (`@devrev/airsync-sdk`) + +This guide covers every connector-facing breaking change between +`@devrev/ts-adaas` v1.x and `@devrev/airsync-sdk` v2.0.0, with before/after +examples drawn from real connectors. + +**The wire protocol is unchanged.** Event payloads sent to the platform, API +routes, headers, the artifact/upload flow, and every surviving event-type +**string value** are byte-for-byte identical to v1. Only the connector-facing +**TypeScript API** changed. A correctly migrated connector talks to the +platform exactly like a v1 connector did. + +> **Why a hard break with no deprecation shims?** v2 is a single bundled-pain +> major release: the package was renamed, so there are no existing +> `@devrev/airsync-sdk` consumers to keep backward-compatible. Everything that +> needed to change changed at once, in one guide, rather than across a long +> deprecation cycle. + +--- + +## TL;DR — what breaks + +| # | Change | Who it affects | +|---|--------|----------------| +| 1 | Package renamed `@devrev/ts-adaas` → `@devrev/airsync-sdk` | every import | +| 2 | `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage` | type annotations (all connectors) | +| 3 | `processTask` → `processExtractionTask` / `processLoadingTask` | every worker file | +| 4 | `adapter.emit(...)` is gone — tasks **return** a `TaskResult` instead | every worker file | +| 5 | `WorkerAdapter` **class** removed → `ExtractionAdapter` / `LoadingAdapter` | helper signatures | +| 6 | `loadItemTypes` / `loadAttachments` / `streamAttachments` now **return** a `TaskResult` | loading + attachment workers | +| 7 | `EventData.external_sync_units` **removed** — external sync units must be pushed to a repo | ESU workers | +| 8 | `adapter.state` is connector-state only; SDK fields readable via `adapter.sdkState`. `lastSyncStarted` / `lastSuccessfulSyncStarted` **removed** | connectors reading/writing SDK bookkeeping | +| 9 | `axios`, `axiosClient`, `formatAxiosError`, `serializeAxiosError`, `HTTPResponse` no longer exported | anyone importing the SDK's HTTP/axios surface | +| 10 | `Mappers` methods return the unwrapped body (`Promise`), not `Promise>` | loaders calling `adapter.mappers.*` | +| 11 | Deprecated v1 modules deleted (`Adapter`, `createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, deprecated `Uploader`) | only legacy code | +| 12 | Deprecated event-type enum **members** deleted; deprecated types/enums (`ExtractionMode`, `EventContextIn`/`Out`, `DomainObjectState`, `ErrorLevel`, `LogRecord`, `AdapterUpdateParams`, `AdapterState`) **removed** | only if you used the old members/types | +| 13 | Several deep `dist/**` import paths moved/removed | deep-importers | +| 14 | Legacy `string[]` attachment-dedup migration dropped | in-flight attachment syncs started on SDK **< 1.15.2** | +| 15 | `spawn`'s deprecated `workerPath` option **removed** — use `baseWorkerPath: __dirname` | connectors still passing `workerPath` | +| 16 | `EventData.progress` field removed (no-op since v1; backend computes progress) | connectors that set `progress` in emit data | + +What does **not** change: `spawn(...)` and its surviving options (`initialState`, +`initialDomainMapping`, `workerPathOverrides`, `baseWorkerPath`, +`options.batchSize`, `timeout`, `isLocalDevelopment`, …) — the one exception is +the long-deprecated `workerPath`, which is removed (§15); default worker paths +(`/workers/data-extraction`, etc.); repos (`initializeRepos`, `getRepo`, +`push`); the normalization +interfaces (`NormalizedItem`, `NormalizedAttachment`, `RepoInterface`, +`ExternalSyncUnit`); `installInitialDomainMapping`; `createMockEvent` / +`MockServer`; HTTP retry behavior; `event_context.extract_from` / +`extract_to`; and every surviving event-type string value on the wire. + +--- + +## 1. Package rename + +```bash +npm uninstall @devrev/ts-adaas +npm install @devrev/airsync-sdk +``` + +Then global-replace the import specifier: + +```ts +// v1 +import { spawn, EventType } from '@devrev/ts-adaas'; +// v2 +import { spawn, EventType } from '@devrev/airsync-sdk'; +``` + +> **Deep imports** like `@devrev/ts-adaas/dist/...` are fragile — several of +> these paths moved or were removed in v2 (see §13). Prefer root imports; the +> v2 barrel now exports several symbols that previously required a deep import +> (`Mappers`, `Item`, `ItemTypeToLoad`). + +## 2. Type renames: `AirdropEvent` → `AirSyncEvent` + +Hard rename, no compatibility alias. The payload **shape** is identical — only +the type name changed. + +| v1 | v2 | +|----|----| +| `AirdropEvent` | `AirSyncEvent` | +| `AirdropMessage` | `AirSyncMessage` | + +```ts +// v1 +const run = async (events: AirdropEvent[]) => { ... }; +// v2 +const run = async (events: AirSyncEvent[]) => { ... }; +``` + +No other public type was renamed — `ConnectionData`, `EventContext`, +`EventData`, `ExtractorEvent`, and `ExternalSyncUnit` keep their v1 names. +Platform-owned strings (`/internal/airdrop.*` routes, the `'ADaaS'` external +system type, the `adaas_library_version` metadata key, `airdrop_*` mapping enum +values) are intentionally unchanged. + +### `AirSyncEvent.context` gained identity fields + +`AirSyncEvent.context` now declares the identity fields the platform already +sends, in addition to the existing `secrets`, `snap_in_id`, and +`snap_in_version_id`: + +```ts +// v2 — AirSyncEvent.context +context: { + secrets: { service_account_token: string }; + snap_in_version_id: string; + snap_in_id: string; + user_id: string; // new + dev_oid: string; // new + source_id: string; // new + service_account_id: string; // new +}; +``` + +If you previously extended the event to read these (e.g. a hand-rolled +`CustomAirdropEvent` that added `user_id`), you can drop the extension and read +`adapter.event.context.user_id` directly. (`snap_in_id` was already present in +v1; only the four fields above are new.) + +> Note: these four fields live on the **top-level** `AirSyncEvent.context`, not +> on the `EventContext` inside `payload.event_context`. The latter is a +> different object and is unchanged. + +## 3 + 4. The new worker contract: **return a `TaskResult`** instead of emitting + +This is the core change of v2. In v1 the connector decided *which event* to +emit and called `adapter.emit(...)`. In v2 the connector only reports *how the +phase ended* by **returning** a `TaskResult`; the SDK maps it to the correct +platform event for the current phase and emits it exactly once. + +```ts +// the exact union (exported from @devrev/airsync-sdk) +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } // note: delaySeconds, not delay + | { status: 'error'; error: ErrorRecord }; // ErrorRecord = { message: string } +``` + +`processTask` is split into two typed entry points — pick the one matching the +worker's phase: + +### Before (v1) + +```ts +import { processTask, ExtractorEventType } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + // ... extract ... + await adapter.emit(ExtractorEventType.DataExtractionDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(ExtractorEventType.DataExtractionProgress, { progress: 50 }); + }, +}); +``` + +### After (v2) + +```ts +import { processExtractionTask } from '@devrev/airsync-sdk'; + +processExtractionTask({ + task: async ({ adapter }) => { + // ... extract ... + return { status: 'success' }; + }, + // onTimeout can be omitted entirely for resumable phases: + // the SDK emits a progress (continuation) result by default. +}); +``` + +Loading workers use `processLoadingTask` the same way. + +> `adapter.emit(...)` is now **`protected`** — calling `adapter.emit(...)` or +> `this.adapter.emit(...)` is a hard **compile error** in v2, not a +> deprecation. Every emit site must be converted. + +### `emit()` → `return` translation table + +| v1 emit call | v2 return | +|--------------|-----------| +| `await adapter.emit(XxxDone)` | `return { status: 'success' }` | +| `await adapter.emit(XxxProgress, { progress })` | `return { status: 'progress' }` | +| `await adapter.emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` | +| `await adapter.emit(XxxError, { error })` | `return { status: 'error', error }` | +| `await adapter.emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to repo + `return { status: 'success' }` — see §7 | + +`progress` (`{ progress: n }`) carried no semantic value in v1 beyond "not done +yet" and is dropped; the platform tracks progress itself. + +### Status → emitted event, per phase + +The SDK picks the platform event from the **current phase** and the returned +status: + +| status | Resumable phases — data/attachment **extraction**, data/attachment **loading** | Non-resumable — external sync units, metadata, state deletion | +|--------|------------------------------------------------------------------------------|---------------------------------------------------------------| +| `'success'` | `*Done` | `*Done` | +| `'progress'` | `*Progress` (continuation) | **`*Error`** (illegal for these phases; emitted with a generated message) | +| `'delay'` | `*Delayed` (with `delaySeconds`) | **`*Error`** (illegal) | +| `'error'` | `*Error` (with the error record) | `*Error` | + +### Emits buried inside helper functions + +A common v1 pattern is emitting deep inside a helper and returning a boolean to +tell the caller to stop. Since only the **task's return value** reaches the SDK +in v2, the helper must **bubble the outcome up** instead. + +A clean way (used by the migrated google-drive connector) is to store the +terminal result on a field and return it once the loop unwinds: + +```ts +// v1 — helper emits, returns false to abort +private async handlePermissionDeniedError(error: unknown) { + await this.adapter.emit(ExtractorEventType.DataExtractionError, { + error: { message: '...' }, + }); +} +``` + +```ts +// v2 — helper records the result; the task returns it +private result: TaskResult | undefined; + +private handlePermissionDeniedError(error: unknown) { + this.result = { status: 'error', error: { message: '...' } }; +} + +// ... and where the loop ends: +private finalResult(): TaskResult { + // a stored error/delay set by a helper, else progress (continuation) + return this.result ?? { status: 'progress' }; +} +``` + +For a simple "stop iterating" signal, return the `TaskResult` directly up the +call chain: + +```ts +// v2 +async function extractList(adapter: ExtractionAdapter): Promise { + if (rateLimited) return { status: 'delay', delaySeconds: retryAfter }; + // ... + return null; // keep going +} + +processExtractionTask({ + task: async ({ adapter }) => { + for (const itemType of itemTypes) { + const stop = await extractList(adapter); + if (stop) return stop; + } + return { status: 'success' }; + }, +}); +``` + +### Timeout handling + +Checking `adapter.isTimeout` in your extraction loop still works as in v1 — but +instead of emitting progress and exiting, **return** progress: + +```ts +// v2 +if (adapter.isTimeout) { + return { status: 'progress' }; // platform sends CONTINUE_* next +} +``` + +Two important behaviors: + +- **The timeout outcome always wins.** Once the soft timeout has fired, the SDK + emits the `onTimeout` result (or the default `progress`) and **ignores + whatever the task returned**, by design: a phase that ran out of time must + hand off for continuation, not report itself complete. Put any cleanup that + must survive a timeout in `onTimeout`, not in the task body. +- **`onTimeout` is optional**, but its default is `{ status: 'progress' }`. + That is correct for resumable phases. For **non-resumable** phases (external + sync units, metadata, state deletion), `progress` is illegal and is emitted + as an **error** — so provide an explicit `onTimeout` there to control the + message: + + ```ts + onTimeout: async () => ({ + status: 'error', + error: { message: 'Failed to extract metadata. Lambda timeout.' }, + }), + ``` + +> Do **not** call `process.exit()` yourself in v2. v1 helpers sometimes called +> `process.exit(0/1)` after emitting; the v2 SDK owns the single worker exit +> after it emits your `TaskResult`. + +## 5. `WorkerAdapter` → `ExtractionAdapter` / `LoadingAdapter` + +The `WorkerAdapter` **class** is gone. Replace the type annotation on your +helpers with the mode-specific adapter: + +```ts +// v1 +async function extractList(adapter: WorkerAdapter) { ... } +// v2 +import { ExtractionAdapter } from '@devrev/airsync-sdk'; +async function extractList(adapter: ExtractionAdapter) { ... } +``` + +| Surface | Lives on | +|---------|----------| +| `initializeRepos`, `getRepo`, `streamAttachments`, `shouldExtract`, `artifacts` | `ExtractionAdapter` | +| `loadItemTypes`, `loadAttachments`, `mappers`, `reports`, `processedFiles` | `LoadingAdapter` | +| `event`, `state`, `sdkState`, `postState`, `isTimeout`, `extractionScope` | both (shared `BaseAdapter`) | + +> Only the **class** was removed. The **types** `WorkerAdapterInterface` and +> `WorkerAdapterOptions` still exist and are still exported — don't blindly +> rename every `WorkerAdapter` token; replace the `WorkerAdapter` +> annotations only. + +> **`mappers` / `reports` / `processedFiles` moved to `LoadingAdapter` only.** +> In v1 they were on the single `WorkerAdapter` and technically reachable in any +> phase. In v2 they are not on `ExtractionAdapter`. Code that touched +> `adapter.mappers` (etc.) during an extraction phase must move to the loading +> path. + +## 6. Loading & attachment methods return a `TaskResult` + +`loadItemTypes`, `loadAttachments`, and `streamAttachments` no longer emit or +exit mid-flight — they **return** a `TaskResult` you pass straight through. +Rate limits (→ `delay`), timeouts (→ `progress`), errors (→ `error`), and +completion (→ `success`) are all encoded in the result; `reports` / +`processed_files` (loading) and artifacts (extraction) are attached to the +emitted event automatically. + +### Loading — before (v1) + +```ts +import { LoaderEventType, processTask } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + await adapter.loadItemTypes({ itemTypesToLoad }); + await adapter.emit(LoaderEventType.DataLoadingDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(LoaderEventType.DataLoadingProgress); + }, +}); +``` + +### Loading — after (v2) + +```ts +import { processLoadingTask } from '@devrev/airsync-sdk'; + +processLoadingTask({ + task: async ({ adapter }) => { + return adapter.loadItemTypes({ itemTypesToLoad }); + }, +}); +``` + +### Attachment streaming — before (v1) + +```ts +const response = await adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +if (response?.delay) { + await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { delay: response.delay }); +} else if (response?.error) { + await adapter.emit(ExtractorEventType.AttachmentExtractionError, { error: response.error }); +} else { + await adapter.emit(ExtractorEventType.AttachmentExtractionDone); +} +``` + +### Attachment streaming — after (v2) + +```ts +return adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +``` + +Custom attachment processors (reducer/iterator) are still supported with the +same call signatures; only their `adapter` parameter type changes from +`WorkerAdapter` to `ExtractionAdapter` (§5). The `getAttachmentStream` +function you implement still returns `{ httpStream }` / `{ delay }` / +`{ error }` — but see §9 for the `httpStream` type change. + +## 7. External sync units go through a repo + +In v1 the SDK accepted `external_sync_units` in the emit data and uploaded them +internally. With emit gone, push them to the `EXTERNAL_SYNC_UNITS` repo +yourself. The `EventData.external_sync_units` field — deprecated in v1 — has +been **removed entirely** in v2: there is no inline ESU path at all, and any +code still referencing `external_sync_units` in emit data is now a compile +error. External sync units leave the worker only as repo artifacts. + +### Before (v1) + +```ts +await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { + external_sync_units: externalSyncUnits, +}); +``` + +### After (v2) + +```ts +import { AirSyncDefaultItemTypes, processExtractionTask } from '@devrev/airsync-sdk'; + +adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + // mirror the batching the v1 SDK used internally for ESUs + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, +]); +await adapter.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push(externalSyncUnits); + +return { status: 'success' }; +``` + +The repo is uploaded automatically before the `Done` event is emitted. + +> If your connector already pushed ESUs to a repo in v1 (some do), this section +> is a no-op for you — just convert the trailing emit to `return`. + +## 8. State split: `adapter.state` vs `adapter.sdkState` + +In v1, connector state and SDK bookkeeping lived in one flat object, persisted +as one blob, and `adapter.state` exposed both. In v2: + +- **`adapter.state` is connector state only** — exactly the shape of the + `initialState` you pass to `spawn`. The getter and setter both survive, so + in-place reads/writes work unchanged: + + ```ts + // works identically in v1 and v2 + adapter.state[itemType].cursor = nextCursor; + adapter.state[itemType].complete = true; + ``` + +- **SDK bookkeeping** (`workersOldest`/`workersNewest`, `pendingWorkers*`, + `toDevRev`, `fromDevRev`, `snapInVersionId`) moved to **`adapter.sdkState`** + (read-only getter). +- On disk, state is persisted as a `{ connectorState, sdkState }` envelope. The + SDK **migrates a v1 flat blob automatically on first read** (recognized SDK + keys split into `sdkState`, the rest into `connectorState`), so in-flight + syncs survive the upgrade. + +> **`lastSyncStarted` / `lastSuccessfulSyncStarted` are gone.** Both fields were +> `@deprecated` in v1 and are **removed from `SdkState` in v2** — they are no +> longer set, persisted, or readable via `adapter.sdkState`. They were +> wall-clock sync-start timestamps; the SDK now resolves the incremental window +> from the extraction-data boundaries instead (`workersOldest`/`workersNewest` +> and the resolved `extract_from`/`extract_to`). The two old key names are still +> recognized by the v1-blob migration, so they are cleanly dropped from any old +> state rather than leaking into your connector state. + +### If your v1 connector mixed SDK fields into its own `State` interface + +Some connectors declared SDK-owned fields (e.g. `toDevRev`, +`lastSuccessfulSyncStarted`) on their hand-written `State` type and seeded them +in `getInitialState()`. Remove those from your interface and initial state — +`toDevRev`/`fromDevRev`/`workers*` are SDK-managed now, and +`lastSyncStarted`/`lastSuccessfulSyncStarted` no longer exist at all: + +```ts +// v1 — State interface mixing SDK fields (remove these) +export interface State { + // ... your fields ... + lastSyncStarted?: string; // removed in v2 — no replacement field + lastSuccessfulSyncStarted?: string; // removed in v2 — read workersNewest / extract_to instead + toDevRev?: ToDevRev; // also drop the dist/state/state.interfaces import +} +``` + +### Reading the incremental-sync window + +In v1 you read `lastSuccessfulSyncStarted` to decide an incremental cursor. That +field is gone (above). Read the window the SDK resolves for you instead: + +```ts +// preferred — the resolved window for this invocation, on the event context +const { extract_from, extract_to } = adapter.event.payload.event_context; + +// the persisted cross-sync high-water mark (committed at end of each cycle) +const lastExtractedTo = adapter.sdkState.workersNewest; +``` + +`extract_from`/`extract_to` are the per-invocation window the SDK computes from +the platform's `extraction_start_time`/`extraction_end_time`; +`workersOldest`/`workersNewest` are the persisted boundaries the SDK commits at +the end of a completed cycle and the true replacement for the old +"last successful sync" resume point. + +> **`AdapterState` is removed.** The deprecated flat +> `ConnectorState & SdkState` alias no longer exists. If you annotated anything +> `AdapterState`, drop the import — use your own connector `State` type for +> `adapter.state`, and read SDK fields from `adapter.sdkState` (or the event +> context). The v2 on-disk shape is the `AdapterStateEnvelope` +> (`{ connectorState, sdkState }`), which the SDK manages for you. + +> **Edge regression (old in-flight incremental syncs):** the SDK used to fall +> back to `lastSuccessfulSyncStarted` when resolving a `WORKERS_NEWEST` window +> on state that predated `workersNewest` (SDK **< 1.17.1**). With the field +> removed, that fallback is gone: such a window now resolves to the unbounded +> epoch, so an incremental sync still mid-flight across the upgrade re-extracts +> from the beginning **once**. The next completed cycle commits `workersNewest` +> and the sync self-heals; the platform deduplicates downstream. New syncs and +> any sync whose first full cycle completed on ≥ 1.17.1 are unaffected. Drain +> in-flight incremental syncs before upgrading if the one-time re-extract +> matters for your deployment. + +> **Edge case:** if your v1 connector state had a top-level key literally named +> `connectorState` or `sdkState`, the auto-migration mis-reads it (the envelope +> is detected by those key names). Rename such a key **before** upgrading. + +## 9. HTTP / axios surface removed from the SDK + +> This is the area the previously-published guide got **wrong** — it claimed +> the axios surface was "still exported." It is not. This was verified by +> typechecking a real connector against the v2 build. + +The SDK no longer re-exports any axios surface from its public entry point: + +| v1 export | v2 | +|-----------|----| +| `axios` (raw instance) | **removed** — import `axios` in your connector | +| `axiosClient` (retry-wrapped instance) | **removed from the public API** (still exists internally, but not exported and not at the old deep path) | +| `formatAxiosError` | **removed** (deleted from source) | +| `serializeAxiosError` | **removed from the public API** — use `serializeError` | +| `HTTPResponse` | **removed** | + +`axios` and `axios-retry` are still runtime **dependencies** of the SDK (so +they're available transitively), but you must construct your own client: + +### Before (v1) + +```ts +import { axios, axiosClient } from '@devrev/ts-adaas'; + +const res = await axiosClient.get(url, { responseType: 'stream' }); +``` + +### After (v2) + +```ts +import axios from 'axios'; +import axiosRetry from 'axios-retry'; + +const axiosClient = axios.create(); +axiosRetry(axiosClient, { retries: 5, retryDelay: axiosRetry.exponentialDelay }); + +const res = await axiosClient.get(url, { responseType: 'stream' }); +``` + +> If you previously deep-imported `@devrev/ts-adaas/dist/http/axios-client`, +> that path **no longer exists** (the file was renamed to `http/client` and is +> internal). Bring your own axios instance as above. + +For error logging, replace `formatAxiosError` / `serializeAxiosError` with the +exported `serializeError`: + +```ts +import { serializeError } from '@devrev/airsync-sdk'; +console.error(serializeError(error)); +``` + +### `httpStream` type changed + +The connector-implemented attachment-stream function returns +`ExternalSystemAttachmentStreamingResponse`, whose `httpStream` field changed +from axios's `AxiosResponse` to the new public `HttpStreamResponse` +(`{ data: any; headers: Record }`). An axios stream response still +satisfies it structurally, but if you annotated the stream with `AxiosResponse` +imported *from the SDK*, switch the annotation to `HttpStreamResponse` (or +import `AxiosResponse` from `axios` directly). + +## 10. `Mappers` methods return the unwrapped body + +`Mappers.getByTargetId` / `getByExternalId` / `create` / `update` changed their +return type from `Promise>` to `Promise` — they now return +the response **body** directly. Drop the `.data` access: + +### Before (v1) + +```ts +const mapperResponse = await this.mappers.getByTargetId({ sync_unit, target }); +const resolvedId = mapperResponse.data.sync_mapper_record.external_ids[0]; +``` + +### After (v2) + +```ts +const mapperResponse = await this.mappers.getByTargetId({ sync_unit, target }); +const resolvedId = mapperResponse.sync_mapper_record.external_ids[0]; +``` + +This is a **silent** change for code that reads `.data` (it will fail to +type-check, or read `undefined` if loosely typed). The `Mappers` class is also +now exported from the package root, so you no longer need to deep-import it from +`dist/mappers/mappers`. + +## 11. Deleted legacy modules + +Everything under the v1 `deprecated/` tree is gone: + +| Removed | Replacement | +|---------|-------------| +| `Adapter`, `createAdapter` | `ExtractionAdapter` / `LoadingAdapter` + `processExtractionTask` / `processLoadingTask` | +| `DemoExtractor` | — (reference implementation only) | +| `HTTPClient`, `defaultResponse` | your own axios client (§9) | +| deprecated `Uploader` | repos (`initializeRepos` / `getRepo` / `push`) | + +(The SDK has an internal `Uploader` class, but it was never part of the public +API in either version — the *public* v1 `Uploader` was the deprecated one.) + +## 12. Deleted deprecated enum members, types & enums + +The old/new duplicate enum members were collapsed; only the modern names +remain. **The string values of the surviving members are byte-identical to +v1**, so nothing changes on the wire — only the TypeScript member names. + +> ⚠️ The **deleted** `EventType` / `ExtractorEventType` members carried +> *different, older* string values than their replacements (e.g. v1 +> `ExtractionDataStart = 'EXTRACTION_DATA_START'` vs the surviving +> `StartExtractingData = 'START_EXTRACTING_DATA'`). The modern members already +> existed in v1 with the modern values, so survivors are wire-compatible — but +> a deleted member and its replacement did **not** share a string value. + +**`EventType` (incoming):** + +| Deleted (v1 deprecated) | Use instead | +|--------------------------|-------------| +| `ExtractionExternalSyncUnitsStart` | `StartExtractingExternalSyncUnits` | +| `ExtractionMetadataStart` | `StartExtractingMetadata` | +| `ExtractionDataStart` | `StartExtractingData` | +| `ExtractionDataContinue` | `ContinueExtractingData` | +| `ExtractionDataDelete` | `StartDeletingExtractorState` | +| `ExtractionAttachmentsStart` | `StartExtractingAttachments` | +| `ExtractionAttachmentsContinue` | `ContinueExtractingAttachments` | +| `ExtractionAttachmentsDelete` | `StartDeletingExtractorAttachmentsState` | + +**`ExtractorEventType` (outgoing):** the `Extraction*`-prefixed members +(`ExtractionDataDone`, `ExtractionDataDelay`, `ExtractionAttachmentsProgress`, +…) are deleted; use the `*Extraction*` members (`DataExtractionDone`, +`DataExtractionDelayed`, `AttachmentExtractionProgress`, …). In practice you'll +rarely reference `ExtractorEventType` at all in v2 — see §4. + +**`LoaderEventType`:** the duplicate members `DataLoadingDelay` and +`AttachmentsLoading*` (plural) are deleted; use `DataLoadingDelayed` and +`AttachmentLoading*` (singular). These duplicates shared the same string value +as their survivors, so removing them is a pure source-name change. + +> **No more incoming-event-type translation.** v1 shipped an +> `event-type-translation` module that mapped legacy platform strings onto the +> modern enum members (and translated outgoing types). v2 removed it entirely +> and passes `event_context.event_type` through untouched. Any connector +> importing `translateIncomingEventType` / `translateOutgoingEventType` / +> `translateExtractorEventType` / `translateLoaderEventType` from the SDK will +> fail to compile — drop them; the platform sends modern strings. + +### Removed duplicate `UnknownEventType` members + +`UnknownEventType = 'UNKNOWN_EVENT_TYPE'` was declared on three enums in v1. +The copies on `EventType` and `ExtractorEventType` are **removed**; the +`LoaderEventType.UnknownEventType` member (the one the SDK actually uses as its +"unrecognized event" sentinel) **stays**. If you matched on +`EventType.UnknownEventType` or `ExtractorEventType.UnknownEventType`, switch to +`LoaderEventType.UnknownEventType` or compare against the raw +`'UNKNOWN_EVENT_TYPE'` string (the value is unchanged). + +### Removed deprecated types & enums + +These were `@deprecated` in v1 and are **deleted from the public API** in v2. +None are referenced by the modern worker contract; each row gives the +replacement (or "no replacement" where the concept is gone): + +| Removed | Replacement | +|---------|-------------| +| `ExtractionMode` (enum) | `SyncMode` (adds `LOADING` alongside `INITIAL`/`INCREMENTAL`) | +| `EventContextIn` (interface) | `EventContext` (the single, current event-context type) | +| `EventContextOut` (interface) | `EventContext` | +| `DomainObjectState` (interface) | — (no replacement; was an unused per-object state shape) | +| `ErrorLevel` (enum) | — (logger uses its own internal log level) | +| `LogRecord` (interface) | — (unused) | +| `AdapterUpdateParams` (interface) | — (unused) | +| `AdapterState` (type alias) | your connector `State` for `adapter.state`; `adapter.sdkState` for SDK fields (§8) | + +Also removed from `EventData`: the deprecated `external_sync_units` field (§7) +and the deprecated `progress` field (a no-op since v1 — the backend computes +progress). The `artifacts` field on `EventData` is **kept** — it is how the SDK +attaches uploaded repo artifacts (including external sync units) to the emitted +event. + +## 13. Deep-import paths that moved or broke + +The compiled `dist/` mirrors `src/` 1:1, so a deep import keeps working only if +the underlying source file still lives at the same relative path. Status of the +paths real connectors used: + +| Deep import | Status in v2 | +|-------------|--------------| +| `dist/http/axios-client` (`axiosClient`) | ❌ **broken** — file renamed to `http/client` and made internal. Bring your own axios (§9). | +| `dist/state/state.interfaces` (`ToDevRev`) | ⚠️ still resolves, but `ToDevRev` is SDK-internal now (§8) — **drop** the import, don't repoint it | +| `dist/repo/repo.interfaces` (`Item`) | ✅ resolves — but `Item` is now on the root barrel, prefer the root import | +| `dist/types/loading` (`ItemTypeToLoad`) | ✅ resolves — also now on the root barrel | +| `dist/mappers/mappers` (`Mappers`) | ✅ resolves — also now on the root barrel (and note the return-type change, §10) | +| `dist/types/extraction` (`InitialSyncScope`) | ✅ resolves — also on the root barrel | + +Also: `mappers/mappers.interface` was renamed to `mappers.interfaces` +(singular → plural), so a deep import of `dist/mappers/mappers.interface` +breaks even though every symbol inside is unchanged. + +**Recommended:** replace all deep `dist/**` imports with root imports. If a +symbol you need isn't on the root barrel, request it rather than deep-importing. + +## 14. Edge regression: very old in-flight attachment syncs + +In v1 the SDK migrated the legacy `string[]` form of the processed-attachments +dedup list (`lastProcessedAttachmentsIdsList`) to the current +`{ id, parent_id }[]` form on read. v2 removed that conversion. + +The `string[]` form only exists in state written by SDK **< 1.15.2**. If an +attachment-extraction phase started on a pre-1.15.2 SDK and is **still +mid-flight** when the connector upgrades to v2, the v2 dedup check +(`it.id === …`) won't match the bare-string entries, so attachments already +downloaded in that sync get re-uploaded once. New syncs — and any sync started +on ≥ 1.15.2 — are unaffected. The platform deduplicates downstream, so the only +cost is the wasted re-download/upload on that one continuation. + +If this matters for your deployment, drain in-flight attachment syncs before +upgrading. + +## 15. `spawn`'s deprecated `workerPath` option removed + +`SpawnFactoryInterface.workerPath` was `@deprecated` in v1 and is **removed** in +v2. Point `spawn` at your worker directory with `baseWorkerPath: __dirname` +instead — the SDK resolves the per-event worker file from there +(`workerPathOverrides` still works for custom paths). + +### Before (v1) + +```ts +spawn({ event, initialState, workerPath: __dirname + '/workers/data-extraction' }); +``` + +### After (v2) + +```ts +spawn({ event, initialState, baseWorkerPath: __dirname }); +``` + +--- + +## Migration checklist + +1. `npm uninstall @devrev/ts-adaas && npm install @devrev/airsync-sdk`; replace the import specifier everywhere. +2. Rename `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage`. Drop any `CustomAirdropEvent` cast that only added `user_id`/`dev_oid`/`source_id`/`service_account_id` (§2). +3. Split workers: extraction files use `processExtractionTask`, loading files use `processLoadingTask` (§3). +4. Convert **every** `adapter.emit(...)` into a returned `TaskResult`; bubble outcomes up from helpers (§4). +5. Replace `WorkerAdapter` annotations with `ExtractionAdapter` / `LoadingAdapter` (§5). Move any `mappers`/`reports`/`processedFiles` access into the loading path. +6. Pass the `TaskResult` straight through from `loadItemTypes` / `loadAttachments` / `streamAttachments` (§6). +7. ESU workers: push external sync units to the `EXTERNAL_SYNC_UNITS` repo; remove any `external_sync_units` (and `progress`) from emit data — those fields are gone from `EventData` (§7, §12). +8. Remove SDK-owned fields from your `State` interface and `getInitialState`; `lastSyncStarted`/`lastSuccessfulSyncStarted` are gone — read incremental cursors from `event_context.extract_from`/`extract_to` or `adapter.sdkState.workersNewest` (§8). +9. Replace `axios` / `axiosClient` / `formatAxiosError` / `serializeAxiosError` SDK imports with your own axios client + `serializeError` (§9). +10. Drop `.data` from `adapter.mappers.*` result reads (§10). +11. Remove usage of deleted legacy modules and event-type-translation helpers (§11, §12). +12. Replace deleted enum members with their modern names — values unchanged; drop any use of the removed deprecated types (`ExtractionMode`, `EventContextIn`/`Out`, `DomainObjectState`, `ErrorLevel`, `LogRecord`, `AdapterUpdateParams`, `AdapterState`) (§12). +13. Replace deep `dist/**` imports with root imports; drop the now-internal `ToDevRev` import (§13). +14. Decide per worker whether to keep an explicit `onTimeout` (recommended for ESU / metadata / state-deletion phases to control the error message; can be omitted for resumable phases) (§4). +15. Replace `spawn({ workerPath })` with `spawn({ baseWorkerPath: __dirname })` (§15). +16. Update jest mocks of the SDK module — they hardcode the v1 shape (`processTask`, `WorkerAdapter`, old enum members, `axiosClient`). + +## A note on `2.0.0-beta.0` + +An early beta (`2.0.0-beta.0`) still re-exported `axios` / `axiosClient`. The +GA release removed them (§9). If you migrated a connector against the beta and +imported either symbol from `@devrev/airsync-sdk`, it will fail to compile +against GA — apply §9. diff --git a/README.md b/README.md index 11f7b2c5..84898bf6 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,5 @@ It provides features such as: ## Installation ```bash -npm install @devrev/ts-adaas +npm install @devrev/airsync-sdk ``` - -## Reference - -Please refer to the [REFERENCE.md](./REFERENCE.md) file for more information on the types, interfaces and functions used in the library. diff --git a/REFERENCE.md b/REFERENCE.md deleted file mode 100644 index b949a3b4..00000000 --- a/REFERENCE.md +++ /dev/null @@ -1,1337 +0,0 @@ -## Reference - -### `SdkState` interface - -Defines the base state structure used by the Airdrop SDK. - -'SdkState' is an internal member that is not exported. - -#### Properties - -- _lastSyncStarted_ - - Optional. A **string** representing the timestamp when the last sync operation started. **Deprecated** - use `extract_from` and `extract_to` from the event context instead, which are automatically resolved by the SDK from `extraction_start_time` and `extraction_end_time`. - -- _lastSuccessfulSyncStarted_ - - Optional. A **string** representing the timestamp when the last successful sync operation started. **Deprecated** - use `extract_from` and `extract_to` from the event context instead, which are automatically resolved by the SDK from `extraction_start_time` and `extraction_end_time`. - -- _pendingWorkersOldest_ - - Optional. A **string** representing the pending (not yet committed) oldest extraction boundary as an ISO 8601 timestamp. Set on `StartExtractingData`, reused across subsequent phases, cleared on `AttachmentExtractionDone`. - -- _pendingWorkersNewest_ - - Optional. A **string** representing the pending (not yet committed) newest extraction boundary as an ISO 8601 timestamp. Set on `StartExtractingData`, reused across subsequent phases, cleared on `AttachmentExtractionDone`. - -- _workersOldest_ - - Optional. A **string** representing the oldest point of extraction as an ISO 8601 timestamp. - -- _workersNewest_ - - Optional. A **string** representing the newest point of extraction as an ISO 8601 timestamp. - -- _toDevRev_ - - Optional. An object of type **ToDevRev** containing data to be sent to DevRev. - -- _fromDevRev_ - - Optional. An object of type **FromDevRev** containing data received from DevRev. - -- _snapInVersionId_ - - Optional. A **string** representing the snap-in version ID. - -### `AdapterState` type - -A generic type that combines snap-in-specific state with the SDK's base state. - -#### Usage - -```typescript -type AdapterState = ConnectorState & SdkState; -``` - -The `AdapterState` type extends a snap-in's state type with additional fields from `SdkState`, providing a complete state structure to share with Airdrop platform. - -### `ToDevRev` interface - -Provides additional information within the state that is available only during data synchronization to DevRev (extraction). - -#### Properties - -- _attachmentsMetadata_ - - _artifactIds_: An array of **strings** containing artifact IDs - - _lastProcessed_: A **number** which is the index of the last processed attachment from the array - - _lastProcessedAttachmentsIdsList_: Optional. An array of **ProcessedAttachment** objects for deduplication on the SDK side - -### `FromDevRev` interface - -Provides additional information within the state that is available only during data synchronization from DevRev to external system (loading). - -#### Properties - -- _filesToLoad_ - - An array of **FileToLoad** objects representing files that need to be loaded. - -### `StateInterface` interface - -Defines the configuration structure for initializing state of the worker adapter. - -#### Properties - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _initialState_ - - Required. An object of type **ConnectorState** representing the initial state of the snap-in. - -- _initialDomainMapping_ - - Optional. An object of type **InitialDomainMapping** representing the initial domain mapping configuration. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions** for configuring the worker adapter. - -### `NormalizedItem` interface - -Represents the standardized structure of an item after normalization. - -#### Properties - -- _id_ - - Required. A **string** that uniquely identifies the normalized item. - -- _created_date_ - - Required. A **string** representing the timestamp, formatted as RFC3339, when the item was created. - -- _modified_date_ - - Required. A **string** representing the timestamp, formatted as RFC3339, when the item was last modified. - -- _data_ - - Required. An **object** containing the actual data of the normalized item. - -### `NormalizedAttachment` interface - -Represents the standardized structure of an attachment after normalization in the Airdrop platform. This interface defines the essential properties needed to identify and link attachments to their parent items. - -#### Properties - -- _url_ - - Required. A **string** representing the URL where the attachment can be accessed. - -- _id_ - - Required. A **string** that uniquely identifies the normalized attachment. - -- _file_name_ - - Required. A **string** representing the name of the attachment file. - -- _parent_id_ - - Required. A **string** identifying the parent item this attachment belongs to. - -- _author_id_ - - Optional. A **string** identifying the author or creator of the attachment. - -- _inline_ - - Optional. A **boolean** indicating whether the attachment is inline. - -- _content_type_ - - Optional. A **string** specifying the MIME type of the attachment (e.g. `'application/pdf'`, `'image/png'`). When provided, this takes precedence over the content type from the HTTP response header during streaming. Falls back to the HTTP `Content-Type` header, or `'application/octet-stream'` if neither is available. - -- _grand_parent_id_ - - Optional. A **number** or **string** identifying a higher-level parent entity, if applicable. - -#### Example - -```typescript -const normalizedAttachment: NormalizedAttachment = { - url: 'https://example.com/files/document.pdf', - id: 'att_123456', - file_name: 'document.pdf', - parent_id: 'task_789', - author_id: 'user_456', - inline: false, - content_type: 'application/pdf', - grand_parent_id: 1001, -}; -``` - -### `RepoInterface` interface - -Defines the structure of a repo which is used to store and upload extracted data. This interface provides the basic structure for repositories that handle data extraction and normalization. - -#### Properties - -- _itemType_ - - Required. A **string** that specifies the type of items stored in this repository. - -- _normalize_ - - Optional. A **function** that takes an object and returns either a **NormalizedItem** or **NormalizedAttachment**. This function is responsible for transforming raw data into a standardized format. - -- _overridenOptions_ - - Optional. An object of type **WorkerAdapterOptions** that overrides the default options for this specific repo. - -#### Example - -```typescript -const taskRepo: RepoInterface = { - itemType: 'tasks', - normalize: (rawTask) => ({ - id: rawTask.id, - created_date: rawTask.created_at, - modified_date: rawTask.updated_at, - data: rawTask, - }), -}; -``` - -### `ExternalSyncUnit` interface - -Represents an external sync unit (such as repositories, projects, etc.) that can be extracted. This interface defines the structure for organizing and identifying extractable units of data. - -#### Properties - -- _id_ - - Required. A **string** that uniquely identifies the external sync unit. - -- _name_ - - Required. A **string** representing the name of the external sync unit. - -- _description_ - - Required. A **string** providing a description of the external sync unit. - -- _item_count_ - - Optional. A **number** indicating the total count of items in this external sync unit. - -- _item_type_ - - Optional. A **string** specifying the type of items contained in this external sync unit. - -### `EventContext` interface - -Defines the structure of the event context that is sent to the external connector from Airdrop. - -#### Properties - -- _callback_url_ - - Required. A **string** representing the callback URL. - -- _dev_org_ - - Required. A **string** representing the organization ID. **Deprecated** - use `dev_oid` instead. - -- _dev_oid_ - - Required. A **string** representing the organization ID. - -- _dev_org_id_ - - Required. A **string** representing the organization ID. - -- _dev_user_ - - Required. A **string** representing the user ID. **Deprecated** - use `dev_uid` instead. - -- _dev_user_id_ - - Required. A **string** representing the user ID. **Deprecated** - use `dev_uid` instead. - -- _dev_uid_ - - Required. A **string** representing the user ID. - -- _event_type_adaas_ - - Required. A **string** representing the event type in ADaaS. - -- _external_sync_unit_ - - Required. A **string** representing the external sync unit ID. **Deprecated** - use `external_sync_unit_id` instead. - -- _external_sync_unit_id_ - - Required. A **string** representing the external sync unit ID. - -- _external_sync_unit_name_ - - Required. A **string** representing the external sync unit name. - -- _external_system_ - - Required. A **string** representing the external system. **Deprecated** - use `external_system_id` instead. - -- _external_system_id_ - - Required. A **string** representing the external system ID. - -- _external_system_name_ - - Required. A **string** representing the external system name. - -- _external_system_type_ - - Required. A **string** representing the external system type. - -- _extract_from_ - - Optional. A **string** representing the resolved start timestamp of extraction in ISO 8601 format. Automatically computed by the SDK from `extraction_start_time` and worker state. This is the field developers should read to know when to start extracting from. - -- _extract_to_ - - Optional. A **string** representing the resolved end timestamp of extraction in ISO 8601 format. Automatically computed by the SDK from `extraction_end_time` and worker state. This is the field developers should read to know when to stop extracting at. - -- _extraction_start_time_ - - Optional. An object of type **TimeValue** representing the start time value for extraction as sent by the platform. The SDK resolves this into a concrete ISO 8601 timestamp on `extract_from`. - -- _extraction_end_time_ - - Optional. An object of type **TimeValue** representing the end time value for extraction as sent by the platform. The SDK resolves this into a concrete ISO 8601 timestamp on `extract_to`. - -- _import_slug_ - - Required. A **string** representing the import slug. - -- _initial_sync_scope_ - - Optional. An enum **InitialSyncScope** representing the scope of the initial sync (can be 'full-history' or 'time-scoped'). - -- _mode_ - - Required. A **string** representing the mode (can be 'INITIAL', 'INCREMENTAL', or 'LOADING'). - -- _request_id_ - - Required. A **string** representing the request ID. - -- _request_id_adaas_ - - Required. A **string** representing the ADaaS request ID. - -- _reset_extraction_ - - Optional. A **boolean** signifying the incremental sync should start from the given `extract_from` timestamp if true or from `lastSuccessfulSyncStarted` timestamp if false. **Deprecated** - use `reset_extract_from` instead. - -- _reset_extract_from_ - - Optional. A **boolean** signifying the incremental sync should start from the given `extract_from` timestamp if true or from `lastSuccessfulSyncStarted` timestamp if false. **Deprecated** - use `extraction_start_time`/`extraction_end_time` instead, which are automatically resolved into `extract_from` and `extract_to`. - -- _run_id_ - - Required. A **string** representing the run ID. - -- _sequence_version_ - - Required. A **string** representing the sequence version. - -- _snap_in_slug_ - - Required. A **string** representing the snap-in slug. - -- _snap_in_version_id_ - - Required. A **string** representing the snap-in version ID. - -- _sync_run_ - - Required. A **string** representing the sync run ID. **Deprecated** - use `run_id` instead. - -- _sync_run_id_ - - Required. A **string** representing the sync run ID. **Deprecated** - use `run_id` instead. - -- _sync_tier_ - - Required. A **string** representing the sync tier. - -- _sync_unit_ - - Required. A **string** representing the sync unit ID. - -- _sync_unit_id_ - - Required. A **string** representing the sync unit ID. - -- _uuid_ - - Required. A **string** representing the unique identifier. **Deprecated** - use `request_id_adaas` instead. - -- _worker_data_url_ - - Required. A **string** representing the worker data URL. - -### `AirdropEvent` interface - -Defines the structure of events sent to external extractors from Airdrop platform. This interface encapsulates all necessary information for processing Airdrop events, including authentication, context, and payload data. - -#### Properties - -- _context_ - - Required. An object containing: - - _secrets_: An object containing: - - _service_account_token_: A **string** representing the DevRev authentication token for Airdrop platform - - _snap_in_version_id_: A **string** representing the version ID of the snap-in - - _snap_in_id_: A **string** representing the ID of the snap-in - -- _payload_ - - Required. An object of type **AirdropMessage** containing: - - _connection_data_: An object containing: - - _org_id_: A **string** representing the organization ID - - _org_name_: A **string** representing the organization name - - _key_: A **string** representing the key - - _key_type_: A **string** representing the key type - - _event_context_: An object of type [**EventContext**](#EventContext-interface) - - _event_type_: A value from the **EventType** enum (see `EventType` enum documentation below) - - _event_data_: Optional. An object that may contain: - - _external_sync_units_: Optional array of **ExternalSyncUnit** objects - - _progress_: Optional **number** indicating progress - - _error_: Optional error record - - _delay_: Optional **number** indicating delay - - _reports_: Optional array of loader reports - - _processed_files_: Optional array of **strings** representing processed files - - _stats_file_: Optional **string** representing stats file - -- _execution_metadata_ - - Required. An object containing: - - _devrev_endpoint_: A **string** representing the DevRev endpoint URL - -- _input_data_ - - Required. An object containing input data for snap-ins from '@devrev/typescript-sdk' - -### `EventData` interface - -Defines the structure of event data that is sent from the external extractor to Airdrop. This interface encapsulates various types of data that can be included in events, such as progress updates, errors, and processing results. - -#### Properties - -- _external_sync_units_ - - Optional. An array of **ExternalSyncUnit** objects representing external sync units to be processed. **Deprecated** - external sync units should be pushed to the `AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS` repo instead. - -- _progress_ - - Optional. A **number** indicating the progress of the current operation. **Deprecated** - progress is now calculated on the backend. - -- _error_ - - Optional. An object of type **ErrorRecord** containing error information if an error occurred. - -- _delay_ - - Optional. A **number** specifying a delay duration in seconds. - -- _artifacts_ - - Optional. An array of **Artifact** objects. **Deprecated** - should not be used directly. - -- _reports_ - - Optional. An array of **LoaderReport** objects representing loader reports. - -- _processed_files_ - - Optional. An array of **strings** representing processed file IDs. - -- _stats_file_ - - Optional. A **string** representing the stats file artifact ID. - -### `EventType` enum - -Defines the different types of events that can be sent to the external extractor from ADaaS. The external extractor uses these events to know what to do next in the extraction process. - -#### Extraction Events (preferred) - -- `StartExtractingExternalSyncUnits` = `'START_EXTRACTING_EXTERNAL_SYNC_UNITS'` -- `StartExtractingMetadata` = `'START_EXTRACTING_METADATA'` -- `StartExtractingData` = `'START_EXTRACTING_DATA'` -- `ContinueExtractingData` = `'CONTINUE_EXTRACTING_DATA'` -- `StartDeletingExtractorState` = `'START_DELETING_EXTRACTOR_STATE'` -- `StartExtractingAttachments` = `'START_EXTRACTING_ATTACHMENTS'` -- `ContinueExtractingAttachments` = `'CONTINUE_EXTRACTING_ATTACHMENTS'` -- `StartDeletingExtractorAttachmentsState` = `'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE'` - -#### Extraction Events (deprecated) - -- `ExtractionExternalSyncUnitsStart` **Deprecated** - use `StartExtractingExternalSyncUnits` -- `ExtractionMetadataStart` **Deprecated** - use `StartExtractingMetadata` -- `ExtractionDataStart` **Deprecated** - use `StartExtractingData` -- `ExtractionDataContinue` **Deprecated** - use `ContinueExtractingData` -- `ExtractionDataDelete` **Deprecated** - use `StartDeletingExtractorState` -- `ExtractionAttachmentsStart` **Deprecated** - use `StartExtractingAttachments` -- `ExtractionAttachmentsContinue` **Deprecated** - use `ContinueExtractingAttachments` -- `ExtractionAttachmentsDelete` **Deprecated** - use `StartDeletingExtractorAttachmentsState` - -#### Loading Events - -- `StartLoadingData` = `'START_LOADING_DATA'` -- `ContinueLoadingData` = `'CONTINUE_LOADING_DATA'` -- `StartLoadingAttachments` = `'START_LOADING_ATTACHMENTS'` -- `ContinueLoadingAttachments` = `'CONTINUE_LOADING_ATTACHMENTS'` -- `StartDeletingLoaderState` = `'START_DELETING_LOADER_STATE'` -- `StartDeletingLoaderAttachmentState` = `'START_DELETING_LOADER_ATTACHMENT_STATE'` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `ExtractorEventType` enum - -Defines the different types of events that can be sent from the external extractor to ADaaS. The external extractor uses these events to inform ADaaS about the progress of the extraction process. - -#### Extraction Events (preferred) - -- `ExternalSyncUnitExtractionDone` = `'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE'` -- `ExternalSyncUnitExtractionError` = `'EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR'` -- `MetadataExtractionDone` = `'METADATA_EXTRACTION_DONE'` -- `MetadataExtractionError` = `'METADATA_EXTRACTION_ERROR'` -- `DataExtractionProgress` = `'DATA_EXTRACTION_PROGRESS'` -- `DataExtractionDelayed` = `'DATA_EXTRACTION_DELAYED'` -- `DataExtractionDone` = `'DATA_EXTRACTION_DONE'` -- `DataExtractionError` = `'DATA_EXTRACTION_ERROR'` -- `ExtractorStateDeletionDone` = `'EXTRACTOR_STATE_DELETION_DONE'` -- `ExtractorStateDeletionError` = `'EXTRACTOR_STATE_DELETION_ERROR'` -- `AttachmentExtractionProgress` = `'ATTACHMENT_EXTRACTION_PROGRESS'` -- `AttachmentExtractionDelayed` = `'ATTACHMENT_EXTRACTION_DELAYED'` -- `AttachmentExtractionDone` = `'ATTACHMENT_EXTRACTION_DONE'` -- `AttachmentExtractionError` = `'ATTACHMENT_EXTRACTION_ERROR'` -- `ExtractorAttachmentsStateDeletionDone` = `'EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE'` -- `ExtractorAttachmentsStateDeletionError` = `'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR'` - -#### Extraction Events (deprecated) - -- `ExtractionExternalSyncUnitsDone` **Deprecated** - use `ExternalSyncUnitExtractionDone` -- `ExtractionExternalSyncUnitsError` **Deprecated** - use `ExternalSyncUnitExtractionError` -- `ExtractionMetadataDone` **Deprecated** - use `MetadataExtractionDone` -- `ExtractionMetadataError` **Deprecated** - use `MetadataExtractionError` -- `ExtractionDataProgress` **Deprecated** - use `DataExtractionProgress` -- `ExtractionDataDelay` **Deprecated** - use `DataExtractionDelayed` -- `ExtractionDataDone` **Deprecated** - use `DataExtractionDone` -- `ExtractionDataError` **Deprecated** - use `DataExtractionError` -- `ExtractionDataDeleteDone` **Deprecated** - use `ExtractorStateDeletionDone` -- `ExtractionDataDeleteError` **Deprecated** - use `ExtractorStateDeletionError` -- `ExtractionAttachmentsProgress` **Deprecated** - use `AttachmentExtractionProgress` -- `ExtractionAttachmentsDelay` **Deprecated** - use `AttachmentExtractionDelayed` -- `ExtractionAttachmentsDone` **Deprecated** - use `AttachmentExtractionDone` -- `ExtractionAttachmentsError` **Deprecated** - use `AttachmentExtractionError` -- `ExtractionAttachmentsDeleteDone` **Deprecated** - use `ExtractorAttachmentsStateDeletionDone` -- `ExtractionAttachmentsDeleteError` **Deprecated** - use `ExtractorAttachmentsStateDeletionError` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `LoaderEventType` enum - -Defines the different types of events that can be sent from the loader to ADaaS. - -#### Data Loading Events - -- `DataLoadingProgress` = `'DATA_LOADING_PROGRESS'` -- `DataLoadingDelayed` = `'DATA_LOADING_DELAYED'` -- `DataLoadingDone` = `'DATA_LOADING_DONE'` -- `DataLoadingError` = `'DATA_LOADING_ERROR'` -- `DataLoadingDelay` **Deprecated** - this was a typo, use `DataLoadingDelayed` instead - -#### Attachment Loading Events - -- `AttachmentLoadingProgress` = `'ATTACHMENT_LOADING_PROGRESS'` -- `AttachmentLoadingDelayed` = `'ATTACHMENT_LOADING_DELAYED'` -- `AttachmentLoadingDone` = `'ATTACHMENT_LOADING_DONE'` -- `AttachmentLoadingError` = `'ATTACHMENT_LOADING_ERROR'` - -#### Attachment Loading Events (deprecated aliases) - -- `AttachmentsLoadingProgress` **Deprecated** - use `AttachmentLoadingProgress` -- `AttachmentsLoadingDelayed` **Deprecated** - use `AttachmentLoadingDelayed` -- `AttachmentsLoadingDone` **Deprecated** - use `AttachmentLoadingDone` -- `AttachmentsLoadingError` **Deprecated** - use `AttachmentLoadingError` - -#### State Deletion Events - -- `LoaderStateDeletionDone` = `'LOADER_STATE_DELETION_DONE'` -- `LoaderStateDeletionError` = `'LOADER_STATE_DELETION_ERROR'` -- `LoaderAttachmentStateDeletionDone` = `'LOADER_ATTACHMENT_STATE_DELETION_DONE'` -- `LoaderAttachmentStateDeletionError` = `'LOADER_ATTACHMENT_STATE_DELETION_ERROR'` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `SyncMode` enum - -Defines the different modes of sync that can be used by the external extractor. It can be either INITIAL, INCREMENTAL or LOADING. - -#### Values - -- `INITIAL` = `'INITIAL'` - Used for the first/initial import -- `INCREMENTAL` = `'INCREMENTAL'` - Used for doing syncs -- `LOADING` = `'LOADING'` - Used for loading data from DevRev to the external system - -### `ExtractionMode` enum **Deprecated** - -Defines the different modes of extraction. Use `SyncMode` instead. - -#### Values - -- `INITIAL` = `'INITIAL'` -- `INCREMENTAL` = `'INCREMENTAL'` - -### `InitialSyncScope` enum - -Defines the different scopes of initial sync that can be used by the external extractor. - -#### Values - -- `FULL_HISTORY` = `'full-history'` -- `TIME_SCOPED` = `'time-scoped'` - -### `TimeUnit` enum - -Defines the supported Go duration units for time window calculations. These correspond directly to Go's `time.ParseDuration` units. - -#### Values - -- `NANOSECONDS` = `'ns'` -- `MICROSECONDS` = `'us'` -- `MICROSECONDS_MU` = `'µs'` -- `MILLISECONDS` = `'ms'` -- `SECONDS` = `'s'` -- `MINUTES` = `'m'` -- `HOURS` = `'h'` - -### `TimeValueType` enum - -Defines the type of a time value used in extraction start/end times. The platform sends these types to indicate how the extraction time should be resolved by the SDK. - -#### Values - -- `WORKERS_OLDEST` = `'workers_oldest'` - Oldest timestamp from worker state -- `WORKERS_OLDEST_MINUS_WINDOW` = `'workers_oldest_minus_window'` - Oldest timestamp from worker state minus a duration window -- `WORKERS_NEWEST` = `'workers_newest'` - Newest timestamp from worker state -- `WORKERS_NEWEST_PLUS_WINDOW` = `'workers_newest_plus_window'` - Newest timestamp from worker state plus a duration window -- `CURRENT_TIME` = `'current_time'` - Current time -- `ABSOLUTE_TIME` = `'absolute_time'` - User-specified absolute timestamp -- `UNBOUNDED` = `'unbounded'` - No bound, extract all available data - -### `TimeValue` interface - -Represents a time value used in extraction start/end times. - -#### Properties - -- _type_ - - Required. A **TimeValueType** enum value which denotes how the value should be resolved. - -- _value_ - - Optional. A **string** whose meaning depends on the type: - - For `ABSOLUTE_TIME`: an ISO 8601 timestamp - - For `*_WINDOW` types: a Go duration string (e.g. `'500ms'`, `'30s'`, `'5m'`, `'2h'`) - - For other types: not used - -### `ExtractionScope` type - -Represents the parsed extraction scope from the platform. Each key is an item type name, and the value indicates whether it should be extracted. - -#### Usage - -```typescript -type ExtractionScope = Record; -``` - -### `ExtractionCommonError` const enum - -Provides predefined error codes for common extraction errors. - -#### Values - -- `EXTERNAL_SYNC_UNIT_DELETED` = `'ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED'` -- `EXTERNAL_SYNC_UNIT_DEACTIVATED` = `'ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED'` -- `USER_DELETED` = `'ERROR_CODE=USER_DELETED'` - -### `AirSyncDefaultItemTypes` enum - -Defines the default item types used by the SDK. - -#### Values - -- `EXTERNAL_DOMAIN_METADATA` = `'external_domain_metadata'` -- `ATTACHMENTS` = `'attachments'` -- `EXTERNAL_SYNC_UNITS` = `'external_sync_units'` - -### `UNBOUNDED_DATE_TIME_VALUE` constant - -Sentinel value representing an unbounded (no limit) extraction timestamp. Used as the resolved value for `TimeValueType.UNBOUNDED`. Its value is `'1970-01-01T00:00:00.000Z'`. - -### `spawn` function - -This function initializes a new worker thread and oversees its lifecycle. It should be invoked when the snap-in receives a message from the Airdrop platform. The worker script provided then handles the event accordingly. - -#### Usage - -```typescript -spawn({ event, initialState, options, baseWorkerPath }); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _initialState_ - - Required. Object of **any** type that represents the initial state of the snap-in. - -- _workerPath_ - - Optional. A **string** that represents the path to the worker file. **Deprecated** - use `baseWorkerPath` instead. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions**, which will be passed to the newly created worker. This worker will then initialize a `WorkerAdapter` by invoking the `processTask` function. The options include: - - `isLocalDevelopment` - - A **boolean** flag. If set to `true`, intermediary files containing extracted data will be stored on the local machine, which is useful during development. The default value is `false`. - - - `timeout` - - A **number** that specifies the timeout duration for the lambda function, in milliseconds. The default is 10 minutes (10 \* 60 \* 1000 milliseconds). - - - `batchSize` - - A **number** that determines the maximum number of items to be processed and saved to an intermediary file before being sent to the Airdrop platform. The default batch size is 2,000. - - - `workerPathOverrides` - - Optional. A partial map of **EventType** to **string** paths, allowing you to override the default worker path for specific event types. - - - `skipConfirmation` - - Optional. A **boolean** flag. If set to `true`, skips artifact upload confirmation. - -- _initialDomainMapping_ - - Optional. An object of type **InitialDomainMapping** representing the initial domain mapping configuration. - -- _baseWorkerPath_ - - Optional. A **string** that represents the base path for the worker files, usually `__dirname`. When provided, the SDK automatically resolves the worker script based on the event type. - -#### Return value - -A **promise** that resolves once the worker has completed processing. - -#### Example - -```typescript -const run = async (events: AirdropEvent[]) => { - for (const event of events) { - await spawn({ - event, - initialState, - baseWorkerPath: __dirname, - }); - } -}; -``` - -### `processTask` function - -The `processTask` function retrieves the current state from the Airdrop platform and initializes a new `WorkerAdapter`. It executes the code specified in the `task` parameter, which contains the worker's functionality. If a timeout occurs, the function handles it by executing the `onTimeout` callback, ensuring the worker exits gracefully. Both functions receive an `adapter` parameter, representing the initialized `WorkerAdapter` object. - -#### Usage - -```typescript -processTask({ task, onTimeout }); -``` - -#### Parameters - -- _task_ - - Required. A **function** that defines the logic associated with the given event type. - -- _onTimeout_ - - Required. A **function** managing the timeout of the lambda invocation, including saving any necessary progress at the time of timeout. - -#### Example - -```typescript -// External sync units extraction -processTask({ - task: async ({ adapter }) => { - const httpClient = new HttpClient(adapter.event); - - const todoLists = await httpClient.getTodoLists(); - - const externalSyncUnits: ExternalSyncUnit[] = todoLists.map((todoList) => - normalizeTodoList(todoList) - ); - - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: externalSyncUnits, - }); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { - error: { - message: 'Failed to extract external sync units. Lambda timeout.', - }, - }); - }, -}); -``` - -### `Spawn` class - -`Spawn` class is responsible for spawning a new worker thread and managing the lifecycle of the worker. Provides utilities to emit control events to the platform and exit the worker gracefully. In case of lambda timeout, the class emits a lambda timeout event to the platform. - -#### Usage - -```typescript -new Spawn({ - event, - worker, - options, - resolve, - originalConsole, -}); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _worker_ - - Required. A Node worker of the **Worker** class, created with the createWorker function, which represents an independent JavaScript execution thread. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions**, which defines the options to create a new instance of Spawn class. - -- _resolve_ - - Required. A resolve **function** for the promise inside which the Spawn class was created. - -- _originalConsole_ - - Optional. A **Console** object representing the original console before the SDK logger replaces it. - -#### Example - -```typescript -new Promise((resolve) => { - new Spawn({ - event, - worker, - options, - resolve, - }); -}); -``` - -### `WorkerAdapter` class - -Used to interact with Airdrop platform. Provides utilities to emit events to the Airdrop platform, update the state of the snap-in and upload artifacts (files with data) to the platform. - -### Usage - -```typescript -new WorkerAdapter({ - event, - adapterState, - options, -}); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _adapterState_ - - Required. An object of type **State**, which represents the initial state of the adapter. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions** that specifies additional configuration options for the `WorkerAdapter`. This object is passed via the `spawn` function. - -#### Example - -```typescript -const adapter = new WorkerAdapter({ - event, - adapterState, - options, -}); -``` - -### `WorkerAdapter.state` property - -Getter and setter methods for working with the adapter state. - -### Usage - -```typescript -// get state -const adapterState = adapter.state; - -// set state -adapter.state = newAdapterState; -``` - -#### Example - -```typescript -export const initialState: ExtractorState = { - users: { completed: false }, - tasks: { completed: false }, - attachments: { completed: false }, -}; - -adapter.state = initialState; -``` - -### `WorkerAdapter.extractionScope` property - -Getter for the parsed extraction scope from the platform. Returns an `ExtractionScope` object. - -### Usage - -```typescript -const scope = adapter.extractionScope; -``` - -### `WorkerAdapter.shouldExtract` method - -Returns whether the given item type should be extracted. Defaults to `true` if the scope is empty or the item type is not listed. - -### Usage - -```typescript -adapter.shouldExtract(itemType); -``` - -#### Parameters - -- _itemType_ - - Required. A **string** representing the item type to check. - -#### Return value - -A **boolean** indicating whether the item type should be extracted. - -#### Example - -```typescript -if (adapter.shouldExtract('tasks')) { - // Extract tasks -} -``` - -### `WorkerAdapter.initializeRepos` method - -Initializes a `Repo` object for each item provided. - -### Usage - -```typescript -adapter.initializeRepos(repos); -``` - -#### Parameters - -- _repos_ - - Required. An array of objects of type `RepoInterface`. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function in the data extraction phase. - -```typescript -const repos = [ - { - itemType: 'tasks', - normalize: normalizeTask, - }, -]; - -adapter.initializeRepos(repos); -``` - -### `WorkerAdapter.getRepo` method - -Finds a Repo from the initialized repos. - -### Usage - -```typescript -adapter.getRepo(itemType); -``` - -#### Parameters - -- _itemType_ - - Required. A **string** that represents the itemType property for the searched repo. - -#### Return value - -An object of type **Repo** if the repo is found, otherwise **undefined**. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function. - -```typescript -// Push users to the repository designated for 'users' data. -await adapter.getRepo('users')?.push(users); -``` - -### `WorkerAdapter.emit` method - -Emits an event to the Airdrop platform. - -### Usage - -```typescript -adapter.emit( newEventType, data ): -``` - -#### Parameters - -- _newEventType_ - - Required. The event type to be emitted, of type **ExtractorEventType** or **LoaderEventType**. - -- _data_ - - Optional. An object of type **EventData** which represents the data to be sent with the event. - -#### Return value - -A **promise**, which resolves to undefined after the emit function completes its execution or rejects with an error. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function. - -```typescript -// Emitting successfully finished data extraction. -await adapter.emit(ExtractorEventType.DataExtractionDone); - -// Emitting a delay in attachments extraction phase. -await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { - delay: 10, -}); -``` - -### `WorkerAdapter.postState` method - -Saves the current adapter state to the Airdrop platform. - -### Usage - -```typescript -await adapter.postState(); -``` - -#### Return value - -A **promise** that resolves once the state has been posted. - -### `WorkerAdapter.mappers` property - -Provides access to the `Mappers` helper within the worker during loading. Use it to look up, create, or update sync mapper records that link external system items to DevRev items. - -#### Usage - -```typescript -// inside processTask({ task }) -await adapter.mappers.getByTargetId({ - sync_unit: adapter.event.payload.event_context.sync_unit, - target: devrevId, -}); -``` - -### `WorkerAdapter.reports` property - -Getter for the accumulated loader reports. Returns an array of **LoaderReport** objects. - -### `WorkerAdapter.processedFiles` property - -Getter for the list of processed file IDs. Returns an array of **strings**. - -### `WorkerAdapter.loadItemTypes` method - -Loads item types from DevRev to the external system during the loading phase. - -#### Usage - -```typescript -const response = await adapter.loadItemTypes({ itemTypesToLoad }); -``` - -#### Parameters - -- _itemTypesToLoad_ - - Required. An array of **ItemTypeToLoad** objects, each containing an `itemType` string, a `create` function, and an `update` function. - -#### Return value - -A **promise** resolving to a **LoadItemTypesResponse** containing `reports` and `processed_files`. - -### `WorkerAdapter.loadAttachments` method - -Loads attachments from DevRev to the external system during the loading phase. - -#### Usage - -```typescript -const response = await adapter.loadAttachments({ create }); -``` - -#### Parameters - -- _create_ - - Required. A function of type **ExternalSystemLoadingFunction\** that creates the attachment in the external system. - -#### Return value - -A **promise** resolving to a **LoadItemTypesResponse** containing `reports` and `processed_files`. - -### `WorkerAdapter.streamAttachments` method - -Streams attachments to the DevRev platform during the attachment extraction phase. Handles batching, deduplication, and progress tracking. - -#### Usage - -```typescript -await adapter.streamAttachments({ stream, processors, batchSize }); -``` - -#### Parameters - -- _stream_ - - Required. A function of type **ExternalSystemAttachmentStreamingFunction** that opens an HTTP stream for a given attachment. - -- _processors_ - - Optional. An object of type **ExternalSystemAttachmentProcessors** for custom attachment processing with `reducer` and `iterator` functions. - -- _batchSize_ - - Optional. A **number** specifying how many attachments to stream concurrently. Default is `1`, maximum is `50`. - -#### Return value - -A **promise** that resolves to a **StreamAttachmentsReturnType** (may contain `delay` or `error`), or `undefined` on success. - -### `WorkerAdapter.processAttachment` method - -Processes a single attachment: streams it from the external system, uploads it to DevRev, and records the SSOR attachment mapping. - -#### Usage - -```typescript -const result = await adapter.processAttachment(attachment, stream); -``` - -#### Parameters - -- _attachment_ - - Required. A **NormalizedAttachment** object representing the attachment to process. - -- _stream_ - - Required. A function of type **ExternalSystemAttachmentStreamingFunction** that returns the HTTP stream for the attachment. - -#### Return value - -A **promise** resolving to a **ProcessAttachmentReturnType** (may contain `error` or `delay`), or `undefined` on success. - ---- - -### `Mappers` class - -Manages sync mapper records that link external system items to DevRev items during loading. Access it via `adapter.mappers` inside your worker code. - -#### Methods - -- `getByTargetId(params)` - - **params**: `MappersGetByTargetIdParams` - - **returns**: `Promise>` - - Use when you know the DevRev ID and want the corresponding mapping. - -- `getByExternalId(params)` - - **params**: `MappersGetByExternalIdParams` - - **returns**: `Promise>` - - Use when you know an external ID and need the DevRev mapping. - -- `create(params)` - - **params**: `MappersCreateParams` - - **returns**: `Promise>` - - Call after creating an item in the external system to persist the mapping. - -- `update(params)` - - **params**: `MappersUpdateParams` - - **returns**: `Promise>` - - Call after updating an item in the external system to add IDs, targets, or version markers. - -### `SyncMapperRecordStatus` enum - -Status of a sync mapper record indicating its operational state. - -#### Values - -- `OPERATIONAL` = `'operational'` - The mapping is active and operational (default) -- `FILTERED` = `'filtered'` - The mapping was filtered out by user filter settings -- `IGNORED` = `'ignored'` - The external object should be ignored in sync operations - -### `SyncMapperRecordTargetType` enum - -Types of DevRev entities that can be targets in sync mapper records. - -#### Values - -- `ACCESS_CONTROL_ENTRY` = `'access_control_entry'` -- `ACCOUNT` = `'account'` -- `AIRDROP_AUTHORIZATION_POLICY` = `'airdrop_authorization_policy'` -- `AIRDROP_FIELD_AUTHORIZATION_POLICY` = `'airdrop_field_authorization_policy'` -- `AIRDROP_PLATFORM_GROUP` = `'airdrop_platform_group'` -- `ARTICLE` = `'article'` -- `ARTIFACT` = `'artifact'` -- `CHAT` = `'chat'` -- `CONVERSATION` = `'conversation'` -- `CUSTOM_OBJECT` = `'custom_object'` -- `DIRECTORY` = `'directory'` -- `GROUP` = `'group'` -- `INCIDENT` = `'incident'` -- `LINK` = `'link'` -- `MEETING` = `'meeting'` -- `OBJECT_MEMBER` = `'object_member'` -- `PART` = `'part'` -- `REV_ORG` = `'rev_org'` -- `ROLE` = `'role'` -- `ROLE_SET` = `'role_set'` -- `TAG` = `'tag'` -- `TIMELINE_COMMENT` = `'timeline_comment'` -- `USER` = `'user'` -- `WORK` = `'work'` - -### `installInitialDomainMapping` function - -Installs the initial domain mapping for a snap-in. This creates recipe blueprints and installs domain mappings via the DevRev API. - -#### Usage - -```typescript -await installInitialDomainMapping(event, initialDomainMappingJson); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent**. - -- _initialDomainMappingJson_ - - Required. An object of type **InitialDomainMapping** containing: - - `starting_recipe_blueprint`: Optional object with the recipe blueprint configuration - - `additional_mappings`: Optional object with additional mapping configuration - -### `MockServer` class - -A lightweight HTTP mock server for local development and testing of connectors. Allows you to define routes with static responses or custom handlers. - -#### Exported types - -- **RequestInfo** - Information about a request received by the mock server (method, url, body) -- **RetryConfig** - Configuration for retry simulation behavior (failureCount, errorStatus, errorBody, headers, delay) -- **RouteConfig** - Configuration object for setting up a route response (path, method, status, body, headers, retry, delay) - -### `formatAxiosError` function - -Formats an Axios error into a structured object for logging. - -#### Usage - -```typescript -const formatted = formatAxiosError(error); -``` - -#### Parameters - -- _error_ - - Required. An **AxiosError** object. - -#### Return value - -An **object** with structured error information. - -### `serializeAxiosError` function - -Serializes an Axios error into a structured response object. - -#### Usage - -```typescript -const serialized = serializeAxiosError(error); -``` - -#### Parameters - -- _error_ - - Required. An **AxiosError** object. - -#### Return value - -An **AxiosErrorResponse** object with structured error details. diff --git a/jest.config.cjs b/jest.config.cjs index be2cf47a..fab696c9 100644 --- a/jest.config.cjs +++ b/jest.config.cjs @@ -11,7 +11,7 @@ module.exports = { 'src/tests/timeout-handling/.*.ts', // These tests are slow (10-15s per) 'src/tests/dummy-connector/metadata-extraction.test.ts', - 'src/http/axios-client-internal.test.ts', + 'src/http/client.test.ts', 'src/tests/event-data-size-limit/.*.test.ts', ], }, @@ -31,7 +31,7 @@ module.exports = { preset: 'ts-jest', testMatch: [ '/src/tests/dummy-connector/metadata-extraction.test.ts', - '/src/http/axios-client-internal.test.ts', + '/src/http/client.test.ts', '/src/tests/event-data-size-limit/size-limit-1.test.ts', ], }, diff --git a/package-lock.json b/package-lock.json index 9d56f7ed..254cd1a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "@devrev/ts-adaas", - "version": "1.19.7", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.4", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@devrev/ts-adaas", - "version": "1.19.7", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.4", "license": "ISC", "dependencies": { "@devrev/typescript-sdk": "^1.1.74", diff --git a/package.json b/package.json index 27d1e7fc..b8f8bf33 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { - "name": "@devrev/ts-adaas", - "version": "1.19.7", - "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.4", + "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.", "type": "commonjs", "main": "./dist/index.js", "typings": "./dist/index.d.ts", "scripts": { - "build": "tsc -p ./tsconfig.json", + "build": "tsc -p ./tsconfig.build.json", "prepare": "npm run build", "start": "ts-node src/index.ts", "lint": "eslint .", diff --git a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts index 9067c4f6..03472492 100644 --- a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts +++ b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts @@ -1,11 +1,9 @@ -import { - ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, -} from '../types'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; export interface AttachmentsStreamingPoolParams { - adapter: WorkerAdapter; + adapter: ExtractionAdapter; attachments: NormalizedAttachment[]; batchSize?: number; stream: ExternalSystemAttachmentStreamingFunction; diff --git a/src/attachments-streaming/attachments-streaming-pool.test.ts b/src/attachments-streaming/attachments-streaming-pool.test.ts index 604ba1d4..3c97881b 100644 --- a/src/attachments-streaming/attachments-streaming-pool.test.ts +++ b/src/attachments-streaming/attachments-streaming-pool.test.ts @@ -1,9 +1,9 @@ -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, ProcessAttachmentReturnType, -} from '../types'; +} from '../types/extraction'; import { AttachmentsStreamingPool } from './attachments-streaming-pool'; interface TestState { @@ -13,7 +13,7 @@ interface TestState { /* eslint-disable @typescript-eslint/no-explicit-any */ describe(AttachmentsStreamingPool.name, () => { - let mockAdapter: jest.Mocked>; + let mockAdapter: jest.Mocked>; let mockStream: jest.MockedFunction; let mockAttachments: NormalizedAttachment[]; @@ -22,6 +22,8 @@ describe(AttachmentsStreamingPool.name, () => { mockAdapter = { state: { attachments: { completed: false }, + }, + sdkState: { toDevRev: { attachmentsMetadata: { lastProcessedAttachmentsIdsList: [], @@ -103,7 +105,7 @@ describe(AttachmentsStreamingPool.name, () => { describe(AttachmentsStreamingPool.prototype.streamAll.name, () => { it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined as any; mockAdapter.processAttachment.mockResolvedValue({}); @@ -121,7 +123,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); }); @@ -169,11 +171,15 @@ describe(AttachmentsStreamingPool.name, () => { expect(result).toEqual({ delay: 5000 }); }); - it('should resume attachment extraction if it encounters old ids', async () => { - // Test migration from old string[] format to new ProcessedAttachment[] format - // Using 'as any' because we're intentionally testing legacy data format - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = - ['attachment-1', 'attachment-2'] as any; + it('should resume attachment extraction, skipping already-processed ids', async () => { + // Resume case: the state already records attachment-1 and attachment-2 as + // processed (in the v2 ProcessedAttachment {id, parent_id} format), so a + // re-run must skip them and only process the remaining attachment-3. + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + [ + { id: 'attachment-1', parent_id: 'parent-1' }, + { id: 'attachment-2', parent_id: 'parent-2' }, + ]; const pool = new AttachmentsStreamingPool({ adapter: mockAdapter, @@ -183,12 +189,11 @@ describe(AttachmentsStreamingPool.name, () => { const result = await pool.streamAll(); + // attachment-1/2 are skipped (already processed); attachment-3 is appended. expect( - mockAdapter.state.toDevRev?.attachmentsMetadata + mockAdapter.sdkState.toDevRev?.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ - { id: 'attachment-1', parent_id: '' }, - { id: 'attachment-2', parent_id: '' }, { id: 'attachment-1', parent_id: 'parent-1' }, { id: 'attachment-2', parent_id: 'parent-2' }, { id: 'attachment-3', parent_id: 'parent-3' }, @@ -243,7 +248,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(warnSpy).toHaveBeenCalledTimes(3); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); }); @@ -251,7 +256,7 @@ describe(AttachmentsStreamingPool.name, () => { describe(AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => { it('should skip already processed attachments', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = [{ id: 'attachment-1', parent_id: 'parent-1' }]; mockAdapter.processAttachment.mockResolvedValue({}); @@ -278,7 +283,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { id: 'attachment-1', parent_id: 'parent-1' }, @@ -308,7 +313,7 @@ describe(AttachmentsStreamingPool.name, () => { error ); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { @@ -338,7 +343,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { id: 'attachment-1', parent_id: 'parent-1' }, @@ -661,7 +666,10 @@ describe(AttachmentsStreamingPool.name, () => { }); mockAdapter.processAttachment.mockImplementation( - async (attachment, stream) => { + async ( + attachment: NormalizedAttachment, + stream: ExternalSystemAttachmentStreamingFunction + ) => { // processAttachment should be called with the user's stream function const result = await stream({ item: attachment, diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index 1690386b..39f2348a 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -1,15 +1,13 @@ -import { sleep } from '../common/helpers'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; -import { ProcessedAttachment } from '../state/state.interfaces'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, ProcessAttachmentReturnType, -} from '../types'; +} from '../types/extraction'; import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces'; export class AttachmentsStreamingPool { - private adapter: WorkerAdapter; + private adapter: ExtractionAdapter; private attachments: NormalizedAttachment[]; private batchSize: number; private delay: number | undefined; @@ -35,46 +33,19 @@ export class AttachmentsStreamingPool { this.totalProcessedCount++; if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) { console.info(`Processed ${this.totalProcessedCount} attachments so far.`); - // Sleep for 100ms to avoid blocking the event loop - await sleep(100); + // Yield once to the event loop so a pending soft-timeout message + // (WorkerMessageExit) can be delivered and adapter.isTimeout can flip + // before the next batch of attachments is processed. + await new Promise((resolve) => setImmediate(resolve)); } } - /** - * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format. - * - * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[]) - * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private migrateProcessedAttachments(attachments: any): ProcessedAttachment[] { - // Handle null/undefined - if (!attachments || !Array.isArray(attachments)) { - return []; - } - - // If already migrated (first element is an object), return as-is - if (attachments.length > 0 && typeof attachments[0] === 'object') { - return attachments as ProcessedAttachment[]; - } - - // Migrate old string[] format - if (attachments.length > 0 && typeof attachments[0] === 'string') { - return attachments.map((it) => ({ - id: it as string, - parent_id: '', - })); - } - - return []; - } - async streamAll(): Promise { console.log( `Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.` ); - if (!this.adapter.state.toDevRev) { + if (!this.adapter.sdkState.toDevRev) { const error = new Error('toDevRev state is not initialized'); console.error(error); return { error }; @@ -83,20 +54,13 @@ export class AttachmentsStreamingPool { // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction. // If no such list exists, create an empty one. if ( - !this.adapter.state.toDevRev.attachmentsMetadata + !this.adapter.sdkState.toDevRev.attachmentsMetadata .lastProcessedAttachmentsIdsList ) { - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = []; } - // Migrate old processed attachments to the new format. - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = - this.migrateProcessedAttachments( - this.adapter.state.toDevRev.attachmentsMetadata - .lastProcessedAttachmentsIdsList - ); - // Start initial batch of promises up to batchSize limit const initialBatchSize = Math.min(this.batchSize, this.attachments.length); const initialPromises = []; @@ -139,8 +103,8 @@ export class AttachmentsStreamingPool { } if ( - this.adapter.state.toDevRev && - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList?.some( + this.adapter.sdkState.toDevRev && + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList?.some( (it) => it.id == attachment.id && it.parent_id == attachment.parent_id ) ) { @@ -180,10 +144,10 @@ export class AttachmentsStreamingPool { // No rate limiting, process normally if ( - this.adapter.state.toDevRev?.attachmentsMetadata + this.adapter.sdkState.toDevRev?.attachmentsMetadata ?.lastProcessedAttachmentsIdsList ) { - this.adapter.state.toDevRev?.attachmentsMetadata.lastProcessedAttachmentsIdsList.push( + this.adapter.sdkState.toDevRev?.attachmentsMetadata.lastProcessedAttachmentsIdsList.push( { id: attachment.id, parent_id: attachment.parent_id } ); } diff --git a/src/common/constants.ts b/src/common/constants.ts index e697144c..8ac7bf31 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -67,6 +67,7 @@ export const EVENT_SIZE_THRESHOLD_BYTES = Math.floor( ); export const SSOR_ATTACHMENT: string = 'ssor_attachment'; +export const UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE'; export enum AirSyncDefaultItemTypes { EXTERNAL_DOMAIN_METADATA = 'external_domain_metadata', diff --git a/src/common/event-type-translation.test.ts b/src/common/event-type-translation.test.ts deleted file mode 100644 index 85b4825b..00000000 --- a/src/common/event-type-translation.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; -import { - translateExtractorEventType, - translateIncomingEventType, - translateLoaderEventType, - translateOutgoingEventType, -} from './event-type-translation'; - -describe(translateIncomingEventType.name, () => { - it.each([ - [ - EventType.ExtractionExternalSyncUnitsStart, - EventType.StartExtractingExternalSyncUnits, - ], - [EventType.ExtractionMetadataStart, EventType.StartExtractingMetadata], - [EventType.ExtractionDataStart, EventType.StartExtractingData], - [EventType.ExtractionDataContinue, EventType.ContinueExtractingData], - [EventType.ExtractionDataDelete, EventType.StartDeletingExtractorState], - [ - EventType.ExtractionAttachmentsStart, - EventType.StartExtractingAttachments, - ], - [ - EventType.ExtractionAttachmentsContinue, - EventType.ContinueExtractingAttachments, - ], - [ - EventType.ExtractionAttachmentsDelete, - EventType.StartDeletingExtractorAttachmentsState, - ], - ])('maps legacy extraction event %s to %s', (legacy, modern) => { - expect(translateIncomingEventType(legacy)).toBe(modern); - }); - - it.each([ - [EventType.StartExtractingExternalSyncUnits], - [EventType.StartExtractingMetadata], - [EventType.StartExtractingData], - [EventType.ContinueExtractingData], - [EventType.StartDeletingExtractorState], - [EventType.StartExtractingAttachments], - [EventType.ContinueExtractingAttachments], - [EventType.StartDeletingExtractorAttachmentsState], - [EventType.StartLoadingData], - [EventType.ContinueLoadingData], - [EventType.StartLoadingAttachments], - [EventType.ContinueLoadingAttachments], - [EventType.StartDeletingLoaderState], - [EventType.StartDeletingLoaderAttachmentState], - [EventType.UnknownEventType], - ])('is a no-op for already-modern event type %s', (eventType) => { - expect(translateIncomingEventType(eventType)).toBe(eventType); - }); - - it('returns the input verbatim for an unrecognised event type', () => { - const result = translateIncomingEventType('NONSENSE_EVENT' as EventType); - - expect(result).toBe('NONSENSE_EVENT'); - }); -}); - -describe(translateExtractorEventType.name, () => { - it.each([ - [ - ExtractorEventType.ExtractionExternalSyncUnitsDone, - ExtractorEventType.ExternalSyncUnitExtractionDone, - ], - [ - ExtractorEventType.ExtractionExternalSyncUnitsError, - ExtractorEventType.ExternalSyncUnitExtractionError, - ], - [ - ExtractorEventType.ExtractionMetadataDone, - ExtractorEventType.MetadataExtractionDone, - ], - [ - ExtractorEventType.ExtractionMetadataError, - ExtractorEventType.MetadataExtractionError, - ], - [ - ExtractorEventType.ExtractionDataProgress, - ExtractorEventType.DataExtractionProgress, - ], - [ - ExtractorEventType.ExtractionDataDelay, - ExtractorEventType.DataExtractionDelayed, - ], - [ - ExtractorEventType.ExtractionDataDone, - ExtractorEventType.DataExtractionDone, - ], - [ - ExtractorEventType.ExtractionDataError, - ExtractorEventType.DataExtractionError, - ], - [ - ExtractorEventType.ExtractionDataDeleteDone, - ExtractorEventType.ExtractorStateDeletionDone, - ], - [ - ExtractorEventType.ExtractionDataDeleteError, - ExtractorEventType.ExtractorStateDeletionError, - ], - [ - ExtractorEventType.ExtractionAttachmentsProgress, - ExtractorEventType.AttachmentExtractionProgress, - ], - [ - ExtractorEventType.ExtractionAttachmentsDelay, - ExtractorEventType.AttachmentExtractionDelayed, - ], - [ - ExtractorEventType.ExtractionAttachmentsDone, - ExtractorEventType.AttachmentExtractionDone, - ], - [ - ExtractorEventType.ExtractionAttachmentsError, - ExtractorEventType.AttachmentExtractionError, - ], - [ - ExtractorEventType.ExtractionAttachmentsDeleteDone, - ExtractorEventType.ExtractorAttachmentsStateDeletionDone, - ], - [ - ExtractorEventType.ExtractionAttachmentsDeleteError, - ExtractorEventType.ExtractorAttachmentsStateDeletionError, - ], - ])('maps legacy extractor event %s to %s', (legacy, modern) => { - expect(translateExtractorEventType(legacy)).toBe(modern); - }); - - it.each([ - [ExtractorEventType.DataExtractionDone], - [ExtractorEventType.DataExtractionProgress], - [ExtractorEventType.AttachmentExtractionDone], - [ExtractorEventType.MetadataExtractionDone], - [ExtractorEventType.UnknownEventType], - ])('is a no-op for already-modern extractor event %s', (eventType) => { - expect(translateExtractorEventType(eventType)).toBe(eventType); - }); -}); - -describe(translateLoaderEventType.name, () => { - it.each([ - [LoaderEventType.DataLoadingDelay, LoaderEventType.DataLoadingDelayed], - [ - LoaderEventType.AttachmentsLoadingProgress, - LoaderEventType.AttachmentLoadingProgress, - ], - [ - LoaderEventType.AttachmentsLoadingDelayed, - LoaderEventType.AttachmentLoadingDelayed, - ], - [ - LoaderEventType.AttachmentsLoadingDone, - LoaderEventType.AttachmentLoadingDone, - ], - [ - LoaderEventType.AttachmentsLoadingError, - LoaderEventType.AttachmentLoadingError, - ], - ])('maps legacy loader event %s to %s', (legacy, modern) => { - expect(translateLoaderEventType(legacy)).toBe(modern); - }); - - it.each([ - [LoaderEventType.DataLoadingDone], - [LoaderEventType.DataLoadingProgress], - [LoaderEventType.AttachmentLoadingDone], - ])('is a no-op for already-modern loader event %s', (eventType) => { - expect(translateLoaderEventType(eventType)).toBe(eventType); - }); -}); - -describe(translateOutgoingEventType.name, () => { - it('routes extractor events through translateExtractorEventType', () => { - expect( - translateOutgoingEventType(ExtractorEventType.ExtractionDataDone) - ).toBe(ExtractorEventType.DataExtractionDone); - }); - - it('routes loader events through translateLoaderEventType', () => { - expect( - translateOutgoingEventType(LoaderEventType.AttachmentsLoadingDone) - ).toBe(LoaderEventType.AttachmentLoadingDone); - }); - - it('passes through unknown event types unchanged', () => { - const unknown = 'SOME_UNKNOWN_EVENT' as ExtractorEventType; - expect(translateOutgoingEventType(unknown)).toBe(unknown); - }); -}); diff --git a/src/common/event-type-translation.ts b/src/common/event-type-translation.ts deleted file mode 100644 index f1a32abe..00000000 --- a/src/common/event-type-translation.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; - -/** - * Maps old incoming event type strings to new EventType enum values. - * This ensures backwards compatibility when the platform sends old event types. - * @param eventTypeString The raw event type string from the platform - * @returns The translated EventType enum value - */ -export function translateIncomingEventType(eventTypeString: string): EventType { - // Create a reverse mapping from OLD string values to NEW enum member names - const eventTypeMap: Record = { - // Old extraction event types from platform -> New enum members - [EventType.ExtractionExternalSyncUnitsStart]: - EventType.StartExtractingExternalSyncUnits, - [EventType.ExtractionMetadataStart]: EventType.StartExtractingMetadata, - [EventType.ExtractionDataStart]: EventType.StartExtractingData, - [EventType.ExtractionDataContinue]: EventType.ContinueExtractingData, - [EventType.ExtractionDataDelete]: EventType.StartDeletingExtractorState, - [EventType.ExtractionAttachmentsStart]: - EventType.StartExtractingAttachments, - [EventType.ExtractionAttachmentsContinue]: - EventType.ContinueExtractingAttachments, - [EventType.ExtractionAttachmentsDelete]: - EventType.StartDeletingExtractorAttachmentsState, - - // New extraction event types (already correct, map to new enum members) - [EventType.StartExtractingExternalSyncUnits]: - EventType.StartExtractingExternalSyncUnits, - [EventType.StartExtractingMetadata]: EventType.StartExtractingMetadata, - [EventType.StartExtractingData]: EventType.StartExtractingData, - [EventType.ContinueExtractingData]: EventType.ContinueExtractingData, - [EventType.StartDeletingExtractorState]: - EventType.StartDeletingExtractorState, - [EventType.StartExtractingAttachments]: - EventType.StartExtractingAttachments, - [EventType.ContinueExtractingAttachments]: - EventType.ContinueExtractingAttachments, - [EventType.StartDeletingExtractorAttachmentsState]: - EventType.StartDeletingExtractorAttachmentsState, - - // Loading events - [EventType.StartLoadingData]: EventType.StartLoadingData, - [EventType.ContinueLoadingData]: EventType.ContinueLoadingData, - [EventType.StartLoadingAttachments]: EventType.StartLoadingAttachments, - [EventType.ContinueLoadingAttachments]: - EventType.ContinueLoadingAttachments, - [EventType.StartDeletingLoaderState]: EventType.StartDeletingLoaderState, - [EventType.StartDeletingLoaderAttachmentState]: - EventType.StartDeletingLoaderAttachmentState, - - // Unknown - [EventType.UnknownEventType]: EventType.UnknownEventType, - }; - - const translated = eventTypeMap[eventTypeString]; - if (!translated) { - console.warn( - `Unknown event type received: ${eventTypeString}. This may indicate a new event type or a typo.` - ); - // Return the original string cast as EventType as a fallback - return eventTypeString as EventType; - } - - return translated; -} - -/** - * Translates ExtractorEventType enum values by converting old enum members to new ones. - * Old enum members are deprecated and should be replaced with new ones. - */ -export function translateExtractorEventType( - eventType: ExtractorEventType -): ExtractorEventType { - // Map old enum members to new enum members - const stringValue = eventType as string; - - const mapping: Record = { - // Old string values -> New enum members - [ExtractorEventType.ExtractionExternalSyncUnitsDone]: - ExtractorEventType.ExternalSyncUnitExtractionDone, - [ExtractorEventType.ExtractionExternalSyncUnitsError]: - ExtractorEventType.ExternalSyncUnitExtractionError, - [ExtractorEventType.ExtractionMetadataDone]: - ExtractorEventType.MetadataExtractionDone, - [ExtractorEventType.ExtractionMetadataError]: - ExtractorEventType.MetadataExtractionError, - [ExtractorEventType.ExtractionDataProgress]: - ExtractorEventType.DataExtractionProgress, - [ExtractorEventType.ExtractionDataDelay]: - ExtractorEventType.DataExtractionDelayed, - [ExtractorEventType.ExtractionDataDone]: - ExtractorEventType.DataExtractionDone, - [ExtractorEventType.ExtractionDataError]: - ExtractorEventType.DataExtractionError, - [ExtractorEventType.ExtractionDataDeleteDone]: - ExtractorEventType.ExtractorStateDeletionDone, - [ExtractorEventType.ExtractionDataDeleteError]: - ExtractorEventType.ExtractorStateDeletionError, - [ExtractorEventType.ExtractionAttachmentsProgress]: - ExtractorEventType.AttachmentExtractionProgress, - [ExtractorEventType.ExtractionAttachmentsDelay]: - ExtractorEventType.AttachmentExtractionDelayed, - [ExtractorEventType.ExtractionAttachmentsDone]: - ExtractorEventType.AttachmentExtractionDone, - [ExtractorEventType.ExtractionAttachmentsError]: - ExtractorEventType.AttachmentExtractionError, - [ExtractorEventType.ExtractionAttachmentsDeleteDone]: - ExtractorEventType.ExtractorAttachmentsStateDeletionDone, - [ExtractorEventType.ExtractionAttachmentsDeleteError]: - ExtractorEventType.ExtractorAttachmentsStateDeletionError, - }; - - // If there's a mapping, use it; otherwise return original (already new) - return mapping[stringValue] ?? eventType; -} - -/** - * Translates LoaderEventType enum values by converting old enum members to new ones. - * Old enum members are deprecated and should be replaced with new ones. - */ -export function translateLoaderEventType( - eventType: LoaderEventType -): LoaderEventType { - // Map old enum members to new enum members - const stringValue = eventType as string; - - const mapping: Record = { - // Old string values -> New enum members - [LoaderEventType.DataLoadingDelay]: LoaderEventType.DataLoadingDelayed, - [LoaderEventType.AttachmentsLoadingProgress]: - LoaderEventType.AttachmentLoadingProgress, - [LoaderEventType.AttachmentsLoadingDelayed]: - LoaderEventType.AttachmentLoadingDelayed, - [LoaderEventType.AttachmentsLoadingDone]: - LoaderEventType.AttachmentLoadingDone, - [LoaderEventType.AttachmentsLoadingError]: - LoaderEventType.AttachmentLoadingError, - }; - - // If there's a mapping, use it; otherwise return original (already new) - return mapping[stringValue] ?? eventType; -} - -/** - * Translates any outgoing event type (Extractor or Loader) to ensure new event types are used. - */ -export function translateOutgoingEventType( - eventType: ExtractorEventType | LoaderEventType -): ExtractorEventType | LoaderEventType { - // Check if it's an ExtractorEventType by checking if the value exists in ExtractorEventType - if ( - Object.values(ExtractorEventType).includes(eventType as ExtractorEventType) - ) { - return translateExtractorEventType(eventType as ExtractorEventType); - } - // Otherwise treat as LoaderEventType - if (Object.values(LoaderEventType).includes(eventType as LoaderEventType)) { - return translateLoaderEventType(eventType as LoaderEventType); - } - // If neither, return as-is - return eventType; -} diff --git a/src/common/helpers.ts b/src/common/helpers.ts index e4abeef9..d4ef030b 100644 --- a/src/common/helpers.ts +++ b/src/common/helpers.ts @@ -2,12 +2,6 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import * as v8 from 'v8'; -import { - MAX_DEVREV_FILENAME_EXTENSION_LENGTH, - MAX_DEVREV_FILENAME_LENGTH, -} from './constants'; -import { MAX_LOG_STRING_LENGTH } from '../logger/logger.constants'; - /** * Gets the library version from the package.json file. * @returns {string} The library version @@ -37,36 +31,9 @@ export function getLibraryVersion() { * @returns {Promise} A promise that resolves after the given number of milliseconds */ export async function sleep(ms: number) { - console.log(`Sleeping for ${ms}ms.`); return new Promise((resolve) => setTimeout(resolve, ms)); } -/** - * Truncates a filename if it exceeds the maximum allowed length. - * @param {string} filename - The filename to truncate - * @returns {string} The truncated filename - */ -export function truncateFilename(filename: string): string { - // If the filename is already within the limit, return it as is. - if (filename.length <= MAX_DEVREV_FILENAME_LENGTH) { - return filename; - } - - console.warn( - `Filename length exceeds the maximum limit of ${MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.` - ); - - const extension = filename.slice(-MAX_DEVREV_FILENAME_EXTENSION_LENGTH); - // Calculate how many characters are available for the name part after accounting for the extension and "..." - const availableNameLength = - MAX_DEVREV_FILENAME_LENGTH - MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..." - - // Truncate the name part and add an ellipsis - const truncatedFilename = filename.slice(0, availableNameLength); - - return `${truncatedFilename}...${extension}`; -} - /** * MemoryInfo is an interface that represents the memory usage information. * @interface MemoryInfo @@ -142,19 +109,3 @@ export function getMemoryUsage(): MemoryInfo { throw err; } } - -/** - * Truncates a message if it exceeds the maximum allowed length. - * Adds a suffix indicating how many characters were omitted. - * - * @param message - The message to truncate - * @returns Truncated message or original if within limits - */ -export function truncateMessage(message: string): string { - if (message.length > MAX_LOG_STRING_LENGTH) { - return `${message.substring(0, MAX_LOG_STRING_LENGTH)}... ${ - message.length - MAX_LOG_STRING_LENGTH - } more characters`; - } - return message; -} diff --git a/src/deprecated/adapter/index.ts b/src/deprecated/adapter/index.ts deleted file mode 100644 index 69547684..00000000 --- a/src/deprecated/adapter/index.ts +++ /dev/null @@ -1,209 +0,0 @@ -import axios from 'axios'; - -import { - AirdropEvent, - EventData, - ExtractorEvent, - ExtractorEventType, -} from '../../types/extraction'; -import { Artifact } from '../../uploader/uploader.interfaces'; - -import { AdapterState } from '../../state/state.interfaces'; - -import { STATELESS_EVENT_TYPES } from '../../common/constants'; -import { getTimeoutExtractorEventType } from '../common/helpers'; -// import { Logger } from '../../logger/logger'; -import { State, createAdapterState } from '../../state/state'; -import { translateIncomingEventType } from '../../common/event-type-translation'; -import { runWithSdkLogContext } from '../../logger/logger.context'; - -/** - * Adapter class is used to interact with Airdrop platform. The class provides - * utilities to - * - emit control events to the platform - * - update the state of the extractor - * - set the last saved state in case of a timeout - * - * @class Adapter - * @constructor - * @deprecated - * @param {AirdropEvent} event - The event object received from the platform - * @param {object=} initialState - The initial state of the adapter - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - */ - -/** - * Creates an adapter instance. - * - * @param {AirdropEvent} event - The event object received from the platform - * @param initialState - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @return The adapter instance - */ - -export async function createAdapter( - event: AirdropEvent, - initialState: ConnectorState, - isLocalDevelopment: boolean = false -) { - event.payload.event_type = translateIncomingEventType(event.payload.event_type); - - const newInitialState = structuredClone(initialState); - const adapterState: State = await createAdapterState({ - event, - initialState: newInitialState, - }); - - const a = new Adapter( - event, - adapterState, - isLocalDevelopment - ); - - return a; -} - -export class Adapter { - private adapterState: State; - private _artifacts: Artifact[]; - - private event: AirdropEvent; - private callbackUrl: string; - private devrevToken: string; - private startTime: number; - private heartBeatFn: ReturnType | undefined; - private exit: boolean = false; - private lambdaTimeout: number = 10 * 60 * 1000; // 10 minutes in milliseconds - private heartBeatInterval: number = 30 * 1000; // 30 seconds in milliseconds - - constructor( - event: AirdropEvent, - adapterState: State, - isLocalDevelopment: boolean = false - ) { - if (!isLocalDevelopment) { - // Logger.init(event); - } - - this.adapterState = adapterState; - this._artifacts = []; - - this.event = event; - this.callbackUrl = event.payload.event_context.callback_url; - this.devrevToken = event.context.secrets.service_account_token; - - this.startTime = Date.now(); - - // Run heartbeat every 30 seconds - this.heartBeatFn = setInterval(async () => { - const b = await this.heartbeat(); - if (b) { - this.exitAdapter(); - } - }, this.heartBeatInterval); - } - - get state(): AdapterState { - return this.adapterState.state; - } - - set state(value: AdapterState) { - this.adapterState.state = value; - } - - get artifacts(): Artifact[] { - return this._artifacts; - } - - set artifacts(value: Artifact[]) { - this._artifacts = value; - } - - /** - * Emits an event to the platform. - * - * @param {ExtractorEventType} newEventType - The event type to be emitted - * @param {EventData=} data - The data to be sent with the event - */ - async emit(newEventType: ExtractorEventType, data?: EventData) { - if (this.exit) { - console.warn( - 'Adapter is already in exit state. No more events can be emitted.' - ); - return; - } - - // We want to save the state every time we emit an event, except for the start and delete events - if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { - runWithSdkLogContext(() => - console.log(`Saving state before emitting event`) - ); - await this.adapterState.postState(this.state); - } - - const newEvent: ExtractorEvent = { - event_type: newEventType, - event_context: this.event.payload.event_context, - event_data: { - ...data, - }, - }; - - try { - await axios.post( - this.callbackUrl, - { ...newEvent }, - { - headers: { - Accept: 'application/json, text/plain, */*', - Authorization: this.devrevToken, - 'Content-Type': 'application/json', - }, - } - ); - - console.log('Successfully emitted event: ' + JSON.stringify(newEvent)); - } catch (error) { - // If this request fails the extraction will be stuck in loop and - // we need to stop it through UI or think about retrying this request - console.log( - 'Failed to emit event: ' + - JSON.stringify(newEvent) + - ', error: ' + - error - ); - } finally { - this.exitAdapter(); - } - } - - /** - * Exit the adapter. This will stop the heartbeat and no - * further events will be emitted. - */ - private exitAdapter() { - this.exit = true; - } - - /** - * Heartbeat function to check if the lambda is about to timeout. - * @returns true if 10 minutes have passed since the start of the lambda. - */ - private async heartbeat(): Promise { - if (this.exit) { - return true; - } - if (Date.now() - this.startTime > this.lambdaTimeout) { - const timeoutEventType = getTimeoutExtractorEventType( - this.event.payload.event_type - ); - if (timeoutEventType !== null) { - const { eventType, isError } = timeoutEventType; - const err = isError ? { message: 'Lambda Timeout' } : undefined; - await this.emit(eventType, { error: err, artifacts: this._artifacts }); - return true; - } - } - return false; - } -} diff --git a/src/deprecated/common/helpers.ts b/src/deprecated/common/helpers.ts deleted file mode 100644 index 41eb9477..00000000 --- a/src/deprecated/common/helpers.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { jsonl } from 'js-jsonl'; - -import { EventType, ExtractorEventType } from '../../types/extraction'; - -export function createFormData( - //eslint-disable-next-line @typescript-eslint/no-explicit-any - preparedArtifact: any, - fetchedObjects: object[] | object -): FormData { - const formData = new FormData(); - for (const item of preparedArtifact.form_data) { - formData.append(item.key, item.value); - } - - const output = jsonl.stringify(fetchedObjects); - formData.append('file', output); - - return formData; -} - -export function getTimeoutExtractorEventType(eventType: EventType): { - eventType: ExtractorEventType; - isError: boolean; -} | null { - switch (eventType) { - case EventType.ExtractionMetadataStart: - return { - eventType: ExtractorEventType.ExtractionMetadataError, - isError: true, - }; - case EventType.ExtractionDataStart: - case EventType.ExtractionDataContinue: - return { - eventType: ExtractorEventType.ExtractionDataProgress, - isError: false, - }; - case EventType.ExtractionAttachmentsStart: - case EventType.ExtractionAttachmentsContinue: - return { - eventType: ExtractorEventType.ExtractionAttachmentsProgress, - isError: false, - }; - case EventType.ExtractionExternalSyncUnitsStart: - return { - eventType: ExtractorEventType.ExtractionExternalSyncUnitsError, - isError: true, - }; - default: - console.log( - 'Event type not recognized in getTimeoutExtractorEventType function: ' + - eventType - ); - return null; - } -} diff --git a/src/deprecated/demo-extractor/external_domain_metadata.json b/src/deprecated/demo-extractor/external_domain_metadata.json deleted file mode 100644 index ea876481..00000000 --- a/src/deprecated/demo-extractor/external_domain_metadata.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "record_types": { - "users": { - "fields": { - "name": { - "is_required": true, - "type": "text", - "name": "Name", - "text": { - "min_length": 1 - } - }, - "email": { - "type": "text", - "name": "Email", - "is_required": true - } - } - }, - "contacts": { - "fields": { - "name": { - "is_required": true, - "type": "text", - "name": "Name", - "text": { - "min_length": 1 - } - }, - "email": { - "type": "text", - "name": "Email", - "is_required": true - } - } - } - } -} diff --git a/src/deprecated/demo-extractor/index.ts b/src/deprecated/demo-extractor/index.ts deleted file mode 100644 index 30e937ff..00000000 --- a/src/deprecated/demo-extractor/index.ts +++ /dev/null @@ -1,235 +0,0 @@ -import { - AirdropEvent, - EventType, - ExternalSyncUnit, - ExtractorEventType, -} from '../../types/extraction'; -import { Adapter } from '../adapter'; -import { Uploader } from '../uploader'; -import externalDomainMetadata from './external_domain_metadata.json'; - -type ConnectorState = object; - -/** - * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS. - * - * @class DemoExtractor - * @deprecated - **/ -export class DemoExtractor { - private event: AirdropEvent; - private adapter: Adapter; - private uploader: Uploader; - - constructor(event: AirdropEvent, adapter: Adapter) { - this.event = event; - this.adapter = adapter; - this.uploader = new Uploader( - this.event.execution_metadata.devrev_endpoint, - this.event.context.secrets.service_account_token - ); - } - - async run() { - switch (this.event.payload.event_type) { - case EventType.ExtractionExternalSyncUnitsStart: { - const externalSyncUnits: ExternalSyncUnit[] = [ - { - id: 'devrev', - name: 'devrev', - description: 'Demo external sync unit', - }, - ]; - - await this.adapter.emit( - ExtractorEventType.ExtractionExternalSyncUnitsDone, - { - external_sync_units: externalSyncUnits, - } - ); - - break; - } - - case EventType.ExtractionMetadataStart: { - const { artifact, error } = await this.uploader.upload( - 'metadata_1.jsonl', - 'external_domain_metadata', - externalDomainMetadata - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionMetadataError, { - error, - }); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionMetadataDone, { - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataStart: { - const contacts = [ - { - id: 'contact-1', - created_date: '1999-12-25T01:00:03+01:00', - modified_date: '1999-12-25T01:00:03+01:00', - data: { - email: 'johnsmith@test.com', - name: 'John Smith', - }, - }, - { - id: 'contact-2', - created_date: '1999-12-27T15:31:34+01:00', - modified_date: '2002-04-09T01:55:31+02:00', - data: { - email: 'janesmith@test.com', - name: 'Jane Smith', - }, - }, - ]; - - const { artifact, error } = await this.uploader.upload( - 'contacts_1.json', - 'contacts', - contacts - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionDataError, { - error, - }); - - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionDataProgress, { - progress: 50, - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataContinue: { - const users = [ - { - id: 'user-1', - created_date: '1999-12-25T01:00:03+01:00', - modified_date: '1999-12-25T01:00:03+01:00', - data: { - email: 'johndoe@test.com', - name: 'John Doe', - }, - }, - { - id: 'user-2', - created_date: '1999-12-27T15:31:34+01:00', - modified_date: '2002-04-09T01:55:31+02:00', - data: { - email: 'janedoe@test.com', - name: 'Jane Doe', - }, - }, - ]; - - const { artifact, error } = await this.uploader.upload( - 'users_1.json', - 'users', - users - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionDataError, { - error, - }); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionDataDone, { - progress: 100, - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataDelete: { - await this.adapter.emit(ExtractorEventType.ExtractionDataDeleteDone); - break; - } - - case EventType.ExtractionAttachmentsStart: { - const attachment1 = ['This is attachment1.txt content']; - const { artifact, error } = await this.uploader.upload( - 'attachment1.txt', - 'attachment', - attachment1 - ); - - if (error || !artifact) { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsError, - { - error, - } - ); - return; - } - - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsProgress, - { - artifacts: [artifact], - } - ); - - break; - } - - case EventType.ExtractionAttachmentsContinue: { - const attachment2 = ['This is attachment2.txt content']; - const { artifact, error } = await this.uploader.upload( - 'attachment2.txt', - 'attachment', - attachment2 - ); - - if (error || !artifact) { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsError, - { - error, - } - ); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDone, { - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionAttachmentsDelete: { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsDeleteDone - ); - break; - } - - default: { - console.error( - 'Event in DemoExtractor run not recognized: ' + - JSON.stringify(this.event.payload.event_type) - ); - } - } - } -} diff --git a/src/deprecated/http/client.ts b/src/deprecated/http/client.ts deleted file mode 100644 index e4430437..00000000 --- a/src/deprecated/http/client.ts +++ /dev/null @@ -1,149 +0,0 @@ -import axios, { - InternalAxiosRequestConfig, - isAxiosError, - RawAxiosRequestHeaders, -} from 'axios'; -import { - RATE_LIMIT_EXCEEDED, - RATE_LIMIT_EXCEEDED_STATUS_CODE, -} from '../../http/constants'; -import { HTTPResponse } from '../../http/types'; - -export const defaultResponse: HTTPResponse = { - data: { - delay: 0, - nextPage: 1, - records: [], - }, - message: '', - success: false, -}; - -/** - * HTTPClient class to make HTTP requests - * @deprecated - */ -export class HTTPClient { - private retryAfter = 0; - private retryAt = 0; - private axiosInstance = axios.create(); - - constructor() { - // Add request interceptor to check for retryAfter before making a request - this.axiosInstance.interceptors.request.use( - (config: InternalAxiosRequestConfig) => { - // Check if retryAfter is not 0 and return a LIMIT_EXCEEDED error - if (this.retryAfter !== 0) { - // check if the current time is greater than the retryAt time - const currentTime = new Date().getTime(); - if (currentTime < this.retryAt) { - console.error( - 'Rate limit exceeded. Interceptor has retryAfter: ' + - this.retryAfter - ); - // Rate limit exceeded. - return Promise.reject(RATE_LIMIT_EXCEEDED); - } else { - // Reset the retryAfter - this.retryAfter = 0; - } - } - return config; - }, - (error) => { - return Promise.reject(error); - } - ); - } - - /** - * - * Function to make a GET call to the endpoint. - * There is special handling for rate limit exceeded error. - * In case of rate limit exceeded, the function returns success as true and the delay time in seconds - * In case of any other error, the function returns success as false and the error message - */ - async getCall( - endpoint: string, - headers: RawAxiosRequestHeaders, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - params?: any - ): Promise { - // Return the LIMIT_EXCEEDED error if the retryAfter is not 0 - try { - const res = await this.axiosInstance.get(endpoint, { - headers: headers, - params: params, - }); - return { - ...defaultResponse, - data: { - delay: 0, - records: res.data, - }, - success: true, - }; - } catch (error: unknown) { - console.log('Error in getCall: ' + JSON.stringify(error)); - // send error to adapter - if (isAxiosError(error)) { - if (error.response?.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) { - this.retryAfter = error.response.headers['retry-after'] - ? error.response.headers['retry-after'] - : 0; - this.retryAt = new Date().getTime() + this.retryAfter * 1000; - console.warn( - 'Rate limit exceeded. Error code: ' + - error.response.status + - ' RetryAfter: ' + - this.retryAfter + - ' RetryAt: ' + - this.retryAt - ); - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: RATE_LIMIT_EXCEEDED, - success: true, - }; - } - if (error.response) { - return { ...defaultResponse, message: error.response.data }; - } else { - return { ...defaultResponse, message: error.message }; - } - } else { - if (this.retryAfter !== 0) { - console.warn( - 'Rate limit exceeded. Going to return the following response: ' + - JSON.stringify(error) - ); - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: - typeof error === 'string' - ? error - : JSON.stringify(error, Object.getOwnPropertyNames(error)), - success: true, - }; - } - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: - typeof error === 'string' - ? error - : JSON.stringify(error, Object.getOwnPropertyNames(error)), - success: false, - }; - } - } - } -} diff --git a/src/deprecated/uploader/index.ts b/src/deprecated/uploader/index.ts deleted file mode 100644 index eca7203b..00000000 --- a/src/deprecated/uploader/index.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { betaSDK, client } from '@devrev/typescript-sdk'; -import fs, { promises as fsPromises } from 'fs'; -import { axiosClient } from '../../http/axios-client-internal'; -import { Artifact, UploadResponse } from '../../uploader/uploader.interfaces'; -import { createFormData } from '../common/helpers'; - -/** - * Uploader class is used to upload files to the DevRev platform. - * The class provides utilities to - * - prepare artifact - * - upload artifact - * - return the artifact information to the platform - * - * @class Uploader - * @constructor - * @param {string} endpoint - The endpoint of the DevRev platform - * @param {string} token - The token to authenticate with the DevRev platform - * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system. - */ -export class Uploader { - private betaDevrevSdk: betaSDK.Api; - private local: boolean; - constructor(endpoint: string, token: string, local = false) { - this.betaDevrevSdk = client.setupBeta({ - endpoint, - token, - }); - this.local = local; - } - - /** - * - * Uploads the file to the DevRev platform. The file is uploaded to the platform - * and the artifact information is returned. - * - * @param {string} filename - The name of the file to be uploaded - * @param {string} entity - The entity type of the file to be uploaded - * @param {object[] | object} fetchedObjects - The objects to be uploaded - * @param filetype - The type of the file to be uploaded - * @returns {Promise} - The response object containing the artifact information - */ - async upload( - filename: string, - entity: string, - fetchedObjects: object[] | object, - filetype: string = 'application/jsonl+json' - ): Promise { - if (this.local) { - await this.downloadToLocal(filename, fetchedObjects); - } - - const preparedArtifact = await this.prepareArtifact(filename, filetype); - - if (!preparedArtifact) { - return { - artifact: undefined, - error: { message: 'Error while preparing artifact' }, - }; - } - - const uploadedArtifact = await this.uploadToArtifact( - preparedArtifact, - fetchedObjects - ); - - if (!uploadedArtifact) { - return { - artifact: undefined, - error: { message: 'Error while uploading artifact' }, - }; - } - - // If file was successfully uploaded we want to post data about that file when emitting - const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1; - const artifact: Artifact = { - id: preparedArtifact.id, - item_type: entity, - item_count: itemCount, - }; - - console.log(`Artifact uploaded successfully: ${artifact.id}`); - - return { artifact, error: undefined }; - } - - private async prepareArtifact( - filename: string, - filetype: string - ): Promise { - try { - const response = await this.betaDevrevSdk.artifactsPrepare({ - file_name: filename, - file_type: filetype, - }); - - return response.data; - } catch (error) { - console.error('Error while preparing artifact: ' + error); - return null; - } - } - - private async uploadToArtifact( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - preparedArtifact: any, - fetchedObjects: object[] | object - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): Promise { - const formData = createFormData(preparedArtifact, fetchedObjects); - try { - const response = await axiosClient.post(preparedArtifact.url, formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }); - - return response; - } catch (error) { - console.error('Error while uploading artifact: ' + error); - return null; - } - } - - private async downloadToLocal( - filePath: string, - fetchedObjects: object | object[] - ) { - console.log(`Uploading ${filePath} to local file system`); - try { - if (!fs.existsSync('extracted_files')) { - fs.mkdirSync('extracted_files'); - } - - const timestamp = new Date().getTime(); - const fileHandle = await fsPromises.open( - `extracted_files/${timestamp}_${filePath}`, - 'w' - ); - let objArray = []; - if (!Array.isArray(fetchedObjects)) { - objArray.push(fetchedObjects); - } else { - objArray = fetchedObjects; - } - for (const jsonObject of objArray) { - const jsonLine = JSON.stringify(jsonObject) + '\n'; - await fileHandle.write(jsonLine); - } - await fileHandle.close(); - console.log('Data successfully written to', filePath); - } catch (error) { - console.error('Error writing data to file:', error); - return Promise.reject(error); - } - } -} diff --git a/src/http/axios-client.ts b/src/http/axios-client.ts deleted file mode 100644 index 703b1e59..00000000 --- a/src/http/axios-client.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Axios client setup with retry capabilities using axios-retry. - * - * This module exports an Axios client instance (`axiosClient`) that is configured to automatically retry - * failed requests under certain conditions. - * - * Retry Conditions: - * 1. Network errors (where no response is received). - * 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT). - * 3. All 5xx server errors. - * - * Retry Strategy: - * - A maximum of 5 retries are attempted. - * - Exponential backoff delay is applied between retries, increasing with each retry attempt. - * - * Additional Features: - * - When the maximum number of retry attempts is reached, sensitive headers (like authorization) - * are removed from error logs for security reasons. - * - * Exported: - * - `axios`: Original axios instance for additional customizations or direct use. - * - `axiosClient`: Configured axios instance with retry logic. - */ - -import axios, { AxiosError } from 'axios'; -import axiosRetry from 'axios-retry'; -import { runWithUserLogContext } from '../logger/logger.context'; - -const axiosClient = axios.create(); - -axiosRetry(axiosClient, { - retries: 5, - retryDelay: (retryCount, error) => { - // exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms - const delay = axiosRetry.exponentialDelay(retryCount, error, 1000); - - runWithUserLogContext(() => - console.warn( - `Request to ${error.config?.url} failed with response status code ${ - error.response?.status - }. Method ${ - error.config?.method - }. Retry count: ${retryCount}. Retrying in ${Math.round( - delay / 1000 - )}s.` - ) - ); - - return delay; - }, - retryCondition: (error: AxiosError) => { - return ( - (axiosRetry.isNetworkOrIdempotentRequestError(error) && - error.response?.status !== 429) || - (error.response?.status ?? 0) >= 500 - ); - }, - onMaxRetryTimesExceeded(error: AxiosError) { - delete error.config?.headers?.authorization; - delete error.config?.headers?.Authorization; - delete error.request._header; - runWithUserLogContext(() => - console.warn('Max retry times exceeded. Error', error) - ); - }, -}); - -export { axios, axiosClient }; diff --git a/src/http/axios-client-internal.test.ts b/src/http/client.test.ts similarity index 99% rename from src/http/axios-client-internal.test.ts rename to src/http/client.test.ts index daa9e07f..8f220cfc 100644 --- a/src/http/axios-client-internal.test.ts +++ b/src/http/client.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { axiosClient } from './axios-client-internal'; +import { axiosClient } from './client'; jest.setTimeout(60000); diff --git a/src/http/axios-client-internal.ts b/src/http/client.ts similarity index 100% rename from src/http/axios-client-internal.ts rename to src/http/client.ts diff --git a/src/http/constants.ts b/src/http/constants.ts deleted file mode 100644 index 25cfbf91..00000000 --- a/src/http/constants.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const RATE_LIMIT_EXCEEDED = 'LIMIT_EXCEEDED'; -export const LAMBDA_LIMIT_EXCEEDED = 'LAMBDA_LIMIT_EXCEEDED'; -export const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429; diff --git a/src/http/index.ts b/src/http/index.ts deleted file mode 100644 index 4edc2c55..00000000 --- a/src/http/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './axios-client'; -export * from './types'; diff --git a/src/http/types.ts b/src/http/types.ts deleted file mode 100644 index 3206229a..00000000 --- a/src/http/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * HTTP Response type - * @deprecated - */ -export type HTTPResponse = { - success: boolean; - message: string; - data: Data; -}; - -interface Data { - records: object[]; // List of records of the entity - delay: number; // Delay in seconds(used for ratelimiting), Time to wait before next call - nextPage?: number; // The next page of the entity to be processed - metadata?: object; // Other information that should be returned -} diff --git a/src/index.ts b/src/index.ts index 26dd8ff5..3ce9465c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,22 +1,169 @@ +// ──────────────────────────────────────────────────────────────────────────── +// Public API barrel for @devrev/airsync-sdk. +// +// This is the SINGLE source of the public surface — every exported symbol is +// named explicitly (no `export *`), so any change to the public API shows up as +// a diff here. Adding a symbol to the package's public contract means adding it +// to this file. +// ──────────────────────────────────────────────────────────────────────────── + +// ── Entry points & adapters ── +export { + processExtractionTask, + processLoadingTask, +} from './multithreading/process-task'; +export { spawn } from './multithreading/spawn/spawn'; +export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter'; +export { LoadingAdapter } from './multithreading/adapters/loading-adapter'; + +// ── Worker contract types ── +export type { + ProcessTaskInterface, + TaskAdapterInterface, + TaskResult, + TaskStatus, + WorkerAdapterInterface, + WorkerAdapterOptions, + SpawnFactoryInterface, + SpawnInterface, + ExtractionScope, + WorkerPathOverrides, +} from './types/workers'; + +// ── Constants & enums ── export { AirSyncDefaultItemTypes } from './common/constants'; -export { ExtractionCommonError } from './common/errors'; -export * from './common/install-initial-domain-mapping'; -export * from './deprecated/adapter'; -export * from './deprecated/demo-extractor'; -export * from './deprecated/http/client'; -export * from './deprecated/uploader'; -export * from './http'; -export { formatAxiosError, serializeAxiosError } from './logger/logger'; -export { MockServer } from './mock-server/mock-server'; +export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants'; +export { ExtractionCommonError } from './types/errors'; + +// ── Domain mapping install ── +export { installInitialDomainMapping } from './state/install-initial-domain-mapping'; + +// ── Error formatting ── +export { serializeError } from './logger/logger'; + +// ── Common types ── +export { SyncMode } from './types/common'; +export type { ErrorRecord, InitialDomainMapping } from './types/common'; + +// ── Extraction types ── +export { + EventType, + ExtractorEventType, + InitialSyncScope, + TimeUnit, + TimeValueType, +} from './types/extraction'; +export type { + AirSyncEvent, + AirSyncMessage, + ConnectionData, + EventContext, + EventData, + ExternalProcessAttachmentFunction, + ExternalSyncUnit, + ExternalSystemAttachmentIteratorFunction, + ExternalSystemAttachmentReducerFunction, + ExternalSystemAttachmentStreamingFunction, + ExternalSystemAttachmentStreamingParams, + ExternalSystemAttachmentStreamingResponse, + ExtractorEvent, + HttpStreamResponse, + ProcessAttachmentReturnType, + TimeValue, +} from './types/extraction'; + +// ── Loading types ── +export { LoaderEventType } from './types/loading'; +export type { + ExternalSystemAttachment, + ExternalSystemItem, + ExternalSystemItemLoadingParams, + ExternalSystemItemLoadingResponse, + ItemTypeToLoad, +} from './types/loading'; + +// ── Repo types ── +export type { + Item, + NormalizedAttachment, + NormalizedItem, + RepoInterface, +} from './repo/repo.interfaces'; + +// ── Mappers ── +export { Mappers } from './mappers/mappers'; +export { + SyncMapperRecordStatus, + SyncMapperRecordTargetType, +} from './mappers/mappers.interfaces'; +export type { + MappersCreateParams, + MappersGetByExternalIdParams, + MappersGetByTargetIdParams, + MappersUpdateParams, +} from './mappers/mappers.interfaces'; + +// ── Uploader types ── +export type { + Artifact, + ArtifactsPrepareResponse, + SsorAttachment, + StreamAttachmentsResponse, + StreamResponse, + UploadResponse, +} from './uploader/uploader.interfaces'; + +// ── External domain metadata types ── +export type { + CollectionData, + ConditionalPrivilegeData, + CustomLinkData, + CustomLinkNames, + CustomStage, + CustomState, + EnumData, + EnumValue, + EnumValueKey, + ExternalDomainMetadata, + Field, + FieldCondition, + FieldConditionComparator, + FieldConditionEffect, + FieldConditions, + FieldKey, + FieldPrivilegeData, + FieldReferenceData, + FieldType, + FloatData, + IntData, + PermissionData, + RecordType, + RecordTypeCategory, + RecordTypeCategoryKey, + RecordTypeKey, + RecordTypePrivilegeData, + RecordTypeScope, + ReferenceData, + ReferenceDetail, + ReferenceType, + SchemaVersion, + StageKey, + StageDiagram, + StateKey, + StructData, + StructTypeKey, + StructType, + TargetTypeKeyData, + TextData, + TypedReferenceData, +} from './types/external-domain-metadata'; + +// ── Testing utilities (public test-support surface) ── +export { MockServer, MOCK_SERVER_DEFAULT_URL } from './testing/mock-server'; export type { RequestInfo, RetryConfig, RouteConfig, -} from './mock-server/mock-server.interfaces'; -export { processTask } from './multithreading/process-task'; -export { spawn } from './multithreading/spawn/spawn'; -export { WorkerAdapter } from './multithreading/worker-adapter/worker-adapter'; -export { createMockEvent, MOCK_SERVER_DEFAULT_URL } from './common/test-utils'; -export type { DeepPartial } from './common/test-utils'; -export * from './types'; -export * from './types/workers'; +} from './testing/mock-server.interfaces'; +export { createMockEvent } from './testing/mock-event'; +export type { DeepPartial } from './testing/mock-event'; diff --git a/src/logger/logger.constants.ts b/src/logger/logger.constants.ts deleted file mode 100644 index b183cb43..00000000 --- a/src/logger/logger.constants.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { InspectOptions } from 'node:util'; - -export const MAX_LOG_STRING_LENGTH = 10000; -export const MAX_LOG_DEPTH = 10; -export const MAX_LOG_ARRAY_LENGTH = 100; - -export const INSPECT_OPTIONS: InspectOptions = { - compact: false, - breakLength: Infinity, - depth: MAX_LOG_DEPTH, - maxArrayLength: MAX_LOG_ARRAY_LENGTH, - maxStringLength: MAX_LOG_STRING_LENGTH, -}; diff --git a/src/logger/logger.context.ts b/src/logger/logger.context.ts deleted file mode 100644 index 59378788..00000000 --- a/src/logger/logger.context.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { AsyncLocalStorage } from 'node:async_hooks'; - -/** - * Async-aware context storage for tracking whether code is executing within SDK or user context. - * - * AsyncLocalStorage is used instead of a simple global variable because it automatically - * propagates the context across async boundaries (await, Promise.then, setTimeout, etc.). - * This ensures that even after multiple await calls in user code, the logging context - * remains correct without requiring explicit context passing through function parameters. - * - * The stored boolean value indicates: - * - `true`: Code is executing within SDK internals (logs tagged as is_sdk_log: true) - * - `false`: Code is executing within user-provided handlers (logs tagged as is_sdk_log: false) - */ -const sdkLogContext = new AsyncLocalStorage(); - -/** - * Executes a function within user log context, marking all logs as user-originated. - * - * Use this to wrap user-provided callback functions (e.g., task handlers, event callbacks). - * The context automatically propagates through any async operations within the function, - * ensuring that all console.log calls inside are correctly tagged as user logs. - * - * @template T - The return type of the function - * @param fn - The function to execute within user context (can be sync or async) - * @returns The result of the function execution - * - * @example - * ```typescript - * await runWithUserLogContext(async () => { - * console.log('This is a user log'); // is_sdk_log: false - * await someAsyncOperation(); - * console.log('Still a user log'); // is_sdk_log: false (context preserved) - * }); - * ``` - */ -export function runWithUserLogContext(fn: () => T): T { - return sdkLogContext.run(false, fn); -} - -/** - * Executes a function within SDK log context, marking all logs as SDK-originated. - * - * Use this to wrap SDK internal operations (e.g., emit, postState, adapter methods). - * The context automatically propagates through any async operations within the function, - * ensuring that all console.log calls inside are correctly tagged as SDK logs. - * - * This allows proper nesting: SDK code can call user code via runWithUserLogContext, - * and when control returns to SDK code, logs are correctly attributed. - * - * @template T - The return type of the function - * @param fn - The function to execute within SDK context (can be sync or async) - * @returns The result of the function execution - * - * @example - * ```typescript - * await runWithSdkLogContext(async () => { - * console.log('SDK internal log'); // is_sdk_log: true - * runWithUserLogContext(() => { - * console.log('User handler log'); // is_sdk_log: false - * }); - * console.log('Back to SDK log'); // is_sdk_log: true - * }); - * ``` - */ -export function runWithSdkLogContext(fn: () => T): T { - return sdkLogContext.run(true, fn); -} - -/** - * Retrieves the current SDK log context value. - * - * Returns whether the current execution context is within SDK code (true) or user code (false). - * If no context has been set (e.g., during testing or edge cases), returns the provided default. - * - * @param defaultValue - The value to return if no context is currently set - * @returns `true` if in SDK context, `false` if in user context, or defaultValue if unset - */ -export function getSdkLogContextValue(defaultValue: boolean): boolean { - const storeValue = sdkLogContext.getStore(); - if (typeof storeValue === 'boolean') { - return storeValue; - } - return defaultValue; -} diff --git a/src/logger/logger.interfaces.ts b/src/logger/logger.interfaces.ts index 9c39bde9..b8f24944 100644 --- a/src/logger/logger.interfaces.ts +++ b/src/logger/logger.interfaces.ts @@ -1,9 +1,9 @@ import type { RawAxiosResponseHeaders } from 'axios'; -import type { AirdropEvent, EventContext } from '../types/extraction'; +import type { AirSyncEvent, EventContext } from '../types/extraction'; import type { WorkerAdapterOptions } from '../types/workers'; export interface LoggerFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } diff --git a/src/logger/logger.test.ts b/src/logger/logger.test.ts index 71374ec7..1e2781ce 100644 --- a/src/logger/logger.test.ts +++ b/src/logger/logger.test.ts @@ -2,19 +2,17 @@ import { AxiosError } from 'axios'; import { inspect } from 'node:util'; import { LIBRARY_VERSION } from '../common/constants'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; -import { AirdropEvent, EventType } from '../types/extraction'; +import { createMockEvent } from '../testing/mock-event'; +import { AirSyncEvent, EventType } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; import { getPrintableState, + INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, Logger, + MAX_LOG_STRING_LENGTH, serializeAxiosError, serializeError, } from './logger'; -import { - INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, - MAX_LOG_STRING_LENGTH, -} from './logger.constants'; // Mock console methods const mockConsoleInfo = jest.spyOn(console, 'info').mockImplementation(); @@ -32,7 +30,7 @@ jest.mock('node:worker_threads', () => { }); describe(Logger.name, () => { - let mockEvent: AirdropEvent; + let mockEvent: AirSyncEvent; let mockOptions: WorkerAdapterOptions; beforeEach(() => { diff --git a/src/logger/logger.ts b/src/logger/logger.ts index 6d06ef09..3d1c5432 100644 --- a/src/logger/logger.ts +++ b/src/logger/logger.ts @@ -1,14 +1,13 @@ import { AxiosError, isAxiosError, RawAxiosResponseHeaders } from 'axios'; +import { AsyncLocalStorage } from 'node:async_hooks'; import { Console } from 'node:console'; -import { inspect } from 'node:util'; +import { inspect, InspectOptions } from 'node:util'; import { isMainThread, parentPort } from 'node:worker_threads'; import { LIBRARY_VERSION } from '../common/constants'; import { WorkerAdapterOptions, WorkerMessageSubject } from '../types/workers'; -import { INSPECT_OPTIONS } from './logger.constants'; -import { getSdkLogContextValue } from './logger.context'; import { AxiosErrorResponse, LoggerFactoryInterface, @@ -17,7 +16,62 @@ import { PrintableArray, PrintableState, } from './logger.interfaces'; -import { truncateMessage } from '../common/helpers'; + +// ── Log formatting ── + +export const MAX_LOG_STRING_LENGTH = 10000; +const MAX_LOG_DEPTH = 10; +const MAX_LOG_ARRAY_LENGTH = 100; + +export const INSPECT_OPTIONS: InspectOptions = { + compact: false, + breakLength: Infinity, + depth: MAX_LOG_DEPTH, + maxArrayLength: MAX_LOG_ARRAY_LENGTH, + maxStringLength: MAX_LOG_STRING_LENGTH, +}; + +/** + * Truncates a message that exceeds the maximum length, appending a suffix + * noting how many characters were omitted. + */ +export function truncateMessage(message: string): string { + if (message.length > MAX_LOG_STRING_LENGTH) { + return `${message.substring(0, MAX_LOG_STRING_LENGTH)}... ${ + message.length - MAX_LOG_STRING_LENGTH + } more characters`; + } + return message; +} + +// ── SDK log context ── + +/** + * Async-aware flag tracking whether code is running in SDK internals (`true`) + * or user-provided handlers (`false`). AsyncLocalStorage propagates the value + * across await/Promise/timer boundaries, so logs stay correctly attributed + * without threading the flag through call signatures. + */ +const sdkLogContext = new AsyncLocalStorage(); + +/** Runs `fn` with logs marked as user-originated (`is_sdk_log: false`). */ +export function runWithUserLogContext(fn: () => T): T { + return sdkLogContext.run(false, fn); +} + +/** Runs `fn` with logs marked as SDK-originated (`is_sdk_log: true`). */ +export function runWithSdkLogContext(fn: () => T): T { + return sdkLogContext.run(true, fn); +} + +/** Returns the current context flag, or `defaultValue` if none is set. */ +export function getSdkLogContextValue(defaultValue: boolean): boolean { + const storeValue = sdkLogContext.getStore(); + if (typeof storeValue === 'boolean') { + return storeValue; + } + return defaultValue; +} /** * Custom logger that extends Node.js Console with context-aware logging. @@ -232,14 +286,3 @@ export function serializeAxiosError(error: AxiosError): AxiosErrorResponse { return serializedAxiosError; } - -/** - * Formats an Axios error to a printable format. - * - * @param error - Axios error to format - * @returns Formatted error object - * @deprecated Use {@link serializeAxiosError} instead - */ -export function formatAxiosError(error: AxiosError): object { - return serializeAxiosError(error); -} diff --git a/src/mappers/mappers.interface.ts b/src/mappers/mappers.interfaces.ts similarity index 99% rename from src/mappers/mappers.interface.ts rename to src/mappers/mappers.interfaces.ts index b37272a1..f601ef98 100644 --- a/src/mappers/mappers.interface.ts +++ b/src/mappers/mappers.interfaces.ts @@ -1,4 +1,4 @@ -import { AirdropEvent } from '../types'; +import { AirSyncEvent } from '../types/extraction'; import { DonV2 } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; @@ -6,7 +6,7 @@ import { WorkerAdapterOptions } from '../types/workers'; * Configuration interface for creating a Mappers instance. */ export interface MappersFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } diff --git a/src/mappers/mappers.test.ts b/src/mappers/mappers.test.ts index 68a8ef23..54e8f3d4 100644 --- a/src/mappers/mappers.test.ts +++ b/src/mappers/mappers.test.ts @@ -1,5 +1,6 @@ -import { createMockEvent, MOCK_SERVER_DEFAULT_URL } from '../common/test-utils'; -import { axiosClient } from '../http/axios-client-internal'; +import { createMockEvent } from '../testing/mock-event'; +import { MOCK_SERVER_DEFAULT_URL } from '../testing/mock-server'; +import { axiosClient } from '../http/client'; import { EventType } from '../types/extraction'; import { Mappers } from './mappers'; import { @@ -9,10 +10,10 @@ import { MappersUpdateParams, SyncMapperRecordStatus, SyncMapperRecordTargetType, -} from './mappers.interface'; +} from './mappers.interfaces'; // Mock the axios client -jest.mock('../http/axios-client-internal'); +jest.mock('../http/client'); const mockAxiosClient = axiosClient as jest.Mocked; describe(Mappers.name, () => { diff --git a/src/mappers/mappers.ts b/src/mappers/mappers.ts index c552b02c..1982c353 100644 --- a/src/mappers/mappers.ts +++ b/src/mappers/mappers.ts @@ -1,6 +1,4 @@ -import { AxiosResponse } from 'axios'; - -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { MappersCreateParams, @@ -12,7 +10,7 @@ import { MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse, -} from './mappers.interface'; +} from './mappers.interfaces'; /** * Manages sync mapper records that link external system items to DevRev items. @@ -34,13 +32,13 @@ export class Mappers { * Used to find the mapping when you know the DevRev ID and want to find the external system ID. * * @param params - Query parameters of type MappersGetByTargetIdParams - * @returns Promise with response data containing the sync mapper record + * @returns Promise resolving to the sync mapper record */ async getByTargetId( params: MappersGetByTargetIdParams - ): Promise> { + ): Promise { const { sync_unit, target } = params; - return axiosClient.get( + const response = await axiosClient.get( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-target`, { headers: { @@ -49,6 +47,7 @@ export class Mappers { params: { sync_unit, target }, } ); + return response.data; } /** @@ -57,13 +56,13 @@ export class Mappers { * Used to find the mapping when you know the external system ID and want to find the DevRev ID. * * @param params - Query parameters of type MappersGetByExternalIdParams - * @returns Promise with response data containing the sync mapper record + * @returns Promise resolving to the sync mapper record */ async getByExternalId( params: MappersGetByExternalIdParams - ): Promise> { + ): Promise { const { sync_unit, external_id, target_type } = params; - return axiosClient.get( + const response = await axiosClient.get( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-external-id`, { headers: { @@ -72,6 +71,7 @@ export class Mappers { params: { sync_unit, external_id, target_type }, } ); + return response.data; } /** @@ -82,12 +82,10 @@ export class Mappers { * the mapping for future synchronization operations. * * @param params - Creation parameters of type MappersCreateParams - * @returns Promise with response data containing the created sync mapper record + * @returns Promise resolving to the created sync mapper record */ - async create( - params: MappersCreateParams - ): Promise> { - return axiosClient.post( + async create(params: MappersCreateParams): Promise { + const response = await axiosClient.post( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.create`, params, { @@ -96,6 +94,7 @@ export class Mappers { }, } ); + return response.data; } /** @@ -105,12 +104,10 @@ export class Mappers { * additional DevRev entities need to be associated. * * @param params - Update parameters of type MappersUpdateParams - * @returns Promise with response data containing the updated sync mapper record + * @returns Promise resolving to the updated sync mapper record */ - async update( - params: MappersUpdateParams - ): Promise> { - return axiosClient.post( + async update(params: MappersUpdateParams): Promise { + const response = await axiosClient.post( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.update`, params, { @@ -119,5 +116,6 @@ export class Mappers { }, } ); + return response.data; } } diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts new file mode 100644 index 00000000..0cca182f --- /dev/null +++ b/src/multithreading/adapters/base-adapter.ts @@ -0,0 +1,220 @@ +import { parentPort } from 'node:worker_threads'; + +import { STATELESS_EVENT_TYPES } from '../../common/constants'; +import { emit } from '../emit'; +import { + runWithSdkLogContext, + serializeError, + truncateMessage, +} from '../../logger/logger'; +import { BaseState } from '../../state/state'; +import { SdkState } from '../../state/state.interfaces'; +import { + AirSyncEvent, + EventData, + ExtractorEventType, +} from '../../types/extraction'; +import { LoaderEventType } from '../../types/loading'; +import { + TaskResult, + WorkerAdapterOptions, + WorkerMessageEmitted, + WorkerMessageSubject, +} from '../../types/workers'; +import { Uploader } from '../../uploader/uploader'; +import { getEventTypeForResult } from '../spawn/spawn.helpers'; + +/** + * BaseAdapter holds the state and behavior shared by both sync modes and owns + * the `emit` control-protocol flow as a template method. Mode-specific adapters + * (`ExtractionAdapter`, `LoadingAdapter`) implement the abstract hooks to inject + * their own pre-emit work and event payload shaping. + * + * @typeParam ConnectorState - the connector-owned state shape + */ +export abstract class BaseAdapter { + readonly event: AirSyncEvent; + readonly options?: WorkerAdapterOptions; + isTimeout: boolean; + hasWorkerEmitted: boolean; + + protected adapterState: BaseState; + protected uploader: Uploader; + + constructor({ + event, + adapterState, + options, + }: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + this.event = event; + this.options = options; + this.adapterState = adapterState; + this.hasWorkerEmitted = false; + this.isTimeout = false; + this.uploader = new Uploader({ + event, + options, + }); + } + + /** Connector-owned state exposed to snap-in code. */ + get state(): ConnectorState { + return this.adapterState.state; + } + + set state(value: ConnectorState) { + this.adapterState.state = value; + } + + /** SDK-internal bookkeeping state. Used by SDK internals; not for connector use. */ + get sdkState(): SdkState { + return this.adapterState.sdkState; + } + + get extractionScope() { + return this.adapterState.extractionScope; + } + + async postState() { + return runWithSdkLogContext(async () => { + await this.adapterState.postState(); + }); + } + + /** + * Pre-emit hook run before any state is persisted or the event is sent. + * Extraction uploads pending repos and updates extraction boundaries here; + * loading has nothing to do. Throwing aborts the emit (the caller signals + * worker exit). + */ + protected abstract beforeEmit( + newEventType: ExtractorEventType | LoaderEventType + ): Promise; + + /** + * Builds the mode-specific extras merged into the emitted event payload + * (extraction: artifacts; loading: reports + processed files). + */ + protected abstract buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData; + + /** + * Post-emit hook run after the event has been sent successfully. Extraction + * clears its accumulated artifacts here; loading has nothing to do. + */ + protected abstract afterEmit( + newEventType: ExtractorEventType | LoaderEventType + ): void; + + /** + * Maps a {@link TaskResult} returned by a worker's task/onTimeout callback to + * the phase-appropriate platform event and emits it exactly once. + * + * This is the SDK-internal bridge between the return-based connector contract + * and the control protocol; it is invoked by the worker driver, not by + * connectors. Connectors signal outcomes by returning a `TaskResult`, never by + * calling `emit` directly. + * + * @param result - The status the worker reported for the current phase. + */ + async emitFromResult(result: TaskResult): Promise { + const { eventType, illegal } = getEventTypeForResult( + this.event.payload.event_type, + result.status + ); + + const data: EventData = {}; + if (result.status === 'delay') { + data.delay = result.delaySeconds; + } else if (result.status === 'error') { + data.error = result.error; + } else if (illegal) { + data.error = { + message: `Worker returned status '${result.status}' for a non-resumable phase (${this.event.payload.event_type}), which is not allowed. Emitting an error event instead.`, + }; + } + + await this.emit(eventType, data); + } + + /** + * Emits an event to the platform. + * + * @param newEventType - The event type to be emitted + * @param data - The data to be sent with the event + */ + protected async emit( + newEventType: ExtractorEventType | LoaderEventType, + data?: EventData + ): Promise { + return runWithSdkLogContext(async () => { + if (this.hasWorkerEmitted) { + console.warn( + `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` + ); + return; + } + + try { + await this.beforeEmit(newEventType); + } catch (error) { + console.error('Error while preparing to emit event', error); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + return; + } + + // We want to save the state every time we emit an event, except for the start and delete events + if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { + console.log( + `Saving state before emitting event with event type: ${newEventType}.` + ); + + try { + await this.adapterState.postState(); + } catch (error) { + console.error('Error while posting state', error); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + return; + } + } + + try { + // Always prune error messages to make them shorter before emit + if (data?.error?.message) { + data.error.message = truncateMessage(data.error.message); + } + + await emit({ + eventType: newEventType, + event: this.event, + data: { + ...data, + ...this.buildEmitPayload(newEventType), + }, + }); + + const message: WorkerMessageEmitted = { + subject: WorkerMessageSubject.WorkerMessageEmitted, + payload: { eventType: newEventType }, + }; + this.afterEmit(newEventType); + parentPort?.postMessage(message); + this.hasWorkerEmitted = true; + } catch (error) { + console.error( + `Error while emitting event with event type: ${newEventType}.`, + serializeError(error) + ); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + } + }); + } +} diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts new file mode 100644 index 00000000..5d01e443 --- /dev/null +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -0,0 +1,485 @@ +import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; +import { + AirSyncDefaultItemTypes, + EVENT_SIZE_THRESHOLD_BYTES, + SSOR_ATTACHMENT, +} from '../../common/constants'; +import { + runWithSdkLogContext, + runWithUserLogContext, + serializeError, +} from '../../logger/logger'; +import { Repo } from '../../repo/repo'; +import { + NormalizedAttachment, + RepoInterface, +} from '../../repo/repo.interfaces'; +import { + AirSyncEvent, + EventData, + ExternalSystemAttachmentProcessors, + ExternalSystemAttachmentStreamingFunction, + ExtractorEventType, + HttpStreamResponse, + ProcessAttachmentReturnType, +} from '../../types/extraction'; +import { LoaderEventType } from '../../types/loading'; +import { BaseState } from '../../state/state'; +import { TaskResult, WorkerAdapterOptions } from '../../types/workers'; +import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; + +import { BaseAdapter } from './base-adapter'; + +/** + * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the + * extraction surface (repos, artifacts, attachment streaming) and uploads + * pending repos and updates the extraction boundaries before emitting. + * + * @public + */ +export class ExtractionAdapter< + ConnectorState +> extends BaseAdapter { + private _artifacts: Artifact[]; + private repos: Repo[] = []; + private currentEventDataLength: number = 0; + + constructor(params: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + super(params); + this._artifacts = []; + } + + /** + * Returns whether the given item type should be extracted. + * Defaults to true if the scope is empty or the item type is not listed. + */ + shouldExtract(itemType: string): boolean { + const scope = this.extractionScope; + if (Object.keys(scope).length === 0) return true; + if (!(itemType in scope)) return true; + return scope[itemType].extract; + } + + initializeRepos(repos: RepoInterface[]) { + this.repos = repos.map((repo) => { + const shouldNormalize = + repo.itemType !== AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && + repo.itemType !== SSOR_ATTACHMENT; + + return new Repo({ + event: this.event, + itemType: repo.itemType, + ...(shouldNormalize && { normalize: repo.normalize }), + onUpload: (artifact: Artifact) => { + // We need to store artifacts ids in state for later use when streaming attachments + if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { + this.sdkState.toDevRev?.attachmentsMetadata.artifactIds.push( + artifact.id + ); + } + + // Calculate size of the entire artifact object that goes in the SQS message + this.currentEventDataLength += Buffer.byteLength( + JSON.stringify(artifact), + 'utf8' + ); + + if ( + this.currentEventDataLength > EVENT_SIZE_THRESHOLD_BYTES && + !this.isTimeout + ) { + this.isTimeout = true; + } + }, + options: { + ...this.options, + ...repo.overridenOptions, + }, + }); + }); + } + + getRepo(itemType: string): Repo | undefined { + return runWithSdkLogContext(() => { + const repo = this.repos.find((repo) => repo.itemType === itemType); + + if (!repo) { + console.error(`Repo for item type ${itemType} not found.`); + return; + } + + return repo; + }); + } + + get artifacts(): Artifact[] { + return this._artifacts; + } + + set artifacts(artifacts: Artifact[]) { + this._artifacts = this._artifacts + .concat(artifacts) + .filter((value, index, self) => self.indexOf(value) === index); + } + + protected async beforeEmit( + newEventType: ExtractorEventType | LoaderEventType + ): Promise { + // Upload all repos before emitting the event + console.log( + `Uploading all repos before emitting event with event type: ${newEventType}.` + ); + await this.uploadAllRepos(); + + // When the full extraction cycle completes, commit the extraction window + // boundaries so subsequent incremental syncs can resume from them. + if (newEventType === ExtractorEventType.AttachmentExtractionDone) { + const sdkState = this.sdkState; + + // Clear pending extraction boundaries now that the cycle is complete + sdkState.pendingWorkersOldest = ''; + sdkState.pendingWorkersNewest = ''; + + // Update workersOldest and workersNewest boundaries from resolved extraction timestamps. + // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest. + const extractionStart = this.event.payload.event_context.extract_from; + const extractionEnd = this.event.payload.event_context.extract_to; + + if ( + extractionStart && + (!sdkState.workersOldest || extractionStart < sdkState.workersOldest) + ) { + console.log( + `Updating workersOldest from '${sdkState.workersOldest}' to '${extractionStart}'.` + ); + sdkState.workersOldest = extractionStart; + } + + if ( + extractionEnd && + (!sdkState.workersNewest || extractionEnd > sdkState.workersNewest) + ) { + console.log( + `Updating workersNewest from '${sdkState.workersNewest}' to '${extractionEnd}'.` + ); + sdkState.workersNewest = extractionEnd; + } + } + } + + protected buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData { + const isExtractionEvent = Object.values(ExtractorEventType).includes( + newEventType as ExtractorEventType + ); + return isExtractionEvent ? { artifacts: this.artifacts } : {}; + } + + protected afterEmit(): void { + this.artifacts = []; + } + + async uploadAllRepos(): Promise { + for (const repo of this.repos) { + const error = await repo.upload(); + this.artifacts.push(...repo.uploadedArtifacts); + if (error) { + throw error; + } + } + } + + async processAttachment( + attachment: NormalizedAttachment, + stream: ExternalSystemAttachmentStreamingFunction + ): Promise { + return runWithSdkLogContext(async () => { + const { httpStream, delay, error } = await runWithUserLogContext( + async () => + stream({ + item: attachment, + event: this.event, + }) + ); + + if (error) { + return { error }; + } else if (delay) { + return { delay }; + } + + if (httpStream) { + const fileType = + attachment.content_type || + httpStream.headers['content-type']?.toString() || + 'application/octet-stream'; + const contentLength = httpStream.headers['content-length']?.toString(); + const fileSize = contentLength ? parseInt(contentLength) : undefined; + + // Get upload URL + const { error: artifactUrlError, response: artifactUrlResponse } = + await this.uploader.getArtifactUploadUrl( + attachment.file_name, + fileType, + fileSize + ); + + if (artifactUrlError) { + this.destroyHttpStream(httpStream); + return { + error: { + message: `Error while preparing artifact for attachment ID ${ + attachment.id + }. Skipping attachment. ${serializeError(artifactUrlError)}`, + fileSize: fileSize, + }, + }; + } + + // Stream attachment + const { error: uploadedArtifactError } = + await this.uploader.streamArtifact(artifactUrlResponse!, httpStream); + + if (uploadedArtifactError) { + this.destroyHttpStream(httpStream); + return { + error: { + message: + `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` + + serializeError(uploadedArtifactError), + fileSize: fileSize, + }, + }; + } + + // Confirm attachment upload + const { error: confirmArtifactUploadError } = + await this.uploader.confirmArtifactUpload( + artifactUrlResponse!.artifact_id + ); + if (confirmArtifactUploadError) { + return { + error: { + message: + `Error while confirming upload for attachment ID ${attachment.id}. ` + + serializeError(confirmArtifactUploadError), + fileSize: fileSize, + }, + }; + } + + const ssorAttachment: SsorAttachment = { + id: { + devrev: artifactUrlResponse!.artifact_id, + external: attachment.id, + }, + parent_id: { + external: attachment.parent_id, + }, + }; + + if (attachment.author_id) { + ssorAttachment.actor_id = { + external: attachment.author_id, + }; + } + + // This will set inline flag in ssor_attachment only if it is explicity + // set in the attachment object. + if (attachment.inline === true) { + ssorAttachment.inline = true; + } else if (attachment.inline === false) { + ssorAttachment.inline = false; + } + + await this.getRepo('ssor_attachment')?.push([ssorAttachment]); + return; + } + return { + error: { + message: `Error while opening attachment stream. Skipping attachment.`, + }, + }; + }); + } + + /** + * Destroys a stream to prevent memory leaks. + * @param httpStream - The axios response stream to destroy + */ + private destroyHttpStream(httpStream: HttpStreamResponse): void { + try { + if (httpStream && httpStream.data) { + if (typeof httpStream.data.destroy === 'function') { + httpStream.data.destroy(); + } else if (typeof httpStream.data.close === 'function') { + httpStream.data.close(); + } + } + } catch (error) { + console.warn('Error while destroying HTTP stream:', error); + } + } + + /** + * Streams the attachments to the DevRev platform. + * The attachments are streamed to the platform and the artifact information is returned. + * @param params - The parameters to stream the attachments + * @returns The response object containing the ssorAttachment artifact information + * or error information if there was an error + */ + async streamAttachments({ + stream, + processors, + batchSize = 1, // By default, we want to stream one attachment at a time + }: { + stream: ExternalSystemAttachmentStreamingFunction; + processors?: ExternalSystemAttachmentProcessors< + ConnectorState, + NormalizedAttachment[], + NewBatch + >; + batchSize?: number; + }): Promise { + return runWithSdkLogContext(async () => { + if (batchSize <= 0) { + console.warn( + `The specified batch size (${batchSize}) is invalid. Using 1 instead.` + ); + batchSize = 1; + } + + if (batchSize > 50) { + console.warn( + `The specified batch size (${batchSize}) is too large. Using 50 instead.` + ); + batchSize = 50; + } + + const repos = [ + { + itemType: 'ssor_attachment', + }, + ]; + this.initializeRepos(repos); + + const attachmentsMetadata = this.sdkState.toDevRev?.attachmentsMetadata; + + // If there are no attachments metadata artifact IDs in state, finish here + if (!attachmentsMetadata?.artifactIds?.length) { + console.log(`No attachments metadata artifact IDs found in state.`); + return { status: 'success' }; + } else { + console.log( + `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` + ); + } + + // Loop through the attachments metadata artifact IDs + while (attachmentsMetadata.artifactIds.length > 0) { + const attachmentsMetadataArtifactId = + attachmentsMetadata.artifactIds[0]; + + console.log( + `Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.` + ); + + const { attachments, error } = + await this.uploader.getAttachmentsFromArtifactId({ + artifact: attachmentsMetadataArtifactId, + }); + + if (error) { + console.error( + `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + return { status: 'error', error }; + } + + if (!attachments || attachments.length === 0) { + console.warn( + `No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + // Remove empty artifact and reset lastProcessed + attachmentsMetadata.artifactIds.shift(); + attachmentsMetadata.lastProcessed = 0; + continue; + } + + console.log( + `Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + + let response; + + if (processors) { + console.log(`Using custom processors for attachments.`); + + const reducer = processors.reducer; + const iterator = processors.iterator; + + const reducedAttachments = runWithUserLogContext(() => + reducer({ + attachments, + adapter: this, + batchSize, + }) + ); + + response = await runWithUserLogContext(async () => { + return await iterator({ + reducedAttachments, + adapter: this, + stream, + }); + }); + } else { + console.log( + `Using attachments streaming pool for attachments streaming.` + ); + + const attachmentsPool = new AttachmentsStreamingPool({ + adapter: this, + attachments, + batchSize, + stream, + }); + + response = await attachmentsPool.streamAll(); + } + + if (response?.error) { + return { status: 'error', error: response.error }; + } + + if (response?.delay) { + return { status: 'delay', delaySeconds: response.delay }; + } + + // On timeout, return progress to allow continuation in a fresh invocation. + if (this.isTimeout) { + console.log( + `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.` + ); + return { status: 'progress' }; + } + + console.log( + `Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + attachmentsMetadata.artifactIds.shift(); + attachmentsMetadata.lastProcessed = 0; + if (attachmentsMetadata.lastProcessedAttachmentsIdsList) { + attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0; + } + } + + return { status: 'success' }; + }); + } +} diff --git a/src/multithreading/worker-adapter/worker-adapter.helpers.ts b/src/multithreading/adapters/loading-adapter.helpers.ts similarity index 100% rename from src/multithreading/worker-adapter/worker-adapter.helpers.ts rename to src/multithreading/adapters/loading-adapter.helpers.ts diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts new file mode 100644 index 00000000..3bf1551b --- /dev/null +++ b/src/multithreading/adapters/loading-adapter.ts @@ -0,0 +1,611 @@ +import axios from 'axios'; + +import { + runWithSdkLogContext, + runWithUserLogContext, + serializeError, +} from '../../logger/logger'; +import { Mappers } from '../../mappers/mappers'; +import { SyncMapperRecordStatus } from '../../mappers/mappers.interfaces'; +import { BaseState } from '../../state/state'; +import { + AirSyncEvent, + EventData, + EventType, + ExtractorEventType, +} from '../../types/extraction'; +import { + ActionType, + ExternalSystemAttachment, + ExternalSystemItem, + ExternalSystemLoadingFunction, + FileToLoad, + ItemTypesToLoadParams, + ItemTypeToLoad, + LoaderEventType, + LoaderReport, + LoadItemResponse, + StatsFileObject, +} from '../../types/loading'; +import { TaskResult, WorkerAdapterOptions } from '../../types/workers'; + +import { BaseAdapter } from './base-adapter'; +import { + addReportToLoaderReport, + getFilesToLoad, +} from './loading-adapter.helpers'; + +/** + * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading + * surface (item/attachment loading, mappers, loader reports). + * + * @public + */ +export class LoadingAdapter< + ConnectorState +> extends BaseAdapter { + private loaderReports: LoaderReport[]; + private _processedFiles: string[]; + private _mappers: Mappers; + + constructor(params: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + super(params); + this.loaderReports = []; + this._processedFiles = []; + this._mappers = new Mappers({ + event: params.event, + options: params.options, + }); + } + + get reports(): LoaderReport[] { + return this.loaderReports; + } + + get processedFiles(): string[] { + return this._processedFiles; + } + + get mappers(): Mappers { + return this._mappers; + } + + protected async beforeEmit(): Promise { + // Loading has no pre-emit work (no repos, no extraction boundaries). + } + + protected buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData { + const isLoaderEvent = Object.values(LoaderEventType).includes( + newEventType as LoaderEventType + ); + return isLoaderEvent + ? { + reports: this.reports, + processed_files: this.processedFiles, + } + : {}; + } + + protected afterEmit(): void { + // Loading keeps its accumulated reports/processed files across emits. + } + + async loadItemTypes({ + itemTypesToLoad, + }: ItemTypesToLoadParams): Promise { + return runWithSdkLogContext(async () => { + if (this.event.payload.event_type === EventType.StartLoadingData) { + const itemTypes = itemTypesToLoad.map( + (itemTypeToLoad) => itemTypeToLoad.itemType + ); + + if (!itemTypes.length) { + console.warn('No item types to load, returning.'); + return { status: 'success' }; + } + + const filesToLoad = await this.getLoaderBatches({ + supportedItemTypes: itemTypes, + }); + this.sdkState.fromDevRev = { + filesToLoad, + }; + } + + if ( + !this.sdkState.fromDevRev || + !this.sdkState.fromDevRev.filesToLoad.length + ) { + console.warn('No files to load, returning.'); + return { status: 'success' }; + } + + console.log( + 'Files to load in state', + this.sdkState.fromDevRev?.filesToLoad + ); + + try { + for (const fileToLoad of this.sdkState.fromDevRev.filesToLoad) { + const itemTypeToLoad = itemTypesToLoad.find( + (itemTypeToLoad: ItemTypeToLoad) => + itemTypeToLoad.itemType === fileToLoad.itemType + ); + + if (!itemTypeToLoad) { + console.error( + `Item type to load not found for item type: ${fileToLoad.itemType}.` + ); + + return { + status: 'error', + error: { + message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, + }, + }; + } + + if (!fileToLoad.completed) { + const { response, error: transformerFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: fileToLoad.id, + isGzipped: true, + }); + + if (transformerFileError) { + console.error( + `Transformer file not found for artifact ID: ${fileToLoad.id}.` + ); + return { + status: 'error', + error: { + message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, + }, + }; + } + + const transformerFile = response as ExternalSystemItem[]; + + for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { + if (this.isTimeout) { + console.log( + 'Timeout detected during data loading. Returning progress to allow continuation.' + ); + return { status: 'progress' }; + } + + const { report, rateLimit } = await this.loadItem({ + item: transformerFile[i], + itemTypeToLoad, + }); + + if (rateLimit?.delay) { + return { status: 'delay', delaySeconds: rateLimit.delay }; + } + + if (report) { + addReportToLoaderReport({ + loaderReports: this.loaderReports, + report, + }); + fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; + } + } + + fileToLoad.completed = true; + this._processedFiles.push(fileToLoad.id); + } + } + } catch (error) { + console.error('Error during data loading.', serializeError(error)); + return { + status: 'error', + error: { + message: `Error during data loading. ${serializeError(error)}`, + }, + }; + } + + return { status: 'success' }; + }); + } + + async getLoaderBatches({ + supportedItemTypes, + }: { + supportedItemTypes: string[]; + }) { + return runWithSdkLogContext(async () => { + const statsFileArtifactId = this.event.payload.event_data?.stats_file; + + if (statsFileArtifactId) { + const { response, error: statsFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: statsFileArtifactId, + }); + + const statsFile = response as StatsFileObject[]; + + if (statsFileError || statsFile.length === 0) { + return [] as FileToLoad[]; + } + + const filesToLoad = getFilesToLoad({ + supportedItemTypes, + statsFile, + }); + + return filesToLoad; + } + + return [] as FileToLoad[]; + }); + } + + async loadAttachments({ + create, + }: { + create: ExternalSystemLoadingFunction; + }): Promise { + return runWithSdkLogContext(async () => { + if (this.event.payload.event_type === EventType.StartLoadingAttachments) { + this.sdkState.fromDevRev = { + filesToLoad: await this.getLoaderBatches({ + supportedItemTypes: ['attachment'], + }), + }; + } + + if ( + !this.sdkState.fromDevRev || + this.sdkState.fromDevRev?.filesToLoad.length === 0 + ) { + console.log('No files to load, returning.'); + return { status: 'success' }; + } + + const filesToLoad = this.sdkState.fromDevRev?.filesToLoad; + + try { + for (const fileToLoad of filesToLoad) { + if (!fileToLoad.completed) { + const { response, error: transformerFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: fileToLoad.id, + isGzipped: true, + }); + + const transformerFile = response as ExternalSystemAttachment[]; + + if (transformerFileError) { + console.error( + `Transformer file not found for artifact ID: ${fileToLoad.id}.` + ); + return { + status: 'error', + error: { + message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, + }, + }; + } + + for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { + if (this.isTimeout) { + console.log( + 'Timeout detected during attachment loading. Returning progress to allow continuation.' + ); + return { status: 'progress' }; + } + + const { report, rateLimit } = await this.loadAttachment({ + item: transformerFile[i], + create, + }); + + if (rateLimit?.delay) { + return { status: 'delay', delaySeconds: rateLimit.delay }; + } + + if (report) { + addReportToLoaderReport({ + loaderReports: this.loaderReports, + report, + }); + fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; + } + } + + fileToLoad.completed = true; + this._processedFiles.push(fileToLoad.id); + } + } + } catch (error) { + console.error( + 'Error during attachment loading.', + serializeError(error) + ); + return { + status: 'error', + error: { + message: `Error during attachment loading. ${serializeError( + error + )}`, + }, + }; + } + + return { status: 'success' }; + }); + } + + async loadItem({ + item, + itemTypeToLoad, + }: { + item: ExternalSystemItem; + itemTypeToLoad: ItemTypeToLoad; + }): Promise { + return runWithSdkLogContext(async () => { + const devrevId = item.id.devrev; + + try { + const syncMapperRecord = await this._mappers.getByTargetId({ + sync_unit: this.event.payload.event_context.sync_unit, + target: devrevId, + }); + + if (!syncMapperRecord) { + console.warn('Failed to get sync mapper record from response.'); + return { + error: { + message: 'Failed to get sync mapper record from response.', + }, + }; + } + + // Update item in external system + const { id, modifiedDate, delay, error } = await runWithUserLogContext( + async () => { + return await itemTypeToLoad.update({ + item, + mappers: this._mappers, + event: this.event, + }); + } + ); + + if (id) { + try { + const syncMapperRecordUpdate = await this._mappers.update({ + id: syncMapperRecord.sync_mapper_record.id, + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + ...(modifiedDate && { + external_versions: { + add: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }, + }), + external_ids: { + add: [id], + }, + targets: { + add: [devrevId], + }, + }); + + console.log( + 'Successfully updated sync mapper record.', + syncMapperRecordUpdate + ); + } catch (error) { + console.warn( + 'Failed to update sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to update sync mapper record' + serializeError(error), + }, + }; + } + + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.UPDATED]: 1, + }, + }; + } else if (delay) { + console.log( + `Rate limited while updating item in external system, delaying for ${delay} seconds.` + ); + + return { + rateLimit: { + delay, + }, + }; + } else { + console.warn('Failed to update item in external system', error); + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.FAILED]: 1, + }, + }; + } + + // TODO: Update mapper (optional) + } catch (error) { + if (axios.isAxiosError(error)) { + if (error.response?.status === 404) { + // Create item in external system if mapper record not found + const { id, modifiedDate, delay, error } = + await runWithUserLogContext(async () => { + return await itemTypeToLoad.create({ + item, + mappers: this._mappers, + event: this.event, + }); + }); + + if (id) { + // Create mapper + try { + const syncMapperRecordCreate = await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + external_ids: [id], + targets: [devrevId], + ...(modifiedDate && { + external_versions: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }), + }); + + console.log( + 'Successfully created sync mapper record.', + syncMapperRecordCreate + ); + + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.CREATED]: 1, + }, + }; + } catch (error) { + console.warn( + 'Failed to create sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to create sync mapper record. ' + + serializeError(error), + }, + }; + } + } else if (delay) { + return { + rateLimit: { + delay, + }, + }; + } else { + console.warn( + 'Failed to create item in external system.', + serializeError(error) + ); + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.FAILED]: 1, + }, + }; + } + } else { + console.warn( + 'Failed to get sync mapper record.', + serializeError(error) + ); + return { + error: { + message: error.message, + }, + }; + } + } + + console.warn( + 'Failed to get sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to get sync mapper record. ' + serializeError(error), + }, + }; + } + }); + } + + async loadAttachment({ + item, + create, + }: { + item: ExternalSystemAttachment; + create: ExternalSystemLoadingFunction; + }): Promise { + return runWithSdkLogContext(async () => { + // Create item + const { id, delay, error } = await runWithUserLogContext(async () => + create({ + item, + mappers: this._mappers, + event: this.event, + }) + ); + + if (delay) { + return { + rateLimit: { + delay, + }, + }; + } else if (id) { + try { + const syncMapperRecordCreate = await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + external_ids: [id], + targets: [item.reference_id], + status: SyncMapperRecordStatus.OPERATIONAL, + }); + + console.log( + 'Successfully created sync mapper record.', + syncMapperRecordCreate + ); + } catch (error) { + console.warn( + 'Failed to create sync mapper record.', + serializeError(error) + ); + } + + return { + report: { + item_type: 'attachment', + [ActionType.CREATED]: 1, + }, + }; + } else { + console.warn('Failed to create attachment in external system', error); + return { + report: { + item_type: 'attachment', + [ActionType.FAILED]: 1, + }, + }; + } + }); + } +} diff --git a/src/multithreading/create-worker.test.ts b/src/multithreading/create-worker.test.ts index 0c9eb07e..6aa23bbd 100644 --- a/src/multithreading/create-worker.test.ts +++ b/src/multithreading/create-worker.test.ts @@ -1,7 +1,7 @@ import { isMainThread, Worker } from 'worker_threads'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; import { createWorker } from './create-worker'; @@ -10,7 +10,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); // Act @@ -38,7 +38,7 @@ describe(createWorker.name, () => { (isMainThread as any) = false; const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); // Act & Assert @@ -58,7 +58,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); if (isMainThread) { @@ -85,7 +85,7 @@ describe(createWorker.name, () => { }, }; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }); if (isMainThread) { @@ -106,7 +106,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionMetadataStart }, + payload: { event_type: EventType.StartExtractingMetadata }, }); if (isMainThread) { diff --git a/src/common/control-protocol.ts b/src/multithreading/emit.ts similarity index 71% rename from src/common/control-protocol.ts rename to src/multithreading/emit.ts index c372c91d..f3e41ac3 100644 --- a/src/common/control-protocol.ts +++ b/src/multithreading/emit.ts @@ -1,18 +1,17 @@ import { AxiosResponse } from 'axios'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { - AirdropEvent, + AirSyncEvent, EventData, ExtractorEvent, ExtractorEventType, LoaderEvent, } from '../types/extraction'; import { LoaderEventType } from '../types/loading'; -import { LIBRARY_VERSION } from './constants'; -import { translateOutgoingEventType } from './event-type-translation'; +import { LIBRARY_VERSION } from '../common/constants'; export interface EmitInterface { - event: AirdropEvent; + event: AirSyncEvent; eventType: ExtractorEventType | LoaderEventType; data?: EventData; } @@ -22,12 +21,8 @@ export const emit = async ({ eventType, data, }: EmitInterface): Promise => { - // Translate outgoing event type to ensure we always send new event types - // TODO: Remove when the old types are completely phased out - const translatedEventType = translateOutgoingEventType(eventType); - const newEvent: ExtractorEvent | LoaderEvent = { - event_type: translatedEventType, + event_type: eventType, event_context: event.payload.event_context, event_data: { ...data, diff --git a/src/multithreading/process-task.test.ts b/src/multithreading/process-task.test.ts index 015c5612..9c868ce3 100644 --- a/src/multithreading/process-task.test.ts +++ b/src/multithreading/process-task.test.ts @@ -3,9 +3,6 @@ import { WorkerEvent, WorkerMessageSubject } from '../types/workers'; // These tests cover logic that is NOT exercised by the end-to-end integration // tests under src/tests/timeout-handling/: -// - translation of legacy wire event types into the new enum (mutates event in place) -// - the hasWorkerEmitted guard that prevents onTimeout from firing after a -// successful emit (integration tests only exercise the positive case) // - the error branch that posts WorkerMessageFailed and exits(1) // - the WorkerMessage handler's guard that only flips isTimeout on // WorkerMessageExit (integration tests can't cleanly target non-Exit messages) @@ -34,10 +31,6 @@ jest.mock('node:worker_threads', () => ({ }, })); -jest.mock('../common/event-type-translation', () => ({ - translateIncomingEventType: jest.fn((t: string) => t), -})); - jest.mock('../logger/logger', () => ({ Logger: jest.fn().mockImplementation(() => ({ log: jest.fn(), @@ -47,29 +40,25 @@ jest.mock('../logger/logger', () => ({ logFn: jest.fn(), })), serializeError: jest.fn((e: unknown) => String(e)), -})); - -jest.mock('../logger/logger.context', () => ({ runWithSdkLogContext: jest.fn((fn: () => unknown) => fn()), runWithUserLogContext: jest.fn((fn: () => unknown) => fn()), })); -jest.mock('../state/state', () => ({ - createAdapterState: jest.fn(), +jest.mock('../state/extraction-state', () => ({ + createExtractionState: jest.fn(), })); -jest.mock('./worker-adapter/worker-adapter', () => ({ - WorkerAdapter: jest.fn().mockImplementation(() => ({ +jest.mock('./adapters/extraction-adapter', () => ({ + ExtractionAdapter: jest.fn().mockImplementation(() => ({ isTimeout: false, - hasWorkerEmitted: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), })), })); -import { processTask } from './process-task'; -import { translateIncomingEventType } from '../common/event-type-translation'; -import { createAdapterState } from '../state/state'; -import { WorkerAdapter } from './worker-adapter/worker-adapter'; -import { createMockEvent } from '../common/test-utils'; +import { processExtractionTask } from './process-task'; +import { createExtractionState } from '../state/extraction-state'; +import { ExtractionAdapter } from './adapters/extraction-adapter'; +import { createMockEvent } from '../testing/mock-event'; function setWorkerData(data: Record) { (global as Record).__workerData__ = data; @@ -84,7 +73,7 @@ function makeEvent(eventType = EventType.StartExtractingData) { // Flush the microtask queue enough to let the async IIFE inside processTask run. const flush = async () => new Promise((r) => setTimeout(r, 0)); -describe(processTask.name, () => { +describe(processExtractionTask.name, () => { let processExitSpy: jest.SpyInstance; beforeEach(() => { @@ -95,68 +84,27 @@ describe(processTask.name, () => { .spyOn(process, 'exit') .mockImplementation((() => {}) as () => never); - (createAdapterState as jest.Mock).mockResolvedValue({}); + (createExtractionState as jest.Mock).mockResolvedValue({}); }); afterEach(() => { processExitSpy.mockRestore(); }); - it('should translate incoming event type before passing to task', async () => { - // Arrange - const event = makeEvent(EventType.StartExtractingData); - setWorkerData({ event, initialState: {}, options: {} }); - (translateIncomingEventType as jest.Mock).mockReturnValue( - EventType.StartExtractingMetadata - ); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); - - // Act - processTask({ task, onTimeout }); - await flush(); - - // Assert - expect(translateIncomingEventType).toHaveBeenCalledWith( - EventType.StartExtractingData - ); - // The event is mutated in place — downstream code (including task) sees the - // translated type, not the original wire type. - expect(event.payload.event_type).toBe(EventType.StartExtractingMetadata); - }); - - it('should NOT call onTimeout when the worker already emitted before timeout check', async () => { - // Arrange - const event = makeEvent(); - setWorkerData({ event, initialState: {}, options: {} }); - // Both flags true: a timeout arrived but the worker had already emitted — - // onTimeout must be skipped. This is the guard the integration suite cannot - // target cleanly because it requires a precise race between emit and timeout. - const mockAdapter = { isTimeout: true, hasWorkerEmitted: true }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); - - // Act - processTask({ task, onTimeout }); - await flush(); - - // Assert - expect(onTimeout).not.toHaveBeenCalled(); - expect(processExitSpy).toHaveBeenCalledWith(0); - }); - it('should NOT flip adapter.isTimeout when a non-Exit WorkerMessage arrives', async () => { // Arrange const event = makeEvent(); setWorkerData({ event, initialState: {}, options: {} }); - const mockAdapter = { isTimeout: false, hasWorkerEmitted: false }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const mockAdapter = { + isTimeout: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const task = jest.fn().mockResolvedValue({ status: 'success' }); + const onTimeout = jest.fn().mockResolvedValue({ status: 'progress' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task, onTimeout }); await flush(); // Grab the handler registered for WorkerMessage events and invoke it with @@ -178,14 +126,17 @@ describe(processTask.name, () => { // Arrange const event = makeEvent(); setWorkerData({ event, initialState: {}, options: {} }); - const mockAdapter = { isTimeout: false, hasWorkerEmitted: false }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const mockAdapter = { + isTimeout: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); const taskError = new Error('task boom'); const task = jest.fn().mockRejectedValue(taskError); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const onTimeout = jest.fn().mockResolvedValue({ status: 'progress' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task, onTimeout }); await flush(); // Assert diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 4f3e75a3..f6a594f0 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -1,81 +1,166 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads'; -import { translateIncomingEventType } from '../common/event-type-translation'; -import { Logger, serializeError } from '../logger/logger'; + import { + Logger, runWithSdkLogContext, runWithUserLogContext, -} from '../logger/logger.context'; -import { createAdapterState } from '../state/state'; + serializeError, +} from '../logger/logger'; +import { createExtractionState } from '../state/extraction-state'; +import { createLoadingState } from '../state/loading-state'; import { ProcessTaskInterface, + TaskResult, WorkerEvent, WorkerMessageSubject, } from '../types/workers'; -import { WorkerAdapter } from './worker-adapter/worker-adapter'; -export function processTask({ +import { BaseAdapter } from './adapters/base-adapter'; +import { ExtractionAdapter } from './adapters/extraction-adapter'; +import { LoadingAdapter } from './adapters/loading-adapter'; + +/** + * Shared worker-thread driver. Builds the logger context, runs the task and + * (on timeout) the onTimeout callback against the provided adapter, maps the + * returned {@link TaskResult} to a platform event and emits it exactly once, + * and wires the error/exit plumbing. + * + * The adapter is constructed by the caller so each entry point can build its + * own typed adapter. + * + * If `onTimeout` is omitted, the SDK emits a phase-appropriate default on + * timeout: `progress` (resumable phases) or `error` (non-resumable phases) is + * handled by the status->event mapping when we emit a `progress` result. + * + * On timeout the timeout outcome always wins: once the soft timeout has fired, + * the SDK emits the `onTimeout` result (or the default `progress`) and ignores + * whatever the task returned, because a phase that ran out of time must hand off + * for continuation rather than report itself complete. + */ +async function runWorkerTask>( + buildAdapter: () => Promise, + { task, onTimeout }: ProcessTaskInterface +): Promise { + await runWithSdkLogContext(async () => { + try { + const adapter = await buildAdapter(); + + parentPort?.on(WorkerEvent.WorkerMessage, (message) => { + if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { + return; + } + console.log('Timeout received. Waiting for the task to finish.'); + adapter.isTimeout = true; + }); + + let result: TaskResult = await runWithUserLogContext(async () => + task({ adapter }) + ); + + // If the soft timeout fired, the timeout outcome wins regardless of what + // the task returned: hand off via onTimeout (or the default progress + // result) so the phase continues in a fresh invocation. + if (adapter.isTimeout) { + result = onTimeout + ? await runWithUserLogContext(async () => onTimeout({ adapter })) + : { status: 'progress' }; + } + + await adapter.emitFromResult(result); + + process.exit(0); + } catch (error) { + runWithUserLogContext(() => { + const errorMessage = `Error while processing task. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + }); + } + }); +} + +/** + * Entry point for an extraction worker. Builds an {@link ExtractionAdapter} and + * runs the provided task against it. + * + * @public + */ +export function processExtractionTask({ task, onTimeout, -}: ProcessTaskInterface) { +}: ProcessTaskInterface>) { if (isMainThread) { return; } - void (async () => { - await runWithSdkLogContext(async () => { - try { - const event = workerData.event; - - // TODO: Remove when the old types are completely phased out - event.payload.event_type = translateIncomingEventType( - event.payload.event_type - ); - - const initialState = workerData.initialState as ConnectorState; - const initialDomainMapping = workerData.initialDomainMapping; - const options = workerData.options; - // eslint-disable-next-line no-global-assign - console = new Logger({ event, options }); - - const adapterState = await createAdapterState({ - event, - initialState, - initialDomainMapping, - options, - }); + void runWorkerTask>( + async () => { + const event = workerData.event; + const initialState = workerData.initialState as ConnectorState; + const initialDomainMapping = workerData.initialDomainMapping; + const options = workerData.options; + // eslint-disable-next-line no-global-assign + console = new Logger({ event, options }); - const adapter = new WorkerAdapter({ - event, - adapterState, - options, - }); + const adapterState = await createExtractionState({ + event, + initialState, + initialDomainMapping, + options, + }); - parentPort?.on(WorkerEvent.WorkerMessage, (message) => { - if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { - return; - } - console.log('Timeout received. Waiting for the task to finish.'); - adapter.isTimeout = true; - }); + return new ExtractionAdapter({ + event, + adapterState, + options, + }); + }, + { task, onTimeout } + ); +} - await runWithUserLogContext(async () => task({ adapter })); - if (adapter.isTimeout && !adapter.hasWorkerEmitted) { - await runWithUserLogContext(async () => onTimeout({ adapter })); - } - process.exit(0); - } catch (error) { - runWithUserLogContext(() => { - const errorMessage = `Error while processing task. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - }); - } - }); - })(); +/** + * Entry point for a loading worker. Builds a {@link LoadingAdapter} and runs the + * provided task against it. + * + * @public + */ +export function processLoadingTask({ + task, + onTimeout, +}: ProcessTaskInterface>) { + if (isMainThread) { + return; + } + + void runWorkerTask>( + async () => { + const event = workerData.event; + const initialState = workerData.initialState as ConnectorState; + const initialDomainMapping = workerData.initialDomainMapping; + const options = workerData.options; + // eslint-disable-next-line no-global-assign + console = new Logger({ event, options }); + + const adapterState = await createLoadingState({ + event, + initialState, + initialDomainMapping, + options, + }); + + return new LoadingAdapter({ + event, + adapterState, + options, + }); + }, + { task, onTimeout } + ); } diff --git a/src/multithreading/spawn/spawn.helpers.test.ts b/src/multithreading/spawn/spawn.helpers.test.ts index 6645ac80..668fdb10 100644 --- a/src/multithreading/spawn/spawn.helpers.test.ts +++ b/src/multithreading/spawn/spawn.helpers.test.ts @@ -1,3 +1,4 @@ +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; @@ -19,9 +20,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.MetadataExtractionError); }); - it('should return MetadataExtractionError for deprecated ExtractionMetadataStart', () => { + it('should return MetadataExtractionError for StartExtractingMetadata (renamed from ExtractionMetadataStart)', () => { // Arrange - const eventType = EventType.ExtractionMetadataStart; + const eventType = EventType.StartExtractingMetadata; // Act const result = getTimeoutErrorEventType(eventType); @@ -54,9 +55,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.DataExtractionError); }); - it('should return DataExtractionError for deprecated ExtractionDataStart', () => { + it('should return DataExtractionError for StartExtractingData (renamed from ExtractionDataStart)', () => { // Arrange - const eventType = EventType.ExtractionDataStart; + const eventType = EventType.StartExtractingData; // Act const result = getTimeoutErrorEventType(eventType); @@ -65,9 +66,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.DataExtractionError); }); - it('should return DataExtractionError for deprecated ExtractionDataContinue', () => { + it('should return DataExtractionError for ContinueExtractingData (renamed from ExtractionDataContinue)', () => { // Arrange - const eventType = EventType.ExtractionDataContinue; + const eventType = EventType.ContinueExtractingData; // Act const result = getTimeoutErrorEventType(eventType); @@ -91,9 +92,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExtractorStateDeletionError for deprecated ExtractionDataDelete', () => { + it('should return ExtractorStateDeletionError for StartDeletingExtractorState (renamed from ExtractionDataDelete)', () => { // Arrange - const eventType = EventType.ExtractionDataDelete; + const eventType = EventType.StartDeletingExtractorState; // Act const result = getTimeoutErrorEventType(eventType); @@ -132,9 +133,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsStart', () => { + it('should return AttachmentExtractionError for StartExtractingAttachments (renamed from ExtractionAttachmentsStart)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsStart; + const eventType = EventType.StartExtractingAttachments; // Act const result = getTimeoutErrorEventType(eventType); @@ -145,9 +146,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsContinue', () => { + it('should return AttachmentExtractionError for ContinueExtractingAttachments (renamed from ExtractionAttachmentsContinue)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsContinue; + const eventType = EventType.ContinueExtractingAttachments; // Act const result = getTimeoutErrorEventType(eventType); @@ -173,9 +174,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExtractorAttachmentsStateDeletionError for deprecated ExtractionAttachmentsDelete', () => { + it('should return ExtractorAttachmentsStateDeletionError for StartDeletingExtractorAttachmentsState (renamed from ExtractionAttachmentsDelete)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsDelete; + const eventType = EventType.StartDeletingExtractorAttachmentsState; // Act const result = getTimeoutErrorEventType(eventType); @@ -201,9 +202,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExternalSyncUnitExtractionError for deprecated ExtractionExternalSyncUnitsStart', () => { + it('should return ExternalSyncUnitExtractionError for StartExtractingExternalSyncUnits (renamed from ExtractionExternalSyncUnitsStart)', () => { // Arrange - const eventType = EventType.ExtractionExternalSyncUnitsStart; + const eventType = EventType.StartExtractingExternalSyncUnits; // Act const result = getTimeoutErrorEventType(eventType); @@ -292,9 +293,9 @@ describe(getTimeoutErrorEventType.name, () => { }); describe('unknown event types', () => { - it('[edge] should return UnknownEventType and log error for unrecognized event type', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE and log error for unrecognized event type', () => { // Arrange - const eventType = EventType.UnknownEventType; + const eventType = 'TOTALLY_UNKNOWN' as EventType; const consoleErrorSpy = jest .spyOn(console, 'error') .mockImplementation(() => {}); @@ -303,7 +304,7 @@ describe(getTimeoutErrorEventType.name, () => { const result = getTimeoutErrorEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Event type not recognized in getTimeoutErrorEventType function: ' + eventType @@ -375,7 +376,7 @@ describe(getNoScriptEventType.name, () => { }); describe('unknown event types', () => { - it('[edge] should return UnknownEventType and log error for unrecognized event type', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE and log error for unrecognized event type', () => { // Arrange const eventType = EventType.StartExtractingData; const consoleErrorSpy = jest @@ -386,7 +387,7 @@ describe(getNoScriptEventType.name, () => { const result = getNoScriptEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Event type not recognized in getNoScriptEventType function: ' + eventType @@ -396,7 +397,7 @@ describe(getNoScriptEventType.name, () => { consoleErrorSpy.mockRestore(); }); - it('[edge] should return UnknownEventType for StartLoadingData', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE for StartLoadingData', () => { // Arrange const eventType = EventType.StartLoadingData; const consoleErrorSpy = jest @@ -407,7 +408,7 @@ describe(getNoScriptEventType.name, () => { const result = getNoScriptEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalled(); // Cleanup diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index abbe9908..e0db6c89 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -1,5 +1,179 @@ +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; +import { TaskStatus } from '../../types/workers'; + +/** + * Resolves the outgoing event type the SDK should emit for a given incoming + * event type and the {@link TaskResult} status a worker returned. + * + * The mapping follows the per-phase contract documented on `TaskResult`: + * resumable phases (data/attachment extraction, data/attachment loading) honor + * every status (success -> *_DONE, progress -> *_PROGRESS, delay -> *_DELAYED, + * error -> *_ERROR); non-resumable phases (external sync units, metadata, state + * deletion) only have a done and an error event, so a `progress`/`delay` status + * there is illegal and is mapped to the phase's error event. + * + * @param eventType - The incoming event type that started this worker. + * @param status - The status the worker's task/onTimeout returned. + * @returns The outgoing extractor/loader event type to emit, plus whether the + * status was illegal for the phase (so the caller can attach a descriptive + * error message). + */ +export function getEventTypeForResult( + eventType: EventType, + status: TaskStatus +): { + eventType: ExtractorEventType | LoaderEventType; + illegal: boolean; +} { + const phase = EVENT_PHASE_MAP[eventType]; + + if (!phase) { + console.error( + 'Event type not recognized in getEventTypeForResult function: ' + + eventType + ); + return { + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, + illegal: true, + }; + } + + // Non-resumable phases only define done/error events. + if (!phase.resumable) { + if (status === 'success') { + return { eventType: phase.done, illegal: false }; + } + // progress/delay are illegal here; collapse them (and error) to the error event. + return { eventType: phase.error, illegal: status !== 'error' }; + } + + switch (status) { + case 'success': + return { eventType: phase.done, illegal: false }; + case 'progress': + return { eventType: phase.progress!, illegal: false }; + case 'delay': + return { eventType: phase.delayed!, illegal: false }; + case 'error': + return { eventType: phase.error, illegal: false }; + } +} + +/** + * Per-phase outgoing event types, keyed by the incoming {@link EventType}. + * `resumable` phases define progress/delayed events; non-resumable ones do not. + */ +const EVENT_PHASE_MAP: Partial< + Record< + EventType, + { + resumable: boolean; + done: ExtractorEventType | LoaderEventType; + error: ExtractorEventType | LoaderEventType; + progress?: ExtractorEventType | LoaderEventType; + delayed?: ExtractorEventType | LoaderEventType; + } + > +> = { + // External sync units (non-resumable) + [EventType.StartExtractingExternalSyncUnits]: { + resumable: false, + done: ExtractorEventType.ExternalSyncUnitExtractionDone, + error: ExtractorEventType.ExternalSyncUnitExtractionError, + }, + // Metadata (non-resumable) + [EventType.StartExtractingMetadata]: { + resumable: false, + done: ExtractorEventType.MetadataExtractionDone, + error: ExtractorEventType.MetadataExtractionError, + }, + // Data extraction (resumable) + [EventType.StartExtractingData]: { + resumable: true, + done: ExtractorEventType.DataExtractionDone, + error: ExtractorEventType.DataExtractionError, + progress: ExtractorEventType.DataExtractionProgress, + delayed: ExtractorEventType.DataExtractionDelayed, + }, + [EventType.ContinueExtractingData]: { + resumable: true, + done: ExtractorEventType.DataExtractionDone, + error: ExtractorEventType.DataExtractionError, + progress: ExtractorEventType.DataExtractionProgress, + delayed: ExtractorEventType.DataExtractionDelayed, + }, + // Extractor state deletion (non-resumable) + [EventType.StartDeletingExtractorState]: { + resumable: false, + done: ExtractorEventType.ExtractorStateDeletionDone, + error: ExtractorEventType.ExtractorStateDeletionError, + }, + // Attachment extraction (resumable) + [EventType.StartExtractingAttachments]: { + resumable: true, + done: ExtractorEventType.AttachmentExtractionDone, + error: ExtractorEventType.AttachmentExtractionError, + progress: ExtractorEventType.AttachmentExtractionProgress, + delayed: ExtractorEventType.AttachmentExtractionDelayed, + }, + [EventType.ContinueExtractingAttachments]: { + resumable: true, + done: ExtractorEventType.AttachmentExtractionDone, + error: ExtractorEventType.AttachmentExtractionError, + progress: ExtractorEventType.AttachmentExtractionProgress, + delayed: ExtractorEventType.AttachmentExtractionDelayed, + }, + // Extractor attachments state deletion (non-resumable) + [EventType.StartDeletingExtractorAttachmentsState]: { + resumable: false, + done: ExtractorEventType.ExtractorAttachmentsStateDeletionDone, + error: ExtractorEventType.ExtractorAttachmentsStateDeletionError, + }, + // Data loading (resumable) + [EventType.StartLoadingData]: { + resumable: true, + done: LoaderEventType.DataLoadingDone, + error: LoaderEventType.DataLoadingError, + progress: LoaderEventType.DataLoadingProgress, + delayed: LoaderEventType.DataLoadingDelayed, + }, + [EventType.ContinueLoadingData]: { + resumable: true, + done: LoaderEventType.DataLoadingDone, + error: LoaderEventType.DataLoadingError, + progress: LoaderEventType.DataLoadingProgress, + delayed: LoaderEventType.DataLoadingDelayed, + }, + // Attachment loading (resumable) + [EventType.StartLoadingAttachments]: { + resumable: true, + done: LoaderEventType.AttachmentLoadingDone, + error: LoaderEventType.AttachmentLoadingError, + progress: LoaderEventType.AttachmentLoadingProgress, + delayed: LoaderEventType.AttachmentLoadingDelayed, + }, + [EventType.ContinueLoadingAttachments]: { + resumable: true, + done: LoaderEventType.AttachmentLoadingDone, + error: LoaderEventType.AttachmentLoadingError, + progress: LoaderEventType.AttachmentLoadingProgress, + delayed: LoaderEventType.AttachmentLoadingDelayed, + }, + // Loader state deletion (non-resumable) + [EventType.StartDeletingLoaderState]: { + resumable: false, + done: LoaderEventType.LoaderStateDeletionDone, + error: LoaderEventType.LoaderStateDeletionError, + }, + // Loader attachment state deletion (non-resumable) + [EventType.StartDeletingLoaderAttachmentState]: { + resumable: false, + done: LoaderEventType.LoaderAttachmentStateDeletionDone, + error: LoaderEventType.LoaderAttachmentStateDeletionError, + }, +}; /** * Gets the event type for the timeout error. @@ -10,48 +184,40 @@ export function getTimeoutErrorEventType(eventType: EventType): { eventType: ExtractorEventType | LoaderEventType; } { switch (eventType) { - // Metadata extraction (handles both old and new enum members) + // Metadata extraction case EventType.StartExtractingMetadata: - case EventType.ExtractionMetadataStart: return { eventType: ExtractorEventType.MetadataExtractionError, }; - // Data extraction (handles both old and new enum members) + // Data extraction case EventType.StartExtractingData: case EventType.ContinueExtractingData: - case EventType.ExtractionDataStart: - case EventType.ExtractionDataContinue: return { eventType: ExtractorEventType.DataExtractionError, }; - // Data deletion (handles both old and new enum members) + // Data deletion case EventType.StartDeletingExtractorState: - case EventType.ExtractionDataDelete: return { eventType: ExtractorEventType.ExtractorStateDeletionError, }; - // Attachments extraction (handles both old and new enum members) + // Attachments extraction case EventType.StartExtractingAttachments: case EventType.ContinueExtractingAttachments: - case EventType.ExtractionAttachmentsStart: - case EventType.ExtractionAttachmentsContinue: return { eventType: ExtractorEventType.AttachmentExtractionError, }; - // Attachments deletion (handles both old and new enum members) + // Attachments deletion case EventType.StartDeletingExtractorAttachmentsState: - case EventType.ExtractionAttachmentsDelete: return { eventType: ExtractorEventType.ExtractorAttachmentsStateDeletionError, }; - // External sync units (handles both old and new enum members) + // External sync units case EventType.StartExtractingExternalSyncUnits: - case EventType.ExtractionExternalSyncUnitsStart: return { eventType: ExtractorEventType.ExternalSyncUnitExtractionError, }; @@ -88,7 +254,7 @@ export function getTimeoutErrorEventType(eventType: EventType): { eventType ); return { - eventType: LoaderEventType.UnknownEventType, + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, }; } } @@ -122,7 +288,7 @@ export function getNoScriptEventType(eventType: EventType) { eventType ); return { - eventType: LoaderEventType.UnknownEventType, + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, }; } } diff --git a/src/multithreading/spawn/spawn.test.ts b/src/multithreading/spawn/spawn.test.ts index 9445c20f..543135cb 100644 --- a/src/multithreading/spawn/spawn.test.ts +++ b/src/multithreading/spawn/spawn.test.ts @@ -2,7 +2,7 @@ import { EventEmitter } from 'events'; import { DEFAULT_LAMBDA_TIMEOUT } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { WorkerEvent, WorkerMessageSubject } from '../../types/workers'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; // --------------------------------------------------------------------------- // Mocks @@ -12,7 +12,7 @@ jest.mock('../create-worker', () => ({ createWorker: jest.fn(), })); -jest.mock('../../common/control-protocol', () => ({ +jest.mock('../emit', () => ({ emit: jest.fn().mockResolvedValue({}), })); @@ -38,8 +38,6 @@ jest.mock('../../common/helpers', () => ({ arrayBuffersMB: '5.00', }), sleep: jest.fn(), - truncateFilename: jest.fn((f: string) => f), - truncateMessage: jest.fn((m: string) => m), })); // --------------------------------------------------------------------------- @@ -47,7 +45,7 @@ jest.mock('../../common/helpers', () => ({ // --------------------------------------------------------------------------- import { spawn, Spawn } from './spawn'; import { createWorker } from '../create-worker'; -import { emit } from '../../common/control-protocol'; +import { emit } from '../emit'; import { getMemoryUsage } from '../../common/helpers'; // --------------------------------------------------------------------------- @@ -113,7 +111,7 @@ describe('spawn() factory', () => { it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => { // Arrange const event = createMockEvent('http://localhost:0', { - payload: { event_type: EventType.UnknownEventType }, + payload: { event_type: 'TOTALLY_UNKNOWN' as EventType }, }); // Act @@ -137,7 +135,16 @@ describe('spawn() factory', () => { // Act & Assert await expect( - spawn({ event, initialState: {}, workerPath: '/fake/path.js' }) + spawn({ + event, + initialState: {}, + baseWorkerPath: '', + options: { + workerPathOverrides: { + [event.payload.event_type]: '/fake/path.js', + }, + }, + }) ).rejects.toThrow('worker boom'); }); }); diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 0350d34d..9473df01 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -1,11 +1,10 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import { emit } from '../../common/control-protocol'; -import { translateIncomingEventType } from '../../common/event-type-translation'; +import { emit } from '../emit'; import { getMemoryUsage } from '../../common/helpers'; import { Logger, serializeError } from '../../logger/logger'; -import { AirdropEvent, EventType } from '../../types/extraction'; +import { AirSyncEvent, EventType } from '../../types/extraction'; import { GetWorkerPathInterface, SpawnFactoryInterface, @@ -65,28 +64,18 @@ function getWorkerPath({ * The class provides utilities to emit control events to the platform and exit the worker gracefully. * In case of lambda timeout, the class emits a lambda timeout event to the platform. * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class - * @param {AirdropEvent} options.event - The event object received from the platform + * @param {AirSyncEvent} options.event - The event object received from the platform * @param {object} options.initialState - The initial state of the adapter - * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname` * @returns {Promise} - A new instance of Spawn class */ export async function spawn({ event, initialState, - workerPath, initialDomainMapping, options, baseWorkerPath, }: SpawnFactoryInterface): Promise { - // Translate incoming event type for backwards compatibility. This allows the - // SDK to accept both old and new event type formats. Then update the event with the translated event type. - const originalEventType = event.payload.event_type; - const translatedEventType = translateIncomingEventType( - event.payload.event_type as string - ); - event.payload.event_type = translatedEventType; - // Read the command line arguments to check if the local flag is passed. const argv = await yargs(hideBin(process.argv)).argv; if (argv._.includes('local') || argv.local) { @@ -100,26 +89,19 @@ export async function spawn({ // eslint-disable-next-line no-global-assign console = new Logger({ event, options }); - if (translatedEventType !== originalEventType) { - console.log( - `Event type translated from ${originalEventType} to ${translatedEventType}.` - ); - } if (options?.isLocalDevelopment) { console.log('Snap-in is running in local development mode.'); } let script = null; - if (workerPath != null) { - script = workerPath; - } else if ( + if ( baseWorkerPath != null && options?.workerPathOverrides != null && - options.workerPathOverrides[translatedEventType as EventType] != null + options.workerPathOverrides[event.payload.event_type as EventType] != null ) { script = baseWorkerPath + - options.workerPathOverrides[translatedEventType as EventType]; + options.workerPathOverrides[event.payload.event_type as EventType]; } else { script = getWorkerPath({ event, @@ -169,7 +151,7 @@ export async function spawn({ } export class Spawn { - private event: AirdropEvent; + private event: AirSyncEvent; private alreadyEmitted: boolean; private softTimeoutSent: boolean; private defaultLambdaTimeout: number = DEFAULT_LAMBDA_TIMEOUT; @@ -269,7 +251,7 @@ export class Spawn { // If worker sends a message that it has emitted an event, then set alreadyEmitted to true. if (message?.subject === WorkerMessageSubject.WorkerMessageEmitted) { - console.info('Worker has emitted message to ADaaS.'); + console.info('Worker has emitted message to AirSync.'); this.alreadyEmitted = true; } diff --git a/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts b/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts deleted file mode 100644 index f3bef85e..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { AirSyncDefaultItemTypes } from '../../common/constants'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createItems } from '../../tests/test-helpers'; -import { createMockEvent } from '../../common/test-utils'; -import { Artifact, EventType } from '../../types'; -import { ExternalSyncUnit } from '../../types/extraction'; -import { WorkerAdapter } from './worker-adapter'; - -// 1. Create a mock function for the method you want to override. -const mockUpload = (itemType: string, objects: object[]) => { - return { - error: null, - artifact: { - id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`, - item_type: itemType, - item_count: objects.length, - }, - }; -}; - -// 2. Mock the entire 'uploader' module. -// The factory function () => { ... } returns the mock implementation. -jest.mock('../../uploader/uploader', () => { - return { - // The mocked Uploader class - Uploader: jest.fn().mockImplementation(() => { - // The constructor of the mocked Uploader returns an object - // with the methods you want to control. - return { - upload: mockUpload, - }; - }), - }; -}); - -function checkArtifactOrder( - artifacts: Artifact[], - expectedOrder: { itemType: string }[] -): boolean { - let outerIndex = 0; - for (const artifact of artifacts) { - try { - // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown - while (artifact.item_type != expectedOrder[outerIndex].itemType) { - outerIndex++; - } - } catch (e) { - console.error('Error finding artifact type in repos:', e); - return false; - } - } - return true; -} - -describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => { - interface TestState { - attachments: { completed: boolean }; - } - let testAdapter: WorkerAdapter; - - beforeEach(() => { - // Create a fresh adapter instance for this test to avoid mocking conflicts - const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.StartExtractingData }, - }); - const mockAdapterState = new State({ - event: mockEvent, - initialState: { attachments: { completed: false } }, - }); - - testAdapter = new WorkerAdapter({ - event: mockEvent, - adapterState: mockAdapterState, - options: { - batchSize: 50, - }, - }); - }); - - it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => { - const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(5)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(105)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(4); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should work with more than 2 repos', async () => { - const repos = [ - { itemType: 'ItemTypeA' }, - { itemType: 'ItemTypeB' }, - { itemType: 'ItemTypeC' }, - { itemType: 'ItemTypeD' }, - ]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeC')?.push(createItems(103)); - await testAdapter.getRepo('ItemTypeD')?.push(createItems(104)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(12); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should maintain order with multiple pushes and uploads', async () => { - const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.upload(); - await testAdapter.getRepo('ItemTypeB')?.upload(); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(20); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should not count artifacts if 0 items are pushed to the repo', async () => { - const repos = [{ itemType: 'ItemTypeA' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push([]); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(0); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); -}); - -describe('External sync units splitting into artifacts', () => { - let testAdapter: WorkerAdapter>; - - beforeEach(() => { - const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.StartExtractingExternalSyncUnits }, - }); - const mockAdapterState = new State>({ - event: mockEvent, - initialState: {}, - }); - - testAdapter = new WorkerAdapter({ - event: mockEvent, - adapterState: mockAdapterState, - }); - }); - - it('should split 125k external sync units into 5 artifacts', async () => { - const BATCH_SIZE = 25_000; - const TOTAL_UNITS = 125_000; - - const externalSyncUnits: ExternalSyncUnit[] = Array.from( - { length: TOTAL_UNITS }, - (_, i) => ({ - id: String(i), - name: `Unit ${i}`, - description: `Description ${i}`, - }) - ); - - testAdapter.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: BATCH_SIZE, - skipConfirmation: true, - }, - }, - ]); - - const repo = testAdapter.getRepo( - AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS - ); - const chunkSize = 10_000; - for (let i = 0; i < TOTAL_UNITS; i += chunkSize) { - await repo?.push(externalSyncUnits.slice(i, i + chunkSize)); - } - - await testAdapter.uploadAllRepos(); - - expect(testAdapter.artifacts.length).toBe( - TOTAL_UNITS / BATCH_SIZE // 5 - ); - expect( - testAdapter.artifacts.every( - (a) => a.item_type === AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS - ) - ).toBe(true); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts b/src/multithreading/worker-adapter/worker-adapter.emit.test.ts deleted file mode 100644 index 8afef41c..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts +++ /dev/null @@ -1,593 +0,0 @@ -import { UNBOUNDED_DATE_TIME_VALUE } from '../../common/constants'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - Artifact, - EventType, - ExtractorEventType, - LoaderEventType, -} from '../../types'; -import { ActionType, LoaderReport } from '../../types/loading'; -import { WorkerAdapter } from './worker-adapter'; - -/* eslint-disable @typescript-eslint/no-require-imports */ - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType = EventType.StartExtractingData): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -describe(`${WorkerAdapter.name}.emit`, () => { - let adapter: WorkerAdapter; - let mockPostMessage: jest.Mock; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - - const workerThreads = require('node:worker_threads'); - mockPostMessage = jest.fn(); - if (workerThreads.parentPort) { - jest - .spyOn(workerThreads.parentPort, 'postMessage') - .mockImplementation(mockPostMessage); - } else { - workerThreads.parentPort = { postMessage: mockPostMessage }; - } - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should emit only one event when multiple events of same type are sent', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should emit only once even when a different event type follows', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.DataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.AttachmentExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should correctly emit one event even if postState errors', async () => { - // Arrange - adapter['adapterState'].postState = jest - .fn() - .mockRejectedValue(new Error('postState error')); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should correctly emit one event even if uploadAllRepos errors', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest - .fn() - .mockRejectedValue(new Error('uploadAllRepos error')); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should include artifacts in data for extraction events', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['_artifacts'] = [ - { id: 'art-1', item_count: 10, item_type: 'issues' }, - ] as Artifact[]; - - // Act - await adapter.emit(ExtractorEventType.DataExtractionDone); - - // Assert - expect(mockEmit).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - artifacts: expect.arrayContaining([ - expect.objectContaining({ id: 'art-1' }), - ]), - }), - }) - ); - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('reports'); - expect(callData).not.toHaveProperty('processed_files'); - }); - - it('should include reports and processed_files in data for loader events', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['loaderReports'] = [ - { item_type: 'tasks', [ActionType.CREATED]: 5 }, - ] as LoaderReport[]; - adapter['_processedFiles'] = ['file-1', 'file-2']; - - // Act - await adapter.emit(LoaderEventType.DataLoadingDone); - - // Assert - expect(mockEmit).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - reports: expect.arrayContaining([ - expect.objectContaining({ item_type: 'tasks' }), - ]), - processed_files: ['file-1', 'file-2'], - }), - }) - ); - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('artifacts'); - }); - - it('should not include artifacts, reports, or processed_files for unknown event types', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['_artifacts'] = [ - { id: 'art-1', item_count: 10, item_type: 'issues' }, - ] as Artifact[]; - adapter['loaderReports'] = [ - { item_type: 'tasks', [ActionType.CREATED]: 5 }, - ] as LoaderReport[]; - adapter['_processedFiles'] = ['file-1']; - - // Act - await adapter.emit('SOME_UNKNOWN_EVENT' as ExtractorEventType); - - // Assert - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('artifacts'); - expect(callData).not.toHaveProperty('reports'); - expect(callData).not.toHaveProperty('processed_files'); - }); - - it('should include artifacts for all ExtractorEventType values', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - const extractorEvents = [ - ExtractorEventType.DataExtractionDone, - ExtractorEventType.DataExtractionProgress, - ExtractorEventType.DataExtractionError, - ExtractorEventType.AttachmentExtractionDone, - ExtractorEventType.AttachmentExtractionProgress, - ]; - - for (const eventType of extractorEvents) { - jest.clearAllMocks(); - adapter.hasWorkerEmitted = false; - adapter['adapterState'].postState = jest - .fn() - .mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(eventType); - - // Assert - const callData = mockEmit.mock.calls[0]?.[0]?.data; - expect(callData).toHaveProperty('artifacts'); - expect(callData).not.toHaveProperty('reports'); - } - }); - - it('should include reports and processed_files for all LoaderEventType values', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - const loaderEvents = [ - LoaderEventType.DataLoadingDone, - LoaderEventType.DataLoadingProgress, - LoaderEventType.DataLoadingError, - LoaderEventType.AttachmentLoadingDone, - LoaderEventType.AttachmentLoadingProgress, - ]; - - for (const eventType of loaderEvents) { - jest.clearAllMocks(); - adapter.hasWorkerEmitted = false; - adapter['adapterState'].postState = jest - .fn() - .mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(eventType); - - // Assert - const callData = mockEmit.mock.calls[0]?.[0]?.data; - expect(callData).toHaveProperty('reports'); - expect(callData).toHaveProperty('processed_files'); - expect(callData).not.toHaveProperty('artifacts'); - } - }); - - it('should truncate a long error message, preserving the original prefix', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - const longMessage = 'E'.repeat(20_000); - - // Act - await adapter.emit(ExtractorEventType.DataExtractionError, { - error: { message: longMessage }, - }); - - // Assert - const { emit: mockEmit } = require('../../common/control-protocol'); - const emittedMessage = mockEmit.mock.calls[0][0].data?.error - ?.message as string; - expect(emittedMessage.length).toBeLessThan(longMessage.length); - expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true); - }); -}); - -describe(`${WorkerAdapter.name}.emit — ExternalSyncUnitExtractionDone legacy path`, () => { - it('should upload ESUs via a repo and strip external_sync_units from the emitted payload', async () => { - // Arrange - const { adapter } = makeAdapter(EventType.StartExtractingExternalSyncUnits); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - const pushMock = jest.fn().mockResolvedValue(undefined); - jest.spyOn(adapter, 'initializeRepos'); - jest.spyOn(adapter, 'getRepo').mockReturnValue({ push: pushMock } as never); - const esus = [{ id: 'esu-1' }, { id: 'esu-2' }] as never; - - // Act - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: esus, - }); - - // Assert - expect(pushMock).toHaveBeenCalledWith(esus); - // external_sync_units must NOT appear in the payload sent to the platform - // (it would be too large for SQS — that is the entire reason this path exists). - const { emit: mockEmit } = require('../../common/control-protocol'); - const emittedData = mockEmit.mock.calls[0][0].data as Record< - string, - unknown - >; - expect(emittedData).not.toHaveProperty('external_sync_units'); - }); -}); - -describe('WorkerAdapter — workersOldest / workersNewest boundary updates', () => { - let adapter: WorkerAdapter; - let mockPostMessage: jest.Mock; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - - const workerThreads = require('node:worker_threads'); - mockPostMessage = jest.fn(); - if (workerThreads.parentPort) { - jest - .spyOn(workerThreads.parentPort, 'postMessage') - .mockImplementation(mockPostMessage); - } else { - workerThreads.parentPort = { postMessage: mockPostMessage }; - } - - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - async function emitDone( - adapterInstance: WorkerAdapter, - extractionStart: string | undefined, - extractionEnd: string | undefined - ) { - adapterInstance.event.payload.event_context.extract_from = extractionStart; - adapterInstance.event.payload.event_context.extract_to = extractionEnd; - // Reset the emit guard so we can emit multiple times within one test. - adapterInstance['hasWorkerEmitted'] = false; - - await adapterInstance.emit(ExtractorEventType.AttachmentExtractionDone, { - reports: [], - processed_files: [], - }); - } - - describe('initial import with UNBOUNDED start', () => { - it('should set workersOldest to UNBOUNDED_DATE_TIME_VALUE and workersNewest to extraction end', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation after UNBOUNDED initial import', () => { - it('should NOT overwrite workersOldest when reconciliation start is later than sentinel', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - - it('should NOT overwrite workersOldest even when reconciliation start is very early', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '1980-01-01T00:00:00.000Z', - '1990-01-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - }); - - describe('forward sync after UNBOUNDED initial import', () => { - it('should expand workersNewest forward while preserving workersOldest', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-06-01T00:00:00.000Z', - '2025-07-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-07-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation with end beyond current newest', () => { - it('should expand workersNewest when reconciliation end is later', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-01-01T00:00:00.000Z', - '2025-08-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-08-01T00:00:00.000Z'); - }); - }); - - describe('first sync with absolute dates (no UNBOUNDED)', () => { - it('should set both boundaries from the extraction range', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation after absolute initial sync', () => { - it('should expand workersOldest backward when reconciliation start is earlier', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-06-01T00:00:00.000Z', - '2025-02-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2024-06-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - - it('should NOT change boundaries when reconciliation is within existing range', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-01-15T00:00:00.000Z', - '2025-02-15T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - - it('should expand both boundaries when reconciliation exceeds both', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-06-01T00:00:00.000Z', - '2025-09-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2024-06-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-09-01T00:00:00.000Z'); - }); - }); - - describe('multiple forward syncs', () => { - it('should progressively expand workersNewest while preserving workersOldest', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-06-01T00:00:00.000Z', - '2025-07-01T00:00:00.000Z' - ); - expect(adapter.state.workersNewest).toBe('2025-07-01T00:00:00.000Z'); - - await emitDone( - adapter, - '2025-07-01T00:00:00.000Z', - '2025-08-01T00:00:00.000Z' - ); - expect(adapter.state.workersNewest).toBe('2025-08-01T00:00:00.000Z'); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - }); - }); - - describe('non-AttachmentExtractionDone events should NOT update boundaries', () => { - it.each([ - ExtractorEventType.DataExtractionDone, - ExtractorEventType.DataExtractionProgress, - ExtractorEventType.MetadataExtractionError, - ExtractorEventType.AttachmentExtractionError, - ])('should not update boundaries on %s', async (eventType) => { - adapter.state.workersOldest = '2025-01-01T00:00:00.000Z'; - adapter.state.workersNewest = '2025-03-01T00:00:00.000Z'; - adapter.event.payload.event_context.extract_from = - '2024-01-01T00:00:00.000Z'; - adapter.event.payload.event_context.extract_to = - '2025-12-01T00:00:00.000Z'; - - await adapter.emit(eventType, { - reports: [], - processed_files: [], - }); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts b/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts deleted file mode 100644 index 55b38e88..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts +++ /dev/null @@ -1,1034 +0,0 @@ -import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - Artifact, - EventType, - ExtractorEventType, -} from '../../types'; -import { WorkerAdapter } from './worker-adapter'; - -/* eslint-disable @typescript-eslint/no-require-imports */ - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType = EventType.StartExtractingData): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -describe(`${WorkerAdapter.name}.streamAttachments`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - }); - - it('should process all artifact batches successfully', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - { - url: 'http://example.com/file2.pdf', - id: 'attachment2', - file_name: 'file2.pdf', - parent_id: 'parent2', - }, - ], - }) - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file3.pdf', - id: 'attachment3', - file_name: 'file3.pdf', - parent_id: 'parent3', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.initializeRepos).toHaveBeenCalledWith([ - { itemType: 'ssor_attachment' }, - ]); - expect(adapter.initializeRepos).toHaveBeenCalledTimes(1); - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(2); - - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]); - expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0); - expect(result).toBeUndefined(); - }); - - it('[edge] should handle invalid batch size by using 1 instead', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - batchSize: 0, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - batchSize: 100, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should handle empty attachments metadata artifact IDs', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - }, - }; - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should handle errors when getting attachments', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - const mockError = new Error('Failed to get attachments'); - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - error: mockError, - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ - error: mockError, - }); - }); - - it('[edge] should handle empty attachments array from artifact', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]); - expect(result).toBeUndefined(); - }); - - it('should use custom processors when provided', async () => { - // Arrange - const mockStream = jest.fn(); - const mockReducer = jest.fn().mockReturnValue(['custom-reduced']); - const mockIterator = jest.fn().mockResolvedValue({}); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - processors: { - reducer: mockReducer, - iterator: mockIterator, - }, - }); - - // Assert - expect(mockReducer).toHaveBeenCalledWith({ - attachments: [{ id: 'attachment1' }], - adapter: adapter, - batchSize: 1, - }); - expect(mockIterator).toHaveBeenCalledWith({ - reducedAttachments: ['custom-reduced'], - adapter: adapter, - stream: mockStream, - }); - expect(result).toBeUndefined(); - }); - - it('should handle rate limiting from iterator', async () => { - // Arrange - const mockStream = jest.fn(); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({ delay: 30 }), - })); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ delay: 30 }); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - ]); - }); - - it('should handle error from iterator', async () => { - // Arrange - const mockStream = jest.fn(); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({ - error: 'Mock error', - }), - })); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ error: 'Mock error' }); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - ]); - }); - - it('should emit progress event and exit process on timeout, preserving state for resumption', async () => { - // Arrange - const mockStream = jest.fn(); - - const exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2', 'artifact3'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockImplementation(() => { - adapter.isTimeout = true; - return {}; - }), - })); - - adapter.initializeRepos = jest.fn(); - - const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - - // Act - await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - ExtractorEventType.AttachmentExtractionProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - 'artifact2', - 'artifact3', - ]); - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(1); - - exitSpy.mockRestore(); - }); - - it('should stop after the timeout flips between batches and preserve unprocessed artifacts for resumption', async () => { - // Arrange: three artifacts. The first batch's streamAll completes - // successfully; the second sets isTimeout=true mid-run. The third batch - // must never be reached. - const mockStream = jest.fn(); - const exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2', 'artifact3'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file.pdf', - id: 'attachment-x', - file_name: 'file.pdf', - parent_id: 'parent-x', - }, - ], - }); - - // First call: clean streamAll. Second call: flip isTimeout AFTER streaming. - (AttachmentsStreamingPool as jest.Mock) - .mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({}), - })) - .mockImplementationOnce(() => ({ - streamAll: jest.fn().mockImplementation(() => { - adapter.isTimeout = true; - return {}; - }), - })); - - adapter.initializeRepos = jest.fn(); - const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - - // Act - await adapter.streamAttachments({ stream: mockStream }); - - // Assert - // - Fetched attachments for the first two artifacts only; the third never ran - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(2); - // - Progress emitted and process.exit(0) called once the timeout was detected - expect(emitSpy).toHaveBeenCalledWith( - ExtractorEventType.AttachmentExtractionProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - // - Artifact 1 was shifted out cleanly; artifact 2 remains (timeout caught - // before its shift) along with the untouched artifact 3 - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact2', - 'artifact3', - ]); - - exitSpy.mockRestore(); - }); - - it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => { - // Arrange - const mockStream = jest.fn(); - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - { - url: 'http://example.com/file2.pdf', - id: 'attachment2', - file_name: 'file2.pdf', - parent_id: 'parent2', - }, - { - url: 'http://example.com/file3.pdf', - id: 'attachment3', - file_name: 'file3.pdf', - parent_id: 'parent3', - }, - ], - }); - - adapter.processAttachment = jest.fn().mockResolvedValue(null); - - // Act - await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength( - 0 - ); - expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0); - }); -}); - -describe(`${WorkerAdapter.name}.processAttachment`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.StartExtractingAttachments)); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - const createMockHttpStream = (headers: Record = {}) => ({ - headers, - data: { destroy: jest.fn() }, - }); - - const makeAttachment = (overrides = {}) => ({ - id: 'att-1', - url: 'https://example.com/file.pdf', - file_name: 'file.pdf', - parent_id: 'parent-1', - content_type: 'application/pdf', - ...overrides, - }); - - function setupUploaderHappyPath() { - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art_1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ response: {} }); - adapter['uploader'].confirmArtifactUpload = jest - .fn() - .mockResolvedValue({ response: {} }); - - const pushMock = jest.fn().mockResolvedValue(undefined); - adapter.getRepo = jest.fn().mockReturnValue({ push: pushMock }); - return pushMock; - } - - // ---- content-type resolution ---- - it('should use attachment.content_type when provided, ignoring HTTP header', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({ - 'content-type': 'text/plain', - 'content-length': '100', - }), - }); - - // Act - await adapter.processAttachment( - makeAttachment({ content_type: 'application/pdf' }) as never, - mockStream - ); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'file.pdf', - 'application/pdf', - 100 - ); - }); - - it('should use HTTP header content-type when attachment.content_type is not set', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({ - 'content-type': 'image/jpeg', - 'content-length': '200', - }), - }); - - const attachment = { - id: 'att-2', - url: 'https://example.com/photo.jpg', - file_name: 'photo.jpg', - parent_id: 'parent-2', - }; - - // Act - await adapter.processAttachment(attachment as never, mockStream); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'photo.jpg', - 'image/jpeg', - 200 - ); - }); - - it('should fall back to application/octet-stream when neither content_type nor HTTP header is set', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({}), - }); - - const attachment = { - id: 'att-3', - url: 'https://example.com/file.bin', - file_name: 'file.bin', - parent_id: 'parent-3', - }; - - // Act - await adapter.processAttachment(attachment as never, mockStream); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'file.bin', - 'application/octet-stream', - undefined - ); - }); - - // ---- error paths ---- - it('should return the stream error directly when the stream function returns an error', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ error: new Error('stream failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error).toBeDefined(); - }); - - it('should propagate a rate-limit delay from the stream function', async () => { - // Arrange - const stream = jest.fn().mockResolvedValue({ delay: 5 }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.delay).toBe(5); - }); - - it('should return an error containing the attachment ID when getArtifactUploadUrl fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest - .fn() - .mockResolvedValue({ error: new Error('upload url failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('att-1'); - expect(result?.error?.message).toContain('preparing artifact'); - }); - - it('should return an error when streamArtifact fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art-1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ error: new Error('stream failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('streaming to artifact'); - }); - - it('should return an error when confirmArtifactUpload fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art-1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ response: {} }); - adapter['uploader'].confirmArtifactUpload = jest - .fn() - .mockResolvedValue({ error: new Error('confirm failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('confirming upload'); - }); - - it.each([ - { inline: true, expected: true }, - { inline: false, expected: false }, - ])( - 'should set inline=$expected on the ssorAttachment when attachment.inline=$inline', - async ({ inline, expected }) => { - // Arrange - const pushMock = setupUploaderHappyPath(); - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - - // Act - await adapter.processAttachment( - makeAttachment({ inline }) as never, - stream - ); - - // Assert - const ssorItem = pushMock.mock.calls[0][0][0] as Record; - expect(ssorItem.inline).toBe(expected); - } - ); - - it('should return a descriptive error when the stream function returns no httpStream', async () => { - // Arrange - const stream = jest.fn().mockResolvedValue({ httpStream: null }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain( - 'Error while opening attachment stream' - ); - }); -}); - -describe(`${WorkerAdapter.name}.initializeRepos — event size threshold`, () => { - it('should set isTimeout=true once the cumulative artifact payload exceeds EVENT_SIZE_THRESHOLD_BYTES', () => { - // Arrange - const { adapter } = makeAdapter(); - - let capturedOnUpload: ((artifact: Artifact) => void) | undefined; - const { Repo } = require('../../repo/repo'); - (Repo as jest.Mock).mockImplementationOnce( - (opts: { onUpload: (a: Artifact) => void }) => { - capturedOnUpload = opts.onUpload; - return { itemType: 'issues', upload: jest.fn(), uploadedArtifacts: [] }; - } - ); - - // Act - adapter.initializeRepos([{ itemType: 'issues' }]); - expect(capturedOnUpload).toBeDefined(); - capturedOnUpload!({ - id: 'artifact-x', - item_count: 1, - item_type: 'x'.repeat(200_000), - }); - - // Assert - expect(adapter.isTimeout).toBe(true); - }); -}); - -describe(`${WorkerAdapter.name}.getRepo`, () => { - it('should return undefined when the requested repo was never initialised', () => { - // Arrange - const { adapter } = makeAdapter(); - - // Act - const result = adapter.getRepo('non-existent-type'); - - // Assert - expect(result).toBeUndefined(); - }); -}); - -describe(`${WorkerAdapter.name}.destroyHttpStream`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - ({ adapter } = makeAdapter()); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it.each([ - { - label: 'calls destroy() when available', - data: { destroy: jest.fn(), close: jest.fn() }, - expectDestroy: true, - expectClose: false, - }, - { - label: 'calls close() when destroy is not present', - data: { close: jest.fn() }, - expectDestroy: false, - expectClose: true, - }, - { - label: 'does not throw when neither method is present', - data: {}, - expectDestroy: false, - expectClose: false, - }, - { - label: 'does not throw when data is null', - data: null, - expectDestroy: false, - expectClose: false, - }, - ])('$label', ({ data, expectDestroy, expectClose }) => { - // Arrange - const httpStream = { data } as never; - - // Act & Assert - expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow(); - - if (expectDestroy) { - expect((data as { destroy: jest.Mock }).destroy).toHaveBeenCalled(); - } - if (expectClose) { - expect((data as { close: jest.Mock }).close).toHaveBeenCalled(); - } - }); - - it('should not re-throw when destroy() itself throws', () => { - // Arrange - const httpStream = { - data: { - destroy: () => { - throw new Error('stream error'); - }, - }, - }; - - // Act & Assert - expect(() => - adapter['destroyHttpStream'](httpStream as never) - ).not.toThrow(); - }); -}); - -describe(`${WorkerAdapter.name} — extractionScope`, () => { - it('should return empty object by default', () => { - const { adapter } = makeAdapter(); - expect(adapter.extractionScope).toEqual({}); - }); - - it('should return extraction scope from adapter state', () => { - const { adapter, adapterState } = makeAdapter(); - const extractionScope = { - tasks: { extract: true }, - users: { extract: false }, - }; - - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = extractionScope; - - expect(adapter.extractionScope).toEqual(extractionScope); - }); -}); - -describe(`${WorkerAdapter.name} — shouldExtract`, () => { - it('should return true when extraction scope is empty', () => { - const { adapter } = makeAdapter(); - expect(adapter.shouldExtract('tasks')).toBe(true); - expect(adapter.shouldExtract('users')).toBe(true); - }); - - it('should return true when item type is not in scope', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: true }, - }; - expect(adapter.shouldExtract('users')).toBe(true); - }); - - it('should return true when item type has extract: true', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: true }, - }; - expect(adapter.shouldExtract('tasks')).toBe(true); - }); - - it('should return false when item type has extract: false', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: false }, - users: { extract: true }, - }; - expect(adapter.shouldExtract('tasks')).toBe(false); - expect(adapter.shouldExtract('users')).toBe(true); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts b/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts deleted file mode 100644 index df151d48..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts +++ /dev/null @@ -1,592 +0,0 @@ -import { - ActionType, - ItemTypeToLoad, - LoaderReport, - StatsFileObject, -} from '../../types/loading'; - -import { - addReportToLoaderReport, - getFilesToLoad, -} from './worker-adapter.helpers'; - -describe(getFilesToLoad.name, () => { - let statsFile: StatsFileObject[]; - - beforeEach(() => { - statsFile = [ - { - id: 'test-artifact-1', - file_name: 'test_file_1.json.gz', - item_type: 'issues', - count: '79', - }, - { - id: 'test-artifact-2', - file_name: 'test_file_2.json.gz', - item_type: 'comments', - count: '1079', - }, - { - id: 'test-artifact-3', - file_name: 'test_file_3.json.gz', - item_type: 'issues', - count: '1921', - }, - { - id: 'test-artifact-4', - file_name: 'test_file_4.json.gz', - item_type: 'comments', - count: '921', - }, - { - id: 'test-artifact-5', - file_name: 'test_file_5.json.gz', - item_type: 'attachments', - count: '50', - }, - { - id: 'test-artifact-6', - file_name: 'test_file_6.json.gz', - item_type: 'unknown', - count: '50', - }, - { - id: 'test-artifact-7', - file_name: 'test_file_7.json.gz', - item_type: 'issues', - count: '32', - }, - ]; - }); - - it('should filter files by supported item types and order them correctly', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'attachments', create: jest.fn(), update: jest.fn() }, - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - ]; - const expectedResult = [ - { - id: 'test-artifact-5', - itemType: 'attachments', - count: 50, - file_name: 'test_file_5.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-1', - itemType: 'issues', - count: 79, - file_name: 'test_file_1.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-3', - itemType: 'issues', - count: 1921, - file_name: 'test_file_3.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-7', - itemType: 'issues', - count: 32, - file_name: 'test_file_7.json.gz', - completed: false, - lineToProcess: 0, - }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual(expectedResult); - }); - - it('should ignore files with unrecognized item types in statsFile', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - { itemType: 'unrecognized', create: jest.fn(), update: jest.fn() }, - ]; - const expectedResult = [ - { - id: 'test-artifact-1', - itemType: 'issues', - count: 79, - file_name: 'test_file_1.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-3', - itemType: 'issues', - count: 1921, - file_name: 'test_file_3.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-7', - itemType: 'issues', - count: 32, - file_name: 'test_file_7.json.gz', - completed: false, - lineToProcess: 0, - }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual(expectedResult); - }); - - it('should parse count string to number', () => { - // Arrange - const singleItemStatsFile: StatsFileObject[] = [ - { - id: 'test-artifact-single', - file_name: 'test_file_single.json.gz', - item_type: 'issues', - count: '12345', - }, - ]; - const supportedItemTypes = ['issues']; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: singleItemStatsFile, - }); - - // Assert - expect(result[0].count).toBe(12345); - expect(typeof result[0].count).toBe('number'); - }); - - it('should initialize completed as false and lineToProcess as 0', () => { - // Arrange - const singleItemStatsFile: StatsFileObject[] = [ - { - id: 'test-artifact-init', - file_name: 'test_file_init.json.gz', - item_type: 'issues', - count: '100', - }, - ]; - const supportedItemTypes = ['issues']; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: singleItemStatsFile, - }); - - // Assert - expect(result[0].completed).toBe(false); - expect(result[0].lineToProcess).toBe(0); - }); - - it('[edge] should return an empty array when statsFile is empty', () => { - // Arrange - const emptyStatsFile: StatsFileObject[] = []; - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile: emptyStatsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when supportedItemTypes is empty', () => { - // Arrange - const supportedItemTypes: string[] = []; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when statsFile has no matching items', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'users', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when both statsFile and supportedItemTypes are empty', () => { - // Arrange - const emptyStatsFile: StatsFileObject[] = []; - const supportedItemTypes: string[] = []; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: emptyStatsFile, - }); - - // Assert - expect(result).toEqual([]); - }); -}); - -describe(addReportToLoaderReport.name, () => { - it('should add a new report when no existing report for the item type', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }); - }); - - it('should merge created counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(15); - }); - - it('should merge updated counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.UPDATED]: 20, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.UPDATED]: 8, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.UPDATED]).toBe(28); - }); - - it('should merge failed counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.FAILED]: 3, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.FAILED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.FAILED]).toBe(5); - }); - - it('should merge all action types when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 20, - [ActionType.FAILED]: 3, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 5, - [ActionType.UPDATED]: 8, - [ActionType.FAILED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 15, - [ActionType.UPDATED]: 28, - [ActionType.FAILED]: 5, - }); - }); - - it('should add reports for different item types separately', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'comments', - [ActionType.CREATED]: 50, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(2); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 10, - }); - expect(result[1]).toEqual({ - item_type: 'comments', - [ActionType.CREATED]: 50, - }); - }); - - it('should preserve existing count when new report has undefined for an action type', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(13); - expect(result[0][ActionType.UPDATED]).toBe(5); - }); - - it('should use new report count when existing report has undefined for an action type', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - [ActionType.UPDATED]: 7, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(13); - expect(result[0][ActionType.UPDATED]).toBe(7); - }); - - it('should mutate and return the same loaderReports array', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 10, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toBe(loaderReports); - }); - - it('[edge] should handle report with only item_type and no action counts', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ item_type: 'issues' }); - }); - - it('[edge] should handle merging when both reports have zero counts', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 0, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 0, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(0); - }); - - it('[edge] should handle empty loaderReports array', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'attachments', - [ActionType.CREATED]: 25, - [ActionType.FAILED]: 1, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'attachments', - [ActionType.CREATED]: 25, - [ActionType.FAILED]: 1, - }); - }); - - it('[edge] should preserve existing created count when new report has undefined created', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.UPDATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(10); - expect(result[0][ActionType.UPDATED]).toBe(5); - }); - - it('[edge] should preserve existing updated count when new report has undefined updated', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.UPDATED]: 15, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.UPDATED]).toBe(15); - expect(result[0][ActionType.CREATED]).toBe(3); - }); - - it('[edge] should preserve existing failed count when new report has undefined failed', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.FAILED]: 7, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.FAILED]).toBe(7); - expect(result[0][ActionType.CREATED]).toBe(2); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.loading.test.ts b/src/multithreading/worker-adapter/worker-adapter.loading.test.ts deleted file mode 100644 index 92486ab6..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.loading.test.ts +++ /dev/null @@ -1,729 +0,0 @@ -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - EventType, - LoaderEventType, -} from '../../types'; -import { - ActionType, - ExternalSystemAttachment, - ExternalSystemItem, -} from '../../types/loading'; -import { WorkerAdapter } from './worker-adapter'; - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -function makeLoaderItem(devrevId = 'dev-1'): ExternalSystemItem { - return { - id: { devrev: devrevId, external: 'ext-1' }, - created_date: '', - modified_date: '', - data: {}, - }; -} - -function setupLoaderFile( - adapter: WorkerAdapter, - items: ExternalSystemItem[], - itemType = 'tasks' -) { - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'file.json', - itemType, - count: items.length, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: items }); -} - -describe(`${WorkerAdapter.name}.loadItemTypes — timeout and unexpected errors`, () => { - let adapter: WorkerAdapter; - let exitSpy: jest.SpyInstance; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should emit DataLoadingProgress and exit on timeout', async () => { - // Arrange - const items = [makeLoaderItem('dev-1'), makeLoaderItem('dev-2')]; - setupLoaderFile(adapter, items); - adapter.isTimeout = true; - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith(LoaderEventType.DataLoadingProgress); - expect(exitSpy).toHaveBeenCalledWith(0); - }); - - it('should emit DataLoadingProgress mid-loop when timeout arrives between items', async () => { - // Arrange - const items = [ - makeLoaderItem('dev-1'), - makeLoaderItem('dev-2'), - makeLoaderItem('dev-3'), - ]; - setupLoaderFile(adapter, items); - exitSpy.mockRestore(); - exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => { - throw new Error('process.exit'); - }) as never); - let loadItemCallCount = 0; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/require-await - jest.spyOn(adapter as any, 'loadItem').mockImplementation(async () => { - loadItemCallCount++; - if (loadItemCallCount === 1) { - adapter.isTimeout = true; - } - return { report: { item_type: 'tasks', updated: 1 } }; - }); - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act & Assert - await expect(adapter.loadItemTypes({ itemTypesToLoad })).rejects.toThrow( - 'process.exit' - ); - expect(loadItemCallCount).toBe(1); - expect(emitSpy).toHaveBeenCalledWith(LoaderEventType.DataLoadingProgress); - }); - - it('should emit DataLoadingError and exit(1) on unexpected error', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'file1.json', - itemType: 'tasks', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockRejectedValue(new Error('Unexpected network failure')); - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Error during data loading'), - }), - }) - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); -}); - -describe(`${WorkerAdapter.name}.loadItemTypes — loadItem branch coverage via public API`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - let exitSpy: jest.SpyInstance; - - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - itemTypesToLoad[0].create = jest.fn(); - itemTypesToLoad[0].update = jest.fn(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should accumulate an UPDATED report when the connector updates the item and the mapper sync succeeds', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-1')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - adapter['_mappers'].update = jest.fn().mockResolvedValue({ data: {} }); - itemTypesToLoad[0].update = jest - .fn() - .mockResolvedValue({ id: 'ext-updated-1' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(reports).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - item_type: 'tasks', - [ActionType.UPDATED]: 1, - }), - ]) - ); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should fall back to create and accumulate a CREATED report when the mapper record does not exist (404)', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-2')]); - const axiosError = { isAxiosError: true, response: { status: 404 } }; - adapter['_mappers'].getByTargetId = jest.fn().mockRejectedValue(axiosError); - adapter['_mappers'].create = jest.fn().mockResolvedValue({ data: {} }); - itemTypesToLoad[0].create = jest - .fn() - .mockResolvedValue({ id: 'new-ext-id' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(reports).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - item_type: 'tasks', - [ActionType.CREATED]: 1, - }), - ]) - ); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit DataLoadingDelayed and stop processing when the connector signals a rate-limit delay', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-3')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - itemTypesToLoad[0].update = jest.fn().mockResolvedValue({ delay: 15 }); - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingDelayed, - expect.objectContaining({ delay: 15 }) - ); - }); - - it('should count the item as FAILED when the update succeeds but the mapper sync throws', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-4')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - adapter['_mappers'].update = jest - .fn() - .mockRejectedValue(new Error('mapper down')); - itemTypesToLoad[0].update = jest.fn().mockResolvedValue({ id: 'ext-id' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - expect(reports).toBeDefined(); - }); - - it('should not emit for a non-404 Axios error from the mapper (recorded as item-level error)', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-5')]); - const axiosError = { - isAxiosError: true, - message: 'internal server error', - response: { status: 500 }, - }; - adapter['_mappers'].getByTargetId = jest.fn().mockRejectedValue(axiosError); - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should handle a null sync_mapper_record gracefully and continue loading', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-6')]); - adapter['_mappers'].getByTargetId = jest - .fn() - .mockResolvedValue({ data: null }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - expect(reports).toBeDefined(); - }); -}); - -describe(`${WorkerAdapter.name}.loadItemTypes — additional branches`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should return immediately with empty reports when filesToLoad is empty', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { filesToLoad: [] }; - - // Act - const result = await adapter.loadItemTypes({ - itemTypesToLoad: [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ], - }); - - // Assert - expect(result.reports).toEqual([]); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit DataLoadingError when a file references an item type not in itemTypesToLoad', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'art-1', - file_name: 'file.json', - itemType: 'unknown-type', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: [makeLoaderItem()] }); - - // Act - await adapter.loadItemTypes({ - itemTypesToLoad: [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ], - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('unknown-type'), - }), - }) - ); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachments — timeout, transformer errors, unexpected errors`, () => { - let adapter: WorkerAdapter; - let exitSpy: jest.SpyInstance; - let emitSpy: jest.SpyInstance; - - function setupFilesToLoad( - a: WorkerAdapter, - items: ExternalSystemAttachment[] - ) { - a['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'attachments.json', - itemType: 'attachment', - count: items.length, - lineToProcess: 0, - completed: false, - }, - ], - }; - - a['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: items }); - } - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should emit AttachmentLoadingProgress and exit on timeout', async () => { - // Arrange - const items = [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ] as ExternalSystemAttachment[]; - setupFilesToLoad(adapter, items); - adapter.isTimeout = true; - - // Act - await adapter.loadAttachments({ - create: jest.fn(), - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - }); - - it('should emit AttachmentLoadingError on transformer file error', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'bad-artifact', - file_name: 'attachments.json', - itemType: 'attachment', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ - response: null, - error: new Error('Artifact not found'), - }); - - // Act - await adapter.loadAttachments({ - create: jest.fn(), - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Transformer file not found'), - }), - }) - ); - }); - - it('should emit AttachmentLoadingError and exit(1) on unexpected error', async () => { - // Arrange - const items = [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ] as ExternalSystemAttachment[]; - setupFilesToLoad(adapter, items); - const mockCreate = jest - .fn() - .mockRejectedValue(new Error('Unexpected API failure')); - - // Act - await adapter.loadAttachments({ create: mockCreate }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Error during attachment loading'), - }), - }) - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachments — additional branches`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should return immediately with empty reports when fromDevRev is not set', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = undefined; - - // Act - const result = await adapter.loadAttachments({ create: jest.fn() }); - - // Assert - expect(result.reports).toEqual([]); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit AttachmentLoadingDelayed and stop the loop when the connector signals a rate-limit delay', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'art-1', - file_name: 'attachments.json', - itemType: 'attachment', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ - response: [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ], - }); - jest - // eslint-disable-next-line @typescript-eslint/no-explicit-any - .spyOn(adapter as any, 'loadAttachment') - .mockResolvedValue({ rateLimit: { delay: 20 } } as never); - - // Act - await adapter.loadAttachments({ create: jest.fn() }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingDelayed, - expect.objectContaining({ delay: 20 }) - ); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachment`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - function makeAttachment(): ExternalSystemAttachment { - return { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - } as ExternalSystemAttachment; - } - - it('should return a CREATED report when create succeeds', async () => { - // Arrange - adapter['_mappers'].create = jest.fn().mockResolvedValue({ data: {} }); - const create = jest.fn().mockResolvedValue({ id: 'att-ext-1' }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.item_type).toBe('attachment'); - expect(result.report?.[ActionType.CREATED]).toBe(1); - }); - - it('should still return CREATED even when mapper create fails — attachment loading is resilient', async () => { - // Arrange - adapter['_mappers'].create = jest - .fn() - .mockRejectedValue(new Error('mapper failed')); - const create = jest.fn().mockResolvedValue({ id: 'att-ext-1' }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.[ActionType.CREATED]).toBe(1); - }); - - it('should propagate rate-limit delay when the connector signals one', async () => { - // Arrange - const create = jest.fn().mockResolvedValue({ delay: 30 }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.rateLimit?.delay).toBe(30); - }); - - it('should return a FAILED report when create returns neither id nor delay', async () => { - // Arrange - const create = jest.fn().mockResolvedValue({ id: null, delay: null }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.[ActionType.FAILED]).toBe(1); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts b/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts deleted file mode 100644 index bf488fd7..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { jsonl } from 'js-jsonl'; - -// Pin the serialization contract for items that reach the uploader. -// -// The SDK uploads items via Uploader.upload(), which calls jsonl.stringify() on -// the input. That means user `create`/`update` callbacks and normalizers can -// silently produce inputs that fail or lose information at the wire boundary: -// -// - Circular references throw "Converting circular structure to JSON" -// - BigInt values throw "Do not know how to serialize a BigInt" -// - Date objects are converted to ISO strings (information loss: no Date on -// the other side, just a string) -// - undefined fields are dropped (null fields are preserved) -// -// These tests exist to catch regressions in the serialization layer (e.g., -// silently switching to a different serializer that masks BigInt or mangles -// Dates) before they reach production. - -describe('serialization boundary for items uploaded via jsonl', () => { - it('throws when an item contains a circular reference', () => { - // Arrange - const item: Record = { id: 'a' }; - item.self = item; - - // Act & Assert - expect(() => jsonl.stringify([item])).toThrow(/circular/i); - }); - - it('throws when an item contains a BigInt field', () => { - // Arrange - const item = { id: 'a', counter: BigInt(1) }; - - // Act & Assert - expect(() => jsonl.stringify([item])).toThrow(/BigInt/i); - }); - - it('serializes Date instances to ISO strings (information loss — consumer receives a string)', () => { - // Arrange - const item = { - id: 'a', - created: new Date('2025-01-01T00:00:00.000Z'), - }; - - // Act - const output = jsonl.stringify([item]); - const parsed = JSON.parse(output) as Record; - - // Assert - expect(parsed.created).toBe('2025-01-01T00:00:00.000Z'); - expect(typeof parsed.created).toBe('string'); - }); - - it('drops undefined fields but preserves null fields', () => { - // Arrange - const item = { - id: 'a', - present: null, - missing: undefined, - }; - - // Act - const output = jsonl.stringify([item]); - const parsed = JSON.parse(output) as Record; - - // Assert - expect(parsed).toEqual({ id: 'a', present: null }); - expect(parsed).not.toHaveProperty('missing'); - }); - - it('emits one newline-terminated line per item (jsonl format)', () => { - // Arrange - const items = [{ id: 'a' }, { id: 'b' }, { id: 'c' }]; - - // Act - const output = jsonl.stringify(items); - const lines = output.split('\n').filter((l) => l.length > 0); - - // Assert - expect(lines).toHaveLength(3); - expect(JSON.parse(lines[0])).toEqual({ id: 'a' }); - expect(JSON.parse(lines[2])).toEqual({ id: 'c' }); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts deleted file mode 100644 index 81491292..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ /dev/null @@ -1,1241 +0,0 @@ -import axios, { AxiosResponse } from 'axios'; -import { parentPort } from 'node:worker_threads'; -import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; -import { - AirSyncDefaultItemTypes, - EVENT_SIZE_THRESHOLD_BYTES, - SSOR_ATTACHMENT, - STATELESS_EVENT_TYPES, -} from '../../common/constants'; -import { emit } from '../../common/control-protocol'; -import { - addReportToLoaderReport, - getFilesToLoad, -} from './worker-adapter.helpers'; -import { serializeError } from '../../logger/logger'; -import { - runWithSdkLogContext, - runWithUserLogContext, -} from '../../logger/logger.context'; -import { Mappers } from '../../mappers/mappers'; -import { SyncMapperRecordStatus } from '../../mappers/mappers.interface'; -import { Repo } from '../../repo/repo'; -import { - NormalizedAttachment, - RepoInterface, -} from '../../repo/repo.interfaces'; -import { State } from '../../state/state'; -import { AdapterState } from '../../state/state.interfaces'; -import { - AirdropEvent, - EventData, - EventType, - ExternalSystemAttachmentProcessors, - ExternalSystemAttachmentStreamingFunction, - ExtractorEventType, - ProcessAttachmentReturnType, - StreamAttachmentsReturnType, -} from '../../types/extraction'; -import { - ActionType, - ExternalSystemAttachment, - ExternalSystemItem, - ExternalSystemLoadingFunction, - FileToLoad, - ItemTypesToLoadParams, - ItemTypeToLoad, - LoaderEventType, - LoaderReport, - LoadItemResponse, - LoadItemTypesResponse, - StatsFileObject, -} from '../../types/loading'; -import { - WorkerAdapterInterface, - WorkerAdapterOptions, - WorkerMessageEmitted, - WorkerMessageSubject, -} from '../../types/workers'; -import { Uploader } from '../../uploader/uploader'; -import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; -import { translateOutgoingEventType } from '../../common/event-type-translation'; -import { truncateMessage } from '../../common/helpers'; - -export function createWorkerAdapter({ - event, - adapterState, - options, -}: WorkerAdapterInterface): WorkerAdapter { - return new WorkerAdapter({ - event, - adapterState, - options, - }); -} - -/** - * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in - * as parameter in processTask and onTimeout functions. The class provides - * utilities to emit control events to the platform, update the state of the connector, - * and upload artifacts to the platform. - * @class WorkerAdapter - * @constructor - * @param options - The options to create a new instance of WorkerAdapter class - * @param event - The event object received from the platform - * @param initialState - The initial state of the adapter - * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @param workerPath - The path to the worker file - * - * @public - */ -export class WorkerAdapter { - readonly event: AirdropEvent; - readonly options?: WorkerAdapterOptions; - isTimeout: boolean; - hasWorkerEmitted: boolean; - - private adapterState: State; - private _artifacts: Artifact[]; - private repos: Repo[] = []; - private currentEventDataLength: number = 0; - - // Loader - private loaderReports: LoaderReport[]; - private _processedFiles: string[]; - private _mappers: Mappers; - private uploader: Uploader; - - constructor({ - event, - adapterState, - options, - }: WorkerAdapterInterface) { - this.event = event; - this.options = options; - this.adapterState = adapterState; - this._artifacts = []; - this.hasWorkerEmitted = false; - this.isTimeout = false; - - // Loader - this.loaderReports = []; - this._processedFiles = []; - this._mappers = new Mappers({ - event, - options, - }); - this.uploader = new Uploader({ - event, - options, - }); - } - - get state(): AdapterState { - return this.adapterState.state; - } - - set state(value: AdapterState) { - this.adapterState.state = value; - } - - get reports(): LoaderReport[] { - return this.loaderReports; - } - - get processedFiles(): string[] { - return this._processedFiles; - } - - get mappers(): Mappers { - return this._mappers; - } - - get extractionScope() { - return this.adapterState.extractionScope; - } - - /** - * Returns whether the given item type should be extracted. - * Defaults to true if the scope is empty or the item type is not listed. - */ - shouldExtract(itemType: string): boolean { - const scope = this.extractionScope; - if (Object.keys(scope).length === 0) return true; - if (!(itemType in scope)) return true; - return scope[itemType].extract; - } - - initializeRepos(repos: RepoInterface[]) { - this.repos = repos.map((repo) => { - const shouldNormalize = - repo.itemType !== AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && - repo.itemType !== SSOR_ATTACHMENT; - - return new Repo({ - event: this.event, - itemType: repo.itemType, - ...(shouldNormalize && { normalize: repo.normalize }), - onUpload: (artifact: Artifact) => { - // We need to store artifacts ids in state for later use when streaming attachments - if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { - this.state.toDevRev?.attachmentsMetadata.artifactIds.push( - artifact.id - ); - } - - // Calculate size of the entire artifact object that goes in the SQS message - this.currentEventDataLength += Buffer.byteLength( - JSON.stringify(artifact), - 'utf8' - ); - - if ( - this.currentEventDataLength > EVENT_SIZE_THRESHOLD_BYTES && - !this.isTimeout - ) { - this.isTimeout = true; - } - }, - options: { - ...this.options, - ...repo.overridenOptions, - }, - }); - }); - } - - getRepo(itemType: string): Repo | undefined { - return runWithSdkLogContext(() => { - const repo = this.repos.find((repo) => repo.itemType === itemType); - - if (!repo) { - console.error(`Repo for item type ${itemType} not found.`); - return; - } - - return repo; - }); - } - - async postState() { - return runWithSdkLogContext(async () => { - await this.adapterState.postState(); - }); - } - - get artifacts(): Artifact[] { - return this._artifacts; - } - - set artifacts(artifacts: Artifact[]) { - this._artifacts = this._artifacts - .concat(artifacts) - .filter((value, index, self) => self.indexOf(value) === index); - } - - /** - * Emits an event to the platform. - * - * @param newEventType - The event type to be emitted - * @param data - The data to be sent with the event - */ - async emit( - newEventType: ExtractorEventType | LoaderEventType, - data?: EventData - ): Promise { - return runWithSdkLogContext(async () => { - newEventType = translateOutgoingEventType(newEventType); - - if (this.hasWorkerEmitted) { - console.warn( - `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` - ); - return; - } - - // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting - // TODO: Remove in v2.0.0 - if ( - newEventType === ExtractorEventType.ExternalSyncUnitExtractionDone && - data?.external_sync_units && - data.external_sync_units.length > 0 - ) { - console.log( - `Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.` - ); - - this.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: 25000, - skipConfirmation: true, - }, - }, - ]); - - await this.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push( - data.external_sync_units - ); - - // Remove inline external_sync_units from data to avoid SQS size issues - delete data.external_sync_units; - } - - // Upload all repos before emitting the event - console.log( - `Uploading all repos before emitting event with event type: ${newEventType}.` - ); - - try { - await this.uploadAllRepos(); - } catch (error) { - console.error('Error while uploading repos', error); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - return; - } - - // If the extraction is done, we want to save the timestamp of the last successful sync - if (newEventType === ExtractorEventType.AttachmentExtractionDone) { - console.log( - `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).` - ); - - this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted; - this.state.lastSyncStarted = ''; - - // Clear pending extraction boundaries now that the cycle is complete - this.state.pendingWorkersOldest = ''; - this.state.pendingWorkersNewest = ''; - - // Update workersOldest and workersNewest boundaries from resolved extraction timestamps. - // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest. - const extractionStart = this.event.payload.event_context.extract_from; - const extractionEnd = this.event.payload.event_context.extract_to; - - if ( - extractionStart && - (!this.state.workersOldest || - extractionStart < this.state.workersOldest) - ) { - console.log( - `Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.` - ); - this.state.workersOldest = extractionStart; - } - - if ( - extractionEnd && - (!this.state.workersNewest || - extractionEnd > this.state.workersNewest) - ) { - console.log( - `Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.` - ); - this.state.workersNewest = extractionEnd; - } - } - - // We want to save the state every time we emit an event, except for the start and delete events - if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { - console.log( - `Saving state before emitting event with event type: ${newEventType}.` - ); - - try { - await this.adapterState.postState(this.state); - } catch (error) { - console.error('Error while posting state', error); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - return; - } - } - - try { - // Always prune error messages to make them shorter before emit - if (data?.error?.message) { - data.error.message = truncateMessage(data.error.message); - } - - const isExtractionEvent = Object.values(ExtractorEventType).includes( - newEventType as ExtractorEventType - ); - const isLoaderEvent = Object.values(LoaderEventType).includes( - newEventType as LoaderEventType - ); - - await emit({ - eventType: newEventType, - event: this.event, - data: { - ...data, - ...(isExtractionEvent ? { artifacts: this.artifacts } : {}), - ...(isLoaderEvent - ? { reports: this.reports, processed_files: this.processedFiles } - : {}), - }, - }); - - const message: WorkerMessageEmitted = { - subject: WorkerMessageSubject.WorkerMessageEmitted, - payload: { eventType: newEventType }, - }; - this.artifacts = []; - parentPort?.postMessage(message); - this.hasWorkerEmitted = true; - } catch (error) { - console.error( - `Error while emitting event with event type: ${newEventType}.`, - serializeError(error) - ); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - } - }); - } - - async uploadAllRepos(): Promise { - for (const repo of this.repos) { - const error = await repo.upload(); - this.artifacts.push(...repo.uploadedArtifacts); - if (error) { - throw error; - } - } - } - - async loadItemTypes({ - itemTypesToLoad, - }: ItemTypesToLoadParams): Promise { - return runWithSdkLogContext(async () => { - if (this.event.payload.event_type === EventType.StartLoadingData) { - const itemTypes = itemTypesToLoad.map( - (itemTypeToLoad) => itemTypeToLoad.itemType - ); - - if (!itemTypes.length) { - console.warn('No item types to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - const filesToLoad = await this.getLoaderBatches({ - supportedItemTypes: itemTypes, - }); - this.adapterState.state.fromDevRev = { - filesToLoad, - }; - } - - if ( - !this.adapterState.state.fromDevRev || - !this.adapterState.state.fromDevRev.filesToLoad.length - ) { - console.warn('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - console.log( - 'Files to load in state', - this.adapterState.state.fromDevRev?.filesToLoad - ); - - try { - outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev - .filesToLoad) { - const itemTypeToLoad = itemTypesToLoad.find( - (itemTypeToLoad: ItemTypeToLoad) => - itemTypeToLoad.itemType === fileToLoad.itemType - ); - - if (!itemTypeToLoad) { - console.error( - `Item type to load not found for item type: ${fileToLoad.itemType}.` - ); - - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, - }, - }); - - break; - } - - if (!fileToLoad.completed) { - const { response, error: transformerFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: fileToLoad.id, - isGzipped: true, - }); - - if (transformerFileError) { - console.error( - `Transformer file not found for artifact ID: ${fileToLoad.id}.` - ); - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, - }, - }); - break outerloop; - } - - const transformerFile = response as ExternalSystemItem[]; - - for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { - if (this.isTimeout) { - console.log( - 'Timeout detected during data loading. Emitting progress to allow continuation.' - ); - await this.emit(LoaderEventType.DataLoadingProgress); - process.exit(0); - } - - const { report, rateLimit } = await this.loadItem({ - item: transformerFile[i], - itemTypeToLoad, - }); - - if (rateLimit?.delay) { - await this.emit(LoaderEventType.DataLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; - } - - if (report) { - addReportToLoaderReport({ - loaderReports: this.loaderReports, - report, - }); - fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; - } - } - - fileToLoad.completed = true; - this._processedFiles.push(fileToLoad.id); - } - } - } catch (error) { - console.error('Error during data loading.', serializeError(error)); - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Error during data loading. ${serializeError(error)}`, - }, - }); - process.exit(1); - } - - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - }); - } - - async getLoaderBatches({ - supportedItemTypes, - }: { - supportedItemTypes: string[]; - }) { - return runWithSdkLogContext(async () => { - const statsFileArtifactId = this.event.payload.event_data?.stats_file; - - if (statsFileArtifactId) { - const { response, error: statsFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: statsFileArtifactId, - }); - - const statsFile = response as StatsFileObject[]; - - if (statsFileError || statsFile.length === 0) { - return [] as FileToLoad[]; - } - - const filesToLoad = getFilesToLoad({ - supportedItemTypes, - statsFile, - }); - - return filesToLoad; - } - - return [] as FileToLoad[]; - }); - } - - async loadAttachments({ - create, - }: { - create: ExternalSystemLoadingFunction; - }): Promise { - return runWithSdkLogContext(async () => { - if (this.event.payload.event_type === EventType.StartLoadingAttachments) { - this.adapterState.state.fromDevRev = { - filesToLoad: await this.getLoaderBatches({ - supportedItemTypes: ['attachment'], - }), - }; - } - - if ( - !this.adapterState.state.fromDevRev || - this.adapterState.state.fromDevRev?.filesToLoad.length === 0 - ) { - console.log('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - const filesToLoad = this.adapterState.state.fromDevRev?.filesToLoad; - - try { - outerloop: for (const fileToLoad of filesToLoad) { - if (!fileToLoad.completed) { - const { response, error: transformerFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: fileToLoad.id, - isGzipped: true, - }); - - const transformerFile = response as ExternalSystemAttachment[]; - - if (transformerFileError) { - console.error( - `Transformer file not found for artifact ID: ${fileToLoad.id}.` - ); - await this.emit(LoaderEventType.AttachmentLoadingError, { - error: { - message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, - }, - }); - break outerloop; - } - - for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { - if (this.isTimeout) { - console.log( - 'Timeout detected during attachment loading. Emitting progress to allow continuation.' - ); - await this.emit(LoaderEventType.AttachmentLoadingProgress); - process.exit(0); - } - - const { report, rateLimit } = await this.loadAttachment({ - item: transformerFile[i], - create, - }); - - if (rateLimit?.delay) { - await this.emit(LoaderEventType.AttachmentLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; - } - - if (report) { - addReportToLoaderReport({ - loaderReports: this.loaderReports, - report, - }); - fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; - } - } - - fileToLoad.completed = true; - this._processedFiles.push(fileToLoad.id); - } - } - } catch (error) { - console.error( - 'Error during attachment loading.', - serializeError(error) - ); - await this.emit(LoaderEventType.AttachmentLoadingError, { - error: { - message: `Error during attachment loading. ${serializeError( - error - )}`, - }, - }); - process.exit(1); - } - - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - }); - } - - async loadItem({ - item, - itemTypeToLoad, - }: { - item: ExternalSystemItem; - itemTypeToLoad: ItemTypeToLoad; - }): Promise { - return runWithSdkLogContext(async () => { - const devrevId = item.id.devrev; - - try { - const syncMapperRecordResponse = await this._mappers.getByTargetId({ - sync_unit: this.event.payload.event_context.sync_unit, - target: devrevId, - }); - - const syncMapperRecord = syncMapperRecordResponse.data; - if (!syncMapperRecord) { - console.warn('Failed to get sync mapper record from response.'); - return { - error: { - message: 'Failed to get sync mapper record from response.', - }, - }; - } - - // Update item in external system - const { id, modifiedDate, delay, error } = await runWithUserLogContext( - async () => { - return await itemTypeToLoad.update({ - item, - mappers: this._mappers, - event: this.event, - }); - } - ); - - if (id) { - try { - const syncMapperRecordUpdateResponse = await this._mappers.update({ - id: syncMapperRecord.sync_mapper_record.id, - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - ...(modifiedDate && { - external_versions: { - add: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }, - }), - external_ids: { - add: [id], - }, - targets: { - add: [devrevId], - }, - }); - - console.log( - 'Successfully updated sync mapper record.', - syncMapperRecordUpdateResponse.data - ); - } catch (error) { - console.warn( - 'Failed to update sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to update sync mapper record' + serializeError(error), - }, - }; - } - - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.UPDATED]: 1, - }, - }; - } else if (delay) { - console.log( - `Rate limited while updating item in external system, delaying for ${delay} seconds.` - ); - - return { - rateLimit: { - delay, - }, - }; - } else { - console.warn('Failed to update item in external system', error); - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.FAILED]: 1, - }, - }; - } - - // TODO: Update mapper (optional) - } catch (error) { - if (axios.isAxiosError(error)) { - if (error.response?.status === 404) { - // Create item in external system if mapper record not found - const { id, modifiedDate, delay, error } = - await runWithUserLogContext(async () => { - return await itemTypeToLoad.create({ - item, - mappers: this._mappers, - event: this.event, - }); - }); - - if (id) { - // Create mapper - try { - const syncMapperRecordCreateResponse = - await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - external_ids: [id], - targets: [devrevId], - ...(modifiedDate && { - external_versions: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }), - }); - - console.log( - 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data - ); - - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.CREATED]: 1, - }, - }; - } catch (error) { - console.warn( - 'Failed to create sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to create sync mapper record. ' + - serializeError(error), - }, - }; - } - } else if (delay) { - return { - rateLimit: { - delay, - }, - }; - } else { - console.warn( - 'Failed to create item in external system.', - serializeError(error) - ); - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.FAILED]: 1, - }, - }; - } - } else { - console.warn( - 'Failed to get sync mapper record.', - serializeError(error) - ); - return { - error: { - message: error.message, - }, - }; - } - } - - console.warn( - 'Failed to get sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to get sync mapper record. ' + serializeError(error), - }, - }; - } - }); - } - - async processAttachment( - attachment: NormalizedAttachment, - stream: ExternalSystemAttachmentStreamingFunction - ): Promise { - return runWithSdkLogContext(async () => { - const { httpStream, delay, error } = await runWithUserLogContext( - async () => - stream({ - item: attachment, - event: this.event, - }) - ); - - if (error) { - return { error }; - } else if (delay) { - return { delay }; - } - - if (httpStream) { - const fileType = - attachment.content_type || - httpStream.headers['content-type']?.toString() || - 'application/octet-stream'; - const contentLength = httpStream.headers['content-length']?.toString(); - const fileSize = contentLength ? parseInt(contentLength) : undefined; - - // Get upload URL - const { error: artifactUrlError, response: artifactUrlResponse } = - await this.uploader.getArtifactUploadUrl( - attachment.file_name, - fileType, - fileSize - ); - - if (artifactUrlError) { - this.destroyHttpStream(httpStream); - return { - error: { - message: `Error while preparing artifact for attachment ID ${ - attachment.id - }. Skipping attachment. ${serializeError(artifactUrlError)}`, - fileSize: fileSize, - }, - }; - } - - // Stream attachment - const { error: uploadedArtifactError } = - await this.uploader.streamArtifact(artifactUrlResponse!, httpStream); - - if (uploadedArtifactError) { - this.destroyHttpStream(httpStream); - return { - error: { - message: - `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` + - serializeError(uploadedArtifactError), - fileSize: fileSize, - }, - }; - } - - // Confirm attachment upload - const { error: confirmArtifactUploadError } = - await this.uploader.confirmArtifactUpload( - artifactUrlResponse!.artifact_id - ); - if (confirmArtifactUploadError) { - return { - error: { - message: - `Error while confirming upload for attachment ID ${attachment.id}. ` + - serializeError(confirmArtifactUploadError), - fileSize: fileSize, - }, - }; - } - - const ssorAttachment: SsorAttachment = { - id: { - devrev: artifactUrlResponse!.artifact_id, - external: attachment.id, - }, - parent_id: { - external: attachment.parent_id, - }, - }; - - if (attachment.author_id) { - ssorAttachment.actor_id = { - external: attachment.author_id, - }; - } - - // This will set inline flag in ssor_attachment only if it is explicity - // set in the attachment object. - if (attachment.inline === true) { - ssorAttachment.inline = true; - } else if (attachment.inline === false) { - ssorAttachment.inline = false; - } - - await this.getRepo('ssor_attachment')?.push([ssorAttachment]); - return; - } - return { - error: { - message: `Error while opening attachment stream. Skipping attachment.`, - }, - }; - }); - } - - /** - * Destroys a stream to prevent memory leaks. - * @param httpStream - The axios response stream to destroy - */ - private destroyHttpStream(httpStream: AxiosResponse): void { - try { - if (httpStream && httpStream.data) { - if (typeof httpStream.data.destroy === 'function') { - httpStream.data.destroy(); - } else if (typeof httpStream.data.close === 'function') { - httpStream.data.close(); - } - } - } catch (error) { - console.warn('Error while destroying HTTP stream:', error); - } - } - - async loadAttachment({ - item, - create, - }: { - item: ExternalSystemAttachment; - create: ExternalSystemLoadingFunction; - }): Promise { - return runWithSdkLogContext(async () => { - // Create item - const { id, delay, error } = await runWithUserLogContext(async () => - create({ - item, - mappers: this._mappers, - event: this.event, - }) - ); - - if (delay) { - return { - rateLimit: { - delay, - }, - }; - } else if (id) { - try { - const syncMapperRecordCreateResponse = await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - external_ids: [id], - targets: [item.reference_id], - status: SyncMapperRecordStatus.OPERATIONAL, - }); - - console.log( - 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data - ); - } catch (error) { - console.warn( - 'Failed to create sync mapper record.', - serializeError(error) - ); - } - - return { - report: { - item_type: 'attachment', - [ActionType.CREATED]: 1, - }, - }; - } else { - console.warn('Failed to create attachment in external system', error); - return { - report: { - item_type: 'attachment', - [ActionType.FAILED]: 1, - }, - }; - } - }); - } - - /** - * Streams the attachments to the DevRev platform. - * The attachments are streamed to the platform and the artifact information is returned. - * @param params - The parameters to stream the attachments - * @returns The response object containing the ssorAttachment artifact information - * or error information if there was an error - */ - async streamAttachments({ - stream, - processors, - batchSize = 1, // By default, we want to stream one attachment at a time - }: { - stream: ExternalSystemAttachmentStreamingFunction; - processors?: ExternalSystemAttachmentProcessors< - ConnectorState, - NormalizedAttachment[], - NewBatch - >; - batchSize?: number; - }): Promise { - return runWithSdkLogContext(async () => { - if (batchSize <= 0) { - console.warn( - `The specified batch size (${batchSize}) is invalid. Using 1 instead.` - ); - batchSize = 1; - } - - if (batchSize > 50) { - console.warn( - `The specified batch size (${batchSize}) is too large. Using 50 instead.` - ); - batchSize = 50; - } - - const repos = [ - { - itemType: 'ssor_attachment', - }, - ]; - this.initializeRepos(repos); - - const attachmentsMetadata = this.state.toDevRev?.attachmentsMetadata; - - // If there are no attachments metadata artifact IDs in state, finish here - if (!attachmentsMetadata?.artifactIds?.length) { - console.log(`No attachments metadata artifact IDs found in state.`); - return; - } else { - console.log( - `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` - ); - } - - // Loop through the attachments metadata artifact IDs - while (attachmentsMetadata.artifactIds.length > 0) { - const attachmentsMetadataArtifactId = - attachmentsMetadata.artifactIds[0]; - - console.log( - `Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.` - ); - - const { attachments, error } = - await this.uploader.getAttachmentsFromArtifactId({ - artifact: attachmentsMetadataArtifactId, - }); - - if (error) { - console.error( - `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - return { error }; - } - - if (!attachments || attachments.length === 0) { - console.warn( - `No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - // Remove empty artifact and reset lastProcessed - attachmentsMetadata.artifactIds.shift(); - attachmentsMetadata.lastProcessed = 0; - continue; - } - - console.log( - `Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - - let response; - - if (processors) { - console.log(`Using custom processors for attachments.`); - - const reducer = processors.reducer; - const iterator = processors.iterator; - - const reducedAttachments = runWithUserLogContext(() => - reducer({ - attachments, - adapter: this, - batchSize, - }) - ); - - response = await runWithUserLogContext(async () => { - return await iterator({ - reducedAttachments, - adapter: this, - stream, - }); - }); - } else { - console.log( - `Using attachments streaming pool for attachments streaming.` - ); - - const attachmentsPool = new AttachmentsStreamingPool({ - adapter: this, - attachments, - batchSize, - stream, - }); - - response = await attachmentsPool.streamAll(); - } - - if (response?.delay || response?.error) { - return response; - } - - // On timeout, emit progress and exit to allow continuation. - if (this.isTimeout) { - console.log( - `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.` - ); - await this.emit(ExtractorEventType.AttachmentExtractionProgress); - process.exit(0); - return; - } - - console.log( - `Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - attachmentsMetadata.artifactIds.shift(); - attachmentsMetadata.lastProcessed = 0; - if (attachmentsMetadata.lastProcessedAttachmentsIdsList) { - attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0; - } - } - - return; - }); - } -} diff --git a/src/multithreading/worker.js b/src/multithreading/worker.js index f19e44c6..7f35818b 100644 --- a/src/multithreading/worker.js +++ b/src/multithreading/worker.js @@ -2,8 +2,7 @@ const { workerData } = require('node:worker_threads'); require('ts-node').register(); -const { Logger } = require('../logger/logger'); -const { runWithUserLogContext } = require('../logger/logger.context'); +const { Logger, runWithUserLogContext } = require('../logger/logger'); console = new Logger({ event: workerData.event, options: workerData.options }); diff --git a/src/repo/repo.interfaces.ts b/src/repo/repo.interfaces.ts index 4573a3a0..f790be2a 100644 --- a/src/repo/repo.interfaces.ts +++ b/src/repo/repo.interfaces.ts @@ -1,6 +1,6 @@ import { Artifact } from '../uploader/uploader.interfaces'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; /** @@ -16,7 +16,7 @@ export interface RepoInterface { * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo. */ export interface RepoFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; itemType: string; normalize?: (record: object) => NormalizedItem | NormalizedAttachment; onUpload: (artifact: Artifact) => void; diff --git a/src/repo/repo.test.ts b/src/repo/repo.test.ts index b1d93c76..648f1bb3 100644 --- a/src/repo/repo.test.ts +++ b/src/repo/repo.test.ts @@ -1,8 +1,8 @@ import { AirSyncDefaultItemTypes, SSOR_ATTACHMENT } from '../common/constants'; import { createItems, normalizeItem } from '../tests/test-helpers'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; -import { EventType } from '../types'; +import { createMockEvent } from '../testing/mock-event'; +import { EventType } from '../types/extraction'; import { Repo } from './repo'; jest.mock('../tests/test-helpers', () => ({ @@ -18,7 +18,7 @@ describe(Repo.name, () => { normalize = jest.fn(); repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', normalize, @@ -47,7 +47,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', onUpload: jest.fn(), @@ -74,7 +74,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA, normalize, @@ -94,7 +94,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: SSOR_ATTACHMENT, normalize, @@ -160,7 +160,7 @@ describe(Repo.name, () => { beforeEach(() => { repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', normalize, diff --git a/src/repo/repo.ts b/src/repo/repo.ts index d119eea8..a54e30a4 100644 --- a/src/repo/repo.ts +++ b/src/repo/repo.ts @@ -9,7 +9,7 @@ import { Uploader } from '../uploader/uploader'; import { Artifact } from '../uploader/uploader.interfaces'; import { WorkerAdapterOptions } from '../types/workers'; -import { runWithUserLogContext } from '../logger/logger.context'; +import { runWithUserLogContext } from '../logger/logger'; import { NormalizedAttachment, NormalizedItem, diff --git a/src/state/base-state.ts b/src/state/base-state.ts new file mode 100644 index 00000000..0ce50a94 --- /dev/null +++ b/src/state/base-state.ts @@ -0,0 +1,323 @@ +import axios from 'axios'; +import { parentPort } from 'node:worker_threads'; + +import { installInitialDomainMapping } from './install-initial-domain-mapping'; +import { axiosClient } from '../http/client'; +import { getPrintableState, serializeError } from '../logger/logger'; +import { InitialDomainMapping } from '../types/common'; +import { AirSyncEvent } from '../types/extraction'; +import { WorkerMessageSubject } from '../types/workers'; +import { ExtractionScope } from '../types/workers'; + +import { + AdapterStateEnvelope, + SdkState, + StateInterface, + V1_SDK_STATE_KEYS, +} from './state.interfaces'; + +/** + * BaseState owns the state lifecycle shared by every sync mode: connector vs. + * SDK state separation, fetch/init/post against the platform, the v1->v2 + * migration shim, and the snap-in-version-gated initial domain mapping install. + * + * Mode-specific subclasses (`ExtractionState`, `LoadingState`) seed the + * SDK-owned portion of the state and add mode-specific setup in their factories. + * + * @typeParam ConnectorState - the connector-owned state shape + */ +export abstract class BaseState { + protected _connectorState: ConnectorState; + protected _sdkState: SdkState; + protected _extractionScope: ExtractionScope = {}; + protected readonly initialSdkState: SdkState; + protected readonly event: AirSyncEvent; + private workerUrl: string; + private devrevToken: string; + private syncUnitId: string; + private requestId: string; + + constructor( + { event, initialState }: StateInterface, + initialSdkState: SdkState + ) { + this.event = event; + this.initialSdkState = initialSdkState; + this._connectorState = initialState; + this._sdkState = { ...initialSdkState }; + this.workerUrl = event.payload.event_context.worker_data_url; + this.devrevToken = event.context.secrets.service_account_token; + this.syncUnitId = event.payload.event_context.sync_unit_id; + this.requestId = event.payload.event_context.request_id_adaas; + } + + /** Connector-owned state. This is what `adapter.state` exposes to snap-in code. */ + get state(): ConnectorState { + return this._connectorState; + } + + set state(value: ConnectorState) { + this._connectorState = value; + } + + /** SDK-internal bookkeeping state. Never exposed to connector code. */ + get sdkState(): SdkState { + return this._sdkState; + } + + set sdkState(value: SdkState) { + this._sdkState = value; + } + + get extractionScope(): ExtractionScope { + return this._extractionScope; + } + + /** + * Installs the initial domain mapping when the snap-in version in state does + * not match the version in the event context. Shared by all modes so that a + * loading run still installs the mapping if extraction has not done so. + * @param initialDomainMapping The initial domain mapping passed to spawn + */ + async installInitialDomainMappingIfNeeded( + initialDomainMapping?: InitialDomainMapping + ): Promise { + const snapInVersionId = this.event.context.snap_in_version_id; + const hasSnapInVersionInState = 'snapInVersionId' in this.sdkState; + const shouldUpdateIDM = + !hasSnapInVersionInState || + this.sdkState.snapInVersionId !== snapInVersionId; + + if (!shouldUpdateIDM) { + console.log( + `Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.` + ); + return; + } + + try { + console.log( + `Snap-in version in state "${this.sdkState.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` + ); + + if (initialDomainMapping) { + await installInitialDomainMapping(this.event, initialDomainMapping); + this.sdkState.snapInVersionId = snapInVersionId; + } else { + throw new Error( + 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' + ); + } + } catch (error) { + const errorMessage = `Error while installing initial domain mapping. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + + /** + * Initializes the state for this adapter instance by fetching from API + * or creating an initial state if none exists (404). + * + * Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat + * v1 blob (connector keys merged with SDK keys), migrating the latter on read. + * Always persists the v2 envelope going forward. + * @param initialState The initial connector state provided by the spawn function + */ + async init(initialState: ConnectorState): Promise { + try { + const { state: stringifiedState, objects } = await this.fetchState(); + if (!stringifiedState) { + throw new Error('No state found in response.'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(stringifiedState); + } catch (error) { + throw new Error(`Failed to parse state. ${error}`); + } + + const { connectorState, sdkState } = this.normalizeFetchedState(parsed); + this.state = connectorState; + this.sdkState = sdkState; + + console.log('State fetched successfully. Current state', { + connectorState: getPrintableState( + this.state as Record + ), + sdkState: getPrintableState(this.sdkState), + }); + + if (objects) { + try { + this._extractionScope = JSON.parse(objects); + } catch (error) { + console.warn(`Failed to parse extractionScope. ${error}`); + } + } + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 404) { + console.log('State not found. Initializing state with initial state.'); + this.state = initialState; + this.sdkState = { ...this.initialSdkState }; + await this.postState(); + } else { + const errorMessage = `Failed to init state. ${serializeError(error)}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } + + /** + * Normalizes a parsed on-disk state into the `{ connectorState, sdkState }` + * envelope, migrating a legacy flat v1 blob if needed. + * + * - v2 envelope (`{ connectorState, sdkState }`): used as-is. + * - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`, + * everything else becomes connector state. + * - Malformed envelope (one side present, the other missing) fails loud. + */ + private normalizeFetchedState(parsed: unknown): { + connectorState: ConnectorState; + sdkState: SdkState; + } { + if (parsed === null || typeof parsed !== 'object') { + throw new Error('Fetched state is not a JSON object.'); + } + + const record = parsed as Record; + const hasConnector = 'connectorState' in record; + const hasSdk = 'sdkState' in record; + + if (hasConnector || hasSdk) { + if (!hasConnector || !hasSdk) { + throw new Error( + 'Malformed state envelope: expected both "connectorState" and "sdkState".' + ); + } + return { + connectorState: record.connectorState as ConnectorState, + sdkState: { ...this.initialSdkState, ...(record.sdkState as SdkState) }, + }; + } + + // Legacy flat v1 blob: split known SDK keys out of the connector state. + const connectorState: Record = {}; + const sdkState: Record = {}; + for (const [key, value] of Object.entries(record)) { + if (V1_SDK_STATE_KEYS.has(key)) { + sdkState[key] = value; + } else { + connectorState[key] = value; + } + } + + return { + connectorState: connectorState as ConnectorState, + sdkState: { ...this.initialSdkState, ...(sdkState as SdkState) }, + }; + } + + /** + * Updates the state of the adapter by posting to API. + * Persists the v2 `{ connectorState, sdkState }` envelope. + * @param {object} state - The connector state to be updated + */ + async postState(state?: ConnectorState) { + const url = this.workerUrl + '.update'; + this.state = state || this.state; + + const envelope: AdapterStateEnvelope = { + connectorState: this.state, + sdkState: this.sdkState, + }; + + let stringifiedState: string; + try { + stringifiedState = JSON.stringify(envelope); + } catch (error) { + const errorMessage = `Failed to stringify state. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + + try { + await axiosClient.post( + url, + { + state: stringifiedState, + }, + { + headers: { + Authorization: this.devrevToken, + }, + params: { + sync_unit: this.syncUnitId, + request_id: this.requestId, + }, + } + ); + + console.log('State updated successfully to', { + connectorState: getPrintableState( + this.state as Record + ), + sdkState: getPrintableState(this.sdkState), + }); + } catch (error) { + const errorMessage = `Failed to update the state. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + + /** + * Fetches the state of the adapter from API. + * @return The raw state data from API + */ + async fetchState(): Promise<{ state: string; objects?: string }> { + console.log( + `Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.` + ); + + const url = this.workerUrl + '.get'; + const response = await axiosClient.get(url, { + headers: { + Authorization: this.devrevToken, + }, + params: { + sync_unit: this.syncUnitId, + request_id: this.requestId, + }, + }); + + return { + state: response.data?.state, + objects: response.data?.objects, + }; + } +} diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts new file mode 100644 index 00000000..5548dc78 --- /dev/null +++ b/src/state/extraction-state.ts @@ -0,0 +1,142 @@ +import { parentPort } from 'node:worker_threads'; + +import { STATELESS_EVENT_TYPES } from '../common/constants'; +import { resolveTimeValue } from './time-value-resolver'; +import { serializeError } from '../logger/logger'; +import { EventType } from '../types/extraction'; +import { WorkerMessageSubject } from '../types/workers'; + +import { BaseState } from './base-state'; +import { extractionSdkState, StateInterface } from './state.interfaces'; + +/** + * ExtractionState is the per-mode state for extraction workers. It seeds the + * extraction SDK state (extraction boundaries + attachments bookkeeping) on top + * of the shared lifecycle provided by `BaseState` and adds extraction-window + * resolution. + */ +export class ExtractionState extends BaseState { + constructor(params: StateInterface) { + super(params, extractionSdkState); + } + + /** + * Resolves the extraction window onto the event context. + * + * On StartExtractingMetadata: resolve fresh from the TimeValue objects in the + * event context and cache them as pending boundaries (always overwrite). + * On all other events: reuse the pending boundaries cached during + * StartExtractingMetadata. Finally, validate that extract_from < extract_to. + */ + resolveExtractionWindow(): void { + const sdkState = this.sdkState; + + const eventContext = this.event.payload.event_context; + + if (this.event.payload.event_type === EventType.StartExtractingMetadata) { + const timeFields = [ + { + source: 'extraction_start_time', + target: 'extract_from', + pending: 'pendingWorkersOldest', + }, + { + source: 'extraction_end_time', + target: 'extract_to', + pending: 'pendingWorkersNewest', + }, + ] as const; + + for (const { source, target, pending } of timeFields) { + const timeValue = eventContext[source]; + if (timeValue && timeValue.type) { + try { + const resolved = resolveTimeValue(timeValue, sdkState); + eventContext[target] = resolved; + sdkState[pending] = resolved; + console.log( + `Resolved ${target} to ${resolved}. Stored in ${pending}.` + ); + } catch (error) { + const errorMessage = `Failed to resolve ${source}: ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } + } else { + // Non-StartExtractingMetadata events: reuse pending values from state + if (sdkState.pendingWorkersOldest) { + eventContext.extract_from = sdkState.pendingWorkersOldest; + console.log( + `Reusing pendingWorkersOldest as extract_from: ${sdkState.pendingWorkersOldest}.` + ); + } else { + console.log( + 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' + ); + } + if (sdkState.pendingWorkersNewest) { + eventContext.extract_to = sdkState.pendingWorkersNewest; + console.log( + `Reusing pendingWorkersNewest as extract_to: ${sdkState.pendingWorkersNewest}.` + ); + } else { + console.log( + 'pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.' + ); + } + } + + // Validate that extract_from is before extract_to + if (eventContext.extract_from && eventContext.extract_to) { + if (eventContext.extract_from >= eventContext.extract_to) { + const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } +} + +/** + * Creates and initializes an `ExtractionState` for an extraction worker. + * + * For non-stateless events this fetches persisted state, installs the initial + * domain mapping if the snap-in version changed, then resolves the extraction + * window (time-value resolution + pending boundary reuse) and validates it. + */ +export async function createExtractionState({ + event, + initialState, + initialDomainMapping, + options, +}: StateInterface): Promise> { + // Deep clone the initial state to avoid mutating the original state + const deepCloneInitialState: ConnectorState = structuredClone(initialState); + + const state = new ExtractionState({ + event, + initialState: deepCloneInitialState, + initialDomainMapping, + options, + }); + + if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { + await state.init(deepCloneInitialState); + await state.installInitialDomainMappingIfNeeded(initialDomainMapping); + state.resolveExtractionWindow(); + } + + return state; +} diff --git a/src/common/install-initial-domain-mapping.test.ts b/src/state/install-initial-domain-mapping.test.ts similarity index 95% rename from src/common/install-initial-domain-mapping.test.ts rename to src/state/install-initial-domain-mapping.test.ts index ec56b02c..4e953d54 100644 --- a/src/common/install-initial-domain-mapping.test.ts +++ b/src/state/install-initial-domain-mapping.test.ts @@ -1,8 +1,8 @@ import axios from 'axios'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from './test-utils'; -import { InitialDomainMapping } from '../types'; +import { createMockEvent } from '../testing/mock-event'; +import { InitialDomainMapping } from '../types/common'; import { EventType } from '../types/extraction'; import { installInitialDomainMapping } from './install-initial-domain-mapping'; @@ -11,8 +11,8 @@ jest.mock('axios', () => ({ ...jest.requireActual('axios'), isAxiosError: jest.fn(), })); -jest.mock('../http/axios-client-internal', () => { - const originalModule = jest.requireActual('../http/axios-client-internal'); +jest.mock('../http/client', () => { + const originalModule = jest.requireActual('../http/client'); return { ...originalModule, axiosClient: { @@ -29,7 +29,7 @@ const mockIsAxiosError = axios.isAxiosError as unknown as jest.Mock; describe(installInitialDomainMapping.name, () => { // Create mock objects const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }); const mockInitialDomainMapping: InitialDomainMapping = { diff --git a/src/common/install-initial-domain-mapping.ts b/src/state/install-initial-domain-mapping.ts similarity index 95% rename from src/common/install-initial-domain-mapping.ts rename to src/state/install-initial-domain-mapping.ts index f65026ea..9cfffbd0 100644 --- a/src/common/install-initial-domain-mapping.ts +++ b/src/state/install-initial-domain-mapping.ts @@ -1,11 +1,11 @@ -import { axiosClient } from '../http/axios-client-internal'; -import { AirdropEvent } from '../types/extraction'; +import { axiosClient } from '../http/client'; +import { AirSyncEvent } from '../types/extraction'; import { serializeError } from '../logger/logger'; import { InitialDomainMapping } from '../types/common'; export async function installInitialDomainMapping( - event: AirdropEvent, + event: AirSyncEvent, initialDomainMappingJson: InitialDomainMapping ): Promise { const devrevEndpoint = event.execution_metadata.devrev_endpoint; diff --git a/src/state/loading-state.ts b/src/state/loading-state.ts new file mode 100644 index 00000000..151f5bd9 --- /dev/null +++ b/src/state/loading-state.ts @@ -0,0 +1,45 @@ +import { STATELESS_EVENT_TYPES } from '../common/constants'; + +import { BaseState } from './base-state'; +import { loadingSdkState, StateInterface } from './state.interfaces'; + +/** + * LoadingState is the per-mode state for loading workers. It seeds the loading + * SDK state (files-to-load bookkeeping) on top of the shared lifecycle provided + * by `BaseState`. Loading has no extraction-window resolution. + */ +export class LoadingState extends BaseState { + constructor(params: StateInterface) { + super(params, loadingSdkState); + } +} + +/** + * Creates and initializes a `LoadingState` for a loading worker. + * + * For non-stateless events this fetches persisted state and installs the + * initial domain mapping if the snap-in version changed. + */ +export async function createLoadingState({ + event, + initialState, + initialDomainMapping, + options, +}: StateInterface): Promise> { + // Deep clone the initial state to avoid mutating the original state + const deepCloneInitialState: ConnectorState = structuredClone(initialState); + + const state = new LoadingState({ + event, + initialState: deepCloneInitialState, + initialDomainMapping, + options, + }); + + if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { + await state.init(deepCloneInitialState); + await state.installInitialDomainMappingIfNeeded(initialDomainMapping); + } + + return state; +} diff --git a/src/state/state.extract-window.test.ts b/src/state/state.extract-window.test.ts index 74d76452..51841262 100644 --- a/src/state/state.extract-window.test.ts +++ b/src/state/state.extract-window.test.ts @@ -1,7 +1,7 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; describe('State — extraction window validation', () => { let fetchStateSpy: jest.SpyInstance; @@ -11,7 +11,7 @@ describe('State — extraction window validation', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -43,7 +43,7 @@ describe('State — extraction window validation', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -77,7 +77,7 @@ describe('State — extraction window validation', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -110,7 +110,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -140,7 +140,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -173,7 +173,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index f3c05929..d6b77bf4 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -1,19 +1,9 @@ import { InitialDomainMapping } from '../types/common'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { FileToLoad } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; export interface SdkState { - /** - * @deprecated Use extract_from and extract_to from the event context instead, - * which are automatically resolved by the SDK from extraction_start_time and extraction_end_time. - */ - lastSyncStarted?: string; - /** - * @deprecated Use extract_from and extract_to from the event context instead, - * which are automatically resolved by the SDK from extraction_start_time and extraction_end_time. - */ - lastSuccessfulSyncStarted?: string; /** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp). * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */ pendingWorkersOldest?: string; @@ -30,10 +20,14 @@ export interface SdkState { } /** - * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. - * It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, snapInVersionId and attachmentsMetadata. + * AdapterStateEnvelope is the v2 on-disk state shape: connector state and SDK + * bookkeeping are stored as disjoint sub-objects so SDK internals stay + * encapsulated and never collide with connector keys. */ -export type AdapterState = ConnectorState & SdkState; +export interface AdapterStateEnvelope { + connectorState: ConnectorState; + sdkState: SdkState; +} export interface ToDevRev { attachmentsMetadata: { @@ -56,15 +50,13 @@ export interface FromDevRev { } export interface StateInterface { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; initialDomainMapping?: InitialDomainMapping; options?: WorkerAdapterOptions; } export const extractionSdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', pendingWorkersOldest: '', pendingWorkersNewest: '', workersOldest: '', @@ -85,3 +77,22 @@ export const loadingSdkState = { filesToLoad: [], }, }; + +/** + * The set of top-level state keys owned by the SDK. Derived from the initial + * SDK state constants so it auto-updates whenever a new SDK field is added. + * Used by the migration shim to split a flat v1 state blob into the + * `{ connectorState, sdkState }` envelope: keys in this set go to `sdkState`, + * everything else is connector state. + * + * `lastSyncStarted` / `lastSuccessfulSyncStarted` are retained here even though + * they are no longer fields on `SdkState`: a flat v1 blob may still carry them, + * and they must be recognized as SDK-owned so they are stripped out rather than + * leaking into connector state during migration. + */ +export const V1_SDK_STATE_KEYS: ReadonlySet = new Set([ + ...Object.keys(extractionSdkState), + ...Object.keys(loadingSdkState), + 'lastSyncStarted', + 'lastSuccessfulSyncStarted', +]); diff --git a/src/state/state.pending-boundaries.test.ts b/src/state/state.pending-boundaries.test.ts index 78c00858..01b36685 100644 --- a/src/state/state.pending-boundaries.test.ts +++ b/src/state/state.pending-boundaries.test.ts @@ -1,7 +1,7 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; /* eslint-disable @typescript-eslint/no-require-imports */ @@ -16,10 +16,10 @@ describe('State — pending extraction boundaries', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( - require('../common/install-initial-domain-mapping'), + require('./install-initial-domain-mapping'), 'installInitialDomainMapping' ); jest.spyOn(process, 'exit').mockImplementation(() => { @@ -61,15 +61,17 @@ describe('State — pending extraction boundaries', () => { postStateSpy.mockResolvedValue({ success: true }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Assert - expect(state.state.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z'); - expect(state.state.pendingWorkersNewest).toBe(FIXED_NOW); + expect(state.sdkState.pendingWorkersOldest).toBe( + '1970-01-01T00:00:00.000Z' + ); + expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW); expect(event.payload.event_context.extract_from).toBe( '1970-01-01T00:00:00.000Z' ); @@ -106,16 +108,18 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Assert: pending values are overwritten with fresh resolution, not stale values - expect(state.state.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z'); - expect(state.state.pendingWorkersNewest).toBe(FIXED_NOW); - expect(state.state.pendingWorkersNewest).not.toBe(staleNewest); + expect(state.sdkState.pendingWorkersOldest).toBe( + '1970-01-01T00:00:00.000Z' + ); + expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW); + expect(state.sdkState.pendingWorkersNewest).not.toBe(staleNewest); }); it('should reuse pending values from state on ContinueExtractingData instead of re-resolving', async () => { @@ -149,7 +153,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -159,8 +163,8 @@ describe('State — pending extraction boundaries', () => { expect(event.payload.event_context.extract_from).toBe(pendingOldest); expect(event.payload.event_context.extract_to).toBe(pendingNewest); // Pending values in state remain unchanged - expect(state.state.pendingWorkersOldest).toBe(pendingOldest); - expect(state.state.pendingWorkersNewest).toBe(pendingNewest); + expect(state.sdkState.pendingWorkersOldest).toBe(pendingOldest); + expect(state.sdkState.pendingWorkersNewest).toBe(pendingNewest); }); it('should not set extract_from/extract_to on ContinueExtractingData if no pending values exist', async () => { @@ -178,7 +182,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -209,7 +213,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.post-state.test.ts b/src/state/state.post-state.test.ts index 83ff39e0..b8cac279 100644 --- a/src/state/state.post-state.test.ts +++ b/src/state/state.post-state.test.ts @@ -1,11 +1,11 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; /* eslint-disable @typescript-eslint/no-require-imports */ -describe('State.postState', () => { +describe('ExtractionState.postState', () => { let postStateSpy: jest.SpyInstance; let fetchStateSpy: jest.SpyInstance; let processExitSpy: jest.SpyInstance; @@ -14,8 +14,8 @@ describe('State.postState', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -37,7 +37,7 @@ describe('State.postState', () => { postStateSpy.mockRestore(); - const adapterState = await createAdapterState({ + const adapterState = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -51,11 +51,14 @@ describe('State.postState', () => { expect(requests).toHaveLength(1); const body = requests[0].body as { state: string }; - // Body must contain the stringified state, preserving the original fields + // Body must contain the stringified v2 envelope, with the posted state + // preserved under connectorState. expect(typeof body.state).toBe('string'); - const parsed = JSON.parse(body.state) as Record; - expect(parsed.foo).toBe('bar'); - expect(parsed.snapInVersionId).toBe('1.0.0'); + const parsed = JSON.parse(body.state) as { + connectorState: Record; + }; + expect(parsed.connectorState.foo).toBe('bar'); + expect(parsed.connectorState.snapInVersionId).toBe('1.0.0'); }); it('should exit(1) when postState HTTP request fails', async () => { @@ -70,14 +73,14 @@ describe('State.postState', () => { postStateSpy.mockRestore(); - const adapterState = await createAdapterState({ + const adapterState = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Mock axiosClient.post directly to bypass the retry backoff - const axiosClientModule = require('../http/axios-client-internal'); + const axiosClientModule = require('../http/client'); const axiosPostSpy = jest .spyOn(axiosClientModule.axiosClient, 'post') .mockRejectedValue(new Error('network error')); diff --git a/src/state/state.test.ts b/src/state/state.test.ts index e8a2e5ce..e9223da2 100644 --- a/src/state/state.test.ts +++ b/src/state/state.test.ts @@ -3,14 +3,14 @@ import { STATELESS_EVENT_TYPES, } from '../common/constants'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; import { extractionSdkState } from './state.interfaces'; /* eslint-disable @typescript-eslint/no-require-imports */ -describe(State.name, () => { +describe(ExtractionState.name, () => { let initSpy: jest.SpyInstance; let postStateSpy: jest.SpyInstance; let fetchStateSpy: jest.SpyInstance; @@ -21,11 +21,11 @@ describe(State.name, () => { jest.clearAllMocks(); jest.restoreAllMocks(); - initSpy = jest.spyOn(State.prototype, 'init'); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + initSpy = jest.spyOn(ExtractionState.prototype, 'init'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( - require('../common/install-initial-domain-mapping'), + require('./install-initial-domain-mapping'), 'installInitialDomainMapping' ); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { @@ -42,7 +42,7 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -70,7 +70,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -91,7 +91,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -112,7 +112,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -151,59 +151,20 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + const result = await createExtractionState({ event, initialState, initialDomainMapping: {}, }); - const expectedState = { - ...initialState, - ...extractionSdkState, - }; - expect(postStateSpy).toHaveBeenCalledWith(expectedState); + // Assert: on 404 the SDK persists the full adapter state — the initial + // connector state and the seeded SDK state — via postState. + expect(postStateSpy).toHaveBeenCalled(); + expect(result.state).toEqual(initialState); + expect(result.sdkState).toEqual(extractionSdkState); } ); - it(EventType.StartExtractingData, async () => { - // Arrange - const initialState = { - test: 'test', - }; - const event = createMockEvent(mockServer.baseUrl, { - context: { - snap_in_version_id: '', - }, - payload: { event_type: EventType.StartExtractingData }, - }); - fetchStateSpy.mockRejectedValue({ - isAxiosError: true, - response: { status: 404 }, - }); - installInitialDomainMappingSpy.mockResolvedValue({ - success: true, - }); - postStateSpy.mockResolvedValue({ - success: true, - }); - - // Act - await createAdapterState({ - event, - initialState, - initialDomainMapping: {}, - }); - - // Assert - // Verify that post state is called with object that contains - // lastSyncStarted which is not empty string - expect(postStateSpy).toHaveBeenCalledWith( - expect.objectContaining({ - lastSyncStarted: expect.not.stringMatching(/^$/), - }) - ); - }); - it.each(STATEFUL_EVENT_TYPES)( 'should exit the process if initialDomainMapping is not provided for event type %s', async (eventType) => { @@ -220,7 +181,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: undefined, @@ -248,7 +209,7 @@ describe(State.name, () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act & Assert - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -280,7 +241,7 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -308,7 +269,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -337,7 +298,7 @@ describe(State.name, () => { postStateSpy.mockResolvedValue({ success: true }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -354,7 +315,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -376,7 +337,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.time-value-resolution.test.ts b/src/state/state.time-value-resolution.test.ts index a2265a58..9c4e8285 100644 --- a/src/state/state.time-value-resolution.test.ts +++ b/src/state/state.time-value-resolution.test.ts @@ -1,7 +1,8 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValue, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { BaseState } from './base-state'; +import { createExtractionState } from './extraction-state'; describe('State — TimeValue resolution', () => { let fetchStateSpy: jest.SpyInstance; @@ -11,7 +12,7 @@ describe('State — TimeValue resolution', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + fetchStateSpy = jest.spyOn(BaseState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -19,13 +20,14 @@ describe('State — TimeValue resolution', () => { describe('Enhanced Control Protocol - TimeValue resolution failures', () => { it('should exit the process if extraction_start_time resolution fails', async () => { - // Arrange: WORKERS_NEWEST type but state has no workersNewest + // Arrange: ABSOLUTE_TIME with an invalid ISO timestamp (still rejected in v2) const event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata, event_context: { extraction_start_time: { - type: TimeValueType.WORKERS_NEWEST, + type: TimeValueType.ABSOLUTE_TIME, + value: 'not-a-date', }, }, }, @@ -40,7 +42,7 @@ describe('State — TimeValue resolution', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -50,7 +52,7 @@ describe('State — TimeValue resolution', () => { }); it('should exit the process if extraction_end_time resolution fails', async () => { - // Arrange: WORKERS_NEWEST type but state has no workersNewest + // Arrange: ABSOLUTE_TIME end_time with an invalid ISO timestamp (still rejected in v2) const event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata, @@ -59,7 +61,8 @@ describe('State — TimeValue resolution', () => { type: TimeValueType.UNBOUNDED, }, extraction_end_time: { - type: TimeValueType.WORKERS_NEWEST, + type: TimeValueType.ABSOLUTE_TIME, + value: 'not-a-date', }, }, }, @@ -74,7 +77,7 @@ describe('State — TimeValue resolution', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -109,7 +112,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -121,7 +124,9 @@ describe('State — TimeValue resolution', () => { expect(event.payload.event_context.extract_to).toBe( '2025-06-01T00:00:00.000Z' ); - expect(state.state.pendingWorkersNewest).toBe('2025-06-01T00:00:00.000Z'); + expect(state.sdkState.pendingWorkersNewest).toBe( + '2025-06-01T00:00:00.000Z' + ); }); it('should skip resolution when extraction_end_time has no type', async () => { @@ -148,7 +153,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -187,7 +192,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.ts b/src/state/state.ts index 6f13224c..7ba15891 100644 --- a/src/state/state.ts +++ b/src/state/state.ts @@ -1,342 +1,3 @@ -import axios from 'axios'; -import { parentPort } from 'node:worker_threads'; - -import { STATELESS_EVENT_TYPES } from '../common/constants'; -import { installInitialDomainMapping } from '../common/install-initial-domain-mapping'; -import { resolveTimeValue } from '../common/time-value-resolver'; -import { axiosClient } from '../http/axios-client-internal'; -import { getPrintableState, serializeError } from '../logger/logger'; -import { SyncMode } from '../types/common'; -import { EventType } from '../types/extraction'; -import { WorkerMessageSubject } from '../types/workers'; - -import { - AdapterState, - extractionSdkState, - loadingSdkState, - SdkState, - StateInterface, -} from './state.interfaces'; -import { ExtractionScope } from '../types/workers'; - -export async function createAdapterState({ - event, - initialState, - initialDomainMapping, - options, -}: StateInterface): Promise> { - // Deep clone the initial state to avoid mutating the original state - const deepCloneInitialState: ConnectorState = structuredClone(initialState); - - const as = new State({ - event, - initialState: deepCloneInitialState, - initialDomainMapping, - options, - }); - - if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { - await as.init(deepCloneInitialState); - - // Check if IDM needs to be updated - const snapInVersionId = event.context.snap_in_version_id; - const hasSnapInVersionInState = 'snapInVersionId' in as.state; - const shouldUpdateIDM = - !hasSnapInVersionInState || as.state.snapInVersionId !== snapInVersionId; - - if (!shouldUpdateIDM) { - console.log( - `Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.` - ); - } else { - try { - console.log( - `Snap-in version in state "${as.state.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` - ); - - if (initialDomainMapping) { - await installInitialDomainMapping(event, initialDomainMapping); - as.state.snapInVersionId = snapInVersionId; - } else { - throw new Error( - 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' - ); - } - } catch (error) { - const errorMessage = `Error while installing initial domain mapping. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - - // Set lastSyncStarted if the event type is StartExtractingData - if ( - event.payload.event_type === EventType.StartExtractingData && - !as.state.lastSyncStarted - ) { - as.state.lastSyncStarted = new Date().toISOString(); - console.log(`Setting lastSyncStarted to ${as.state.lastSyncStarted}.`); - } - - // Resolve extraction timestamps from TimeValue objects, or reuse pending values from a prior invocation. - // On StartExtractingMetadata: resolve fresh from TimeValue objects and store in pending state (always overwrite). - // On all other events: reuse the pending values cached during StartExtractingMetadata. - const eventContext = event.payload.event_context; - - if (event.payload.event_type === EventType.StartExtractingMetadata) { - const timeFields = [ - { - source: 'extraction_start_time', - target: 'extract_from', - pending: 'pendingWorkersOldest', - }, - { - source: 'extraction_end_time', - target: 'extract_to', - pending: 'pendingWorkersNewest', - }, - ] as const; - - for (const { source, target, pending } of timeFields) { - const timeValue = eventContext[source]; - if (timeValue && timeValue.type) { - try { - const resolved = resolveTimeValue(timeValue, as.state); - eventContext[target] = resolved; - as.state[pending] = resolved; - console.log( - `Resolved ${target} to ${resolved}. Stored in ${pending}.` - ); - } catch (error) { - const errorMessage = `Failed to resolve ${source}: ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - } else { - // Non-StartExtractingMetadata events: reuse pending values from state - if (as.state.pendingWorkersOldest) { - eventContext.extract_from = as.state.pendingWorkersOldest; - console.log( - `Reusing pendingWorkersOldest as extract_from: ${as.state.pendingWorkersOldest}.` - ); - } else { - console.log( - 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' - ); - } - if (as.state.pendingWorkersNewest) { - eventContext.extract_to = as.state.pendingWorkersNewest; - console.log( - `Reusing pendingWorkersNewest as extract_to: ${as.state.pendingWorkersNewest}.` - ); - } else { - console.log( - 'pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.' - ); - } - } - - // Validate that extract_from is before extract_to - if (eventContext.extract_from && eventContext.extract_to) { - if (eventContext.extract_from >= eventContext.extract_to) { - const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - - return as; -} - -export class State { - private _state: AdapterState; - private _extractionScope: ExtractionScope = {}; - private initialSdkState: SdkState; - private workerUrl: string; - private devrevToken: string; - private syncUnitId: string; - private requestId: string; - - constructor({ event, initialState }: StateInterface) { - this.initialSdkState = - event.payload.event_context.mode === SyncMode.LOADING - ? loadingSdkState - : extractionSdkState; - this._state = { - ...initialState, - ...this.initialSdkState, - } as AdapterState; - this.workerUrl = event.payload.event_context.worker_data_url; - this.devrevToken = event.context.secrets.service_account_token; - this.syncUnitId = event.payload.event_context.sync_unit_id; - this.requestId = event.payload.event_context.request_id_adaas; - } - - get state(): AdapterState { - return this._state; - } - - set state(value: AdapterState) { - this._state = value; - } - - get extractionScope(): ExtractionScope { - return this._extractionScope; - } - - /** - * Initializes the state for this adapter instance by fetching from API - * or creating an initial state if none exists (404). - * @param initialState The initial connector state provided by the spawn function - */ - async init(initialState: ConnectorState): Promise { - try { - const { state: stringifiedState, objects } = await this.fetchState(); - if (!stringifiedState) { - throw new Error('No state found in response.'); - } - - let parsedState: AdapterState; - try { - parsedState = JSON.parse(stringifiedState); - } catch (error) { - throw new Error(`Failed to parse state. ${error}`); - } - - this.state = parsedState; - console.log( - 'State fetched successfully. Current state', - getPrintableState(this.state) - ); - - if (objects) { - try { - this._extractionScope = JSON.parse(objects); - } catch (error) { - console.warn(`Failed to parse extractionScope. ${error}`); - } - } - } catch (error) { - if (axios.isAxiosError(error) && error.response?.status === 404) { - console.log('State not found. Initializing state with initial state.'); - const initialAdapterState: AdapterState = { - ...initialState, - ...this.initialSdkState, - }; - - this.state = initialAdapterState; - await this.postState(initialAdapterState); - } else { - const errorMessage = `Failed to init state. ${serializeError(error)}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - - /** - * Updates the state of the adapter by posting to API. - * @param {object} state - The state to be updated - */ - async postState(state?: AdapterState) { - const url = this.workerUrl + '.update'; - this.state = state || this.state; - - let stringifiedState: string; - try { - stringifiedState = JSON.stringify(this.state); - } catch (error) { - const errorMessage = `Failed to stringify state. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - - try { - await axiosClient.post( - url, - { - state: stringifiedState, - }, - { - headers: { - Authorization: this.devrevToken, - }, - params: { - sync_unit: this.syncUnitId, - request_id: this.requestId, - }, - } - ); - - console.log( - 'State updated successfully to', - getPrintableState(this.state) - ); - } catch (error) { - const errorMessage = `Failed to update the state. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - - /** - * Fetches the state of the adapter from API. - * @return The raw state data from API - */ - async fetchState(): Promise<{ state: string; objects?: string }> { - console.log( - `Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.` - ); - - const url = this.workerUrl + '.get'; - const response = await axiosClient.get(url, { - headers: { - Authorization: this.devrevToken, - }, - params: { - sync_unit: this.syncUnitId, - request_id: this.requestId, - }, - }); - - return { - state: response.data?.state, - objects: response.data?.objects, - }; - } -} +export { BaseState } from './base-state'; +export { ExtractionState, createExtractionState } from './extraction-state'; +export { LoadingState, createLoadingState } from './loading-state'; diff --git a/src/common/time-value-resolver.test.ts b/src/state/time-value-resolver.test.ts similarity index 94% rename from src/common/time-value-resolver.test.ts rename to src/state/time-value-resolver.test.ts index f84a3dab..6cc82a98 100644 --- a/src/common/time-value-resolver.test.ts +++ b/src/state/time-value-resolver.test.ts @@ -1,6 +1,6 @@ import { TimeValueType } from '../types/extraction'; import { SdkState } from '../state/state.interfaces'; -import { UNBOUNDED_DATE_TIME_VALUE } from './constants'; +import { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; import { parseDuration, applyDuration, @@ -154,8 +154,6 @@ describe('time-value-resolver', () => { describe('resolveTimeValue', () => { const baseState: SdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', workersOldest: '2024-01-01T00:00:00.000Z', workersNewest: '2024-06-01T00:00:00.000Z', }; @@ -255,13 +253,12 @@ describe('time-value-resolver', () => { expect(result).toBe('2024-06-01T00:00:00.000Z'); }); - it('should throw if workersNewest is not set', () => { - expect(() => - resolveTimeValue( - { type: TimeValueType.WORKERS_NEWEST }, - { workersNewest: '' } - ) - ).toThrow('workersNewest is not set in state'); + it('should return UNBOUNDED_DATE_TIME_VALUE if workersNewest is not set', () => { + const result = resolveTimeValue( + { type: TimeValueType.WORKERS_NEWEST }, + { workersNewest: '' } + ); + expect(result).toBe(UNBOUNDED_DATE_TIME_VALUE); }); }); @@ -332,16 +329,15 @@ describe('time-value-resolver', () => { expect(result).toBe('2024-06-01T00:30:00.000Z'); }); - it('should throw if workersNewest is not set', () => { - expect(() => - resolveTimeValue( - { - type: TimeValueType.WORKERS_NEWEST_PLUS_WINDOW, - value: '2h', - }, - { workersNewest: '' } - ) - ).toThrow('workersNewest is not set in state'); + it('should return UNBOUNDED_DATE_TIME_VALUE if workersNewest is not set', () => { + const result = resolveTimeValue( + { + type: TimeValueType.WORKERS_NEWEST_PLUS_WINDOW, + value: '30m', + }, + { workersNewest: '' } + ); + expect(result).toBe(UNBOUNDED_DATE_TIME_VALUE); }); it('should throw if value (duration) is missing', () => { @@ -367,8 +363,6 @@ describe('time-value-resolver', () => { const FIXED_NOW = '2026-02-26T15:30:00.000Z'; const scenarioState: SdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', workersOldest: '2024-01-01T00:00:00.000Z', workersNewest: '2024-06-01T00:00:00.000Z', }; diff --git a/src/common/time-value-resolver.ts b/src/state/time-value-resolver.ts similarity index 85% rename from src/common/time-value-resolver.ts rename to src/state/time-value-resolver.ts index ce2494d8..a10721c3 100644 --- a/src/common/time-value-resolver.ts +++ b/src/state/time-value-resolver.ts @@ -1,6 +1,6 @@ import { TimeUnit, TimeValue, TimeValueType } from '../types/extraction'; import { SdkState } from '../state/state.interfaces'; -import { UNBOUNDED_DATE_TIME_VALUE } from './constants'; +import { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; /** * Parses a shorthand duration string into its numeric value and unit. @@ -83,16 +83,15 @@ export function applyDuration( * - ABSOLUTE: Returns the value directly (must be an ISO 8601 timestamp) * - NOW: Returns the current time as ISO 8601 * - UNBOUNDED: Returns UNBOUNDED_DATE_TIME_VALUE ('1970-01-01T00:00:00.000Z') - * - WORKERS_OLDEST: Returns workersOldest from state, or throws if not set - * - WORKERS_NEWEST: Returns workersNewest from state, or throws if not set - * - WORKERS_OLDEST_MINUS_WINDOW: Subtracts duration from workersOldest, or throws if not set - * - WORKERS_NEWEST_PLUS_WINDOW: Adds duration to workersNewest, or throws if not set + * - WORKERS_OLDEST: Returns workersOldest from state, or UNBOUNDED if not set + * - WORKERS_NEWEST: Returns workersNewest from state, or UNBOUNDED if not set + * - WORKERS_OLDEST_MINUS_WINDOW: Subtracts duration from workersOldest, or UNBOUNDED if not set + * - WORKERS_NEWEST_PLUS_WINDOW: Adds duration to workersNewest, or UNBOUNDED if not set * * @param timeValue - The TimeValue to resolve * @param state - The current SDK state containing workersOldest and workersNewest * @returns Resolved ISO 8601 timestamp string * @throws Error if required TimeValue.value is missing for ABSOLUTE or *_WINDOW types - * @throws Error if workersOldest/workersNewest is not set in state for WORKERS_* types */ export function resolveTimeValue( timeValue: TimeValue, @@ -135,13 +134,7 @@ export function resolveTimeValue( case TimeValueType.WORKERS_NEWEST: { if (!state.workersNewest) { // To support backwards-compatibility for the old state - if (state.lastSuccessfulSyncStarted) { - return state.lastSuccessfulSyncStarted; - } - - throw new Error( - 'Field workersNewest is not set in state. Cannot resolve TimeValue of type WORKERS_NEWEST without a prior extraction boundary.' - ); + return UNBOUNDED_DATE_TIME_VALUE; } return state.workersNewest; } @@ -167,13 +160,7 @@ export function resolveTimeValue( } if (!state.workersNewest) { // To support backwards-compatibility for the old state - if (state.lastSuccessfulSyncStarted) { - return state.lastSuccessfulSyncStarted; - } - - throw new Error( - 'Field workersNewest is not set in state. Cannot resolve TimeValue of type WORKERS_NEWEST_PLUS_WINDOW without a prior extraction boundary.' - ); + return UNBOUNDED_DATE_TIME_VALUE; } return applyDuration(state.workersNewest, timeValue.value, 'add'); } diff --git a/src/mock-server/README.md b/src/testing/README.md similarity index 100% rename from src/mock-server/README.md rename to src/testing/README.md diff --git a/src/common/test-utils.ts b/src/testing/mock-event.ts similarity index 88% rename from src/common/test-utils.ts rename to src/testing/mock-event.ts index 1584a345..e503bf65 100644 --- a/src/common/test-utils.ts +++ b/src/testing/mock-event.ts @@ -1,6 +1,6 @@ -import { AirdropEvent, EventType } from '../types/extraction'; +import { AirSyncEvent, EventType } from '../types/extraction'; -export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; +import { MOCK_SERVER_DEFAULT_URL } from './mock-server'; /** * Recursively makes all properties of T optional. @@ -44,25 +44,29 @@ function deepMerge>( } /** - * Creates a mock AirdropEvent for testing. + * Creates a mock AirSyncEvent for testing. * * @param mockServerUrl - Base URL for the mock server. Defaults to {@link MOCK_SERVER_DEFAULT_URL}. * The `callback_url`, `worker_data_url`, and `devrev_endpoint` fields are * derived from this value unless explicitly overridden. - * @param overrides - Deep partial of AirdropEvent. Any provided fields are + * @param overrides - Deep partial of AirSyncEvent. Any provided fields are * deep-merged on top of the defaults. */ export function createMockEvent( mockServerUrl: string = MOCK_SERVER_DEFAULT_URL, - overrides: DeepPartial = {} -): AirdropEvent { - const base: AirdropEvent = { + overrides: DeepPartial = {} +): AirSyncEvent { + const base: AirSyncEvent = { context: { secrets: { service_account_token: 'test_token', }, snap_in_version_id: 'test_snap_in_version_id', snap_in_id: 'test_snap_in_id', + user_id: 'test_user_id', + dev_oid: 'test_dev_oid', + source_id: 'test_source_id', + service_account_id: 'test_service_account_id', }, payload: { connection_data: { @@ -118,7 +122,7 @@ export function createMockEvent( const merged = deepMerge( base as unknown as Record, overrides as DeepPartial> - ) as unknown as AirdropEvent; + ) as unknown as AirSyncEvent; // Ensure mock server URLs always win over overrides, unless the caller // explicitly provided them. diff --git a/src/mock-server/mock-server.interfaces.ts b/src/testing/mock-server.interfaces.ts similarity index 100% rename from src/mock-server/mock-server.interfaces.ts rename to src/testing/mock-server.interfaces.ts diff --git a/src/mock-server/mock-server.ts b/src/testing/mock-server.ts similarity index 97% rename from src/mock-server/mock-server.ts rename to src/testing/mock-server.ts index 7c196996..eaa45dad 100644 --- a/src/mock-server/mock-server.ts +++ b/src/testing/mock-server.ts @@ -10,6 +10,12 @@ import { RouteHandlers, } from './mock-server.interfaces'; +/** + * Default base URL for the mock server. The port `0` lets the OS assign a free + * port at listen time; tests read the resolved URL from `mockServer.baseUrl`. + */ +export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; + const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10mb /** diff --git a/src/tests/control-protocol-reconciliation.test.ts b/src/tests/control-protocol-reconciliation.test.ts index 26752f2f..9cee1039 100644 --- a/src/tests/control-protocol-reconciliation.test.ts +++ b/src/tests/control-protocol-reconciliation.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; describe('Enhanced Control Protocol', () => { diff --git a/src/tests/dummy-connector/data-extraction.test.ts b/src/tests/dummy-connector/data-extraction.test.ts index 82b812a1..89a48ad6 100644 --- a/src/tests/dummy-connector/data-extraction.test.ts +++ b/src/tests/dummy-connector/data-extraction.test.ts @@ -1,15 +1,15 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; describe('Dummy Connector - Data Extraction', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/dummy-connector/data-extraction.ts b/src/tests/dummy-connector/data-extraction.ts index 8f1f7a6f..57311cbe 100644 --- a/src/tests/dummy-connector/data-extraction.ts +++ b/src/tests/dummy-connector/data-extraction.ts @@ -1,7 +1,6 @@ import { - ExtractorEventType, NormalizedItem, - processTask, + processExtractionTask, RepoInterface, } from '../../index'; @@ -25,7 +24,7 @@ const repos: RepoInterface[] = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -44,9 +43,10 @@ processTask({ } await adapter.getRepo('tasks')?.push(tasks); - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/dummy-connector/external-sync-units-extraction.test.ts b/src/tests/dummy-connector/external-sync-units-extraction.test.ts index d0f87c50..a3aeaf14 100644 --- a/src/tests/dummy-connector/external-sync-units-extraction.test.ts +++ b/src/tests/dummy-connector/external-sync-units-extraction.test.ts @@ -1,6 +1,6 @@ import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/dummy-connector/external-sync-units-extraction.ts b/src/tests/dummy-connector/external-sync-units-extraction.ts index 0cb6d2f6..c091f1a1 100644 --- a/src/tests/dummy-connector/external-sync-units-extraction.ts +++ b/src/tests/dummy-connector/external-sync-units-extraction.ts @@ -1,6 +1,10 @@ -import { ExternalSyncUnit, ExtractorEventType, processTask } from '../../index'; +import { + AirSyncDefaultItemTypes, + ExternalSyncUnit, + processExtractionTask, +} from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { const dummyExternalSyncUnits: ExternalSyncUnit[] = [ { @@ -12,15 +16,26 @@ processTask({ }, ]; - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: dummyExternalSyncUnits, - }); + adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, + ]); + + await adapter + .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) + ?.push(dummyExternalSyncUnits); + + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to extract external sync units. Lambda timeout.', }, - }); + }; }, }); diff --git a/src/tests/dummy-connector/extraction.ts b/src/tests/dummy-connector/extraction.ts index 4889fb6a..5ff952b8 100644 --- a/src/tests/dummy-connector/extraction.ts +++ b/src/tests/dummy-connector/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,15 +7,18 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/dummy-connector/metadata-extraction.test.ts b/src/tests/dummy-connector/metadata-extraction.test.ts index 18ac043a..6a35d3b1 100644 --- a/src/tests/dummy-connector/metadata-extraction.test.ts +++ b/src/tests/dummy-connector/metadata-extraction.test.ts @@ -1,17 +1,17 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; jest.setTimeout(60000); describe('Dummy Connector - Metadata Extraction', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata }, diff --git a/src/tests/dummy-connector/metadata-extraction.ts b/src/tests/dummy-connector/metadata-extraction.ts index 3427a559..89b383cf 100644 --- a/src/tests/dummy-connector/metadata-extraction.ts +++ b/src/tests/dummy-connector/metadata-extraction.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; const repos = [ { @@ -6,7 +6,7 @@ const repos = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -16,11 +16,13 @@ processTask({ .getRepo('external_domain_metadata') ?.push([externalDomainMetadata]); - await adapter.emit(ExtractorEventType.MetadataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.MetadataExtractionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to extract metadata. Lambda timeout.' }, - }); + }; }, }); diff --git a/src/tests/event-data-size-limit/extraction.ts b/src/tests/event-data-size-limit/extraction.ts index 9fa4a366..1c082448 100644 --- a/src/tests/event-data-size-limit/extraction.ts +++ b/src/tests/event-data-size-limit/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -13,16 +13,19 @@ const initialDomainMapping = {}; * With 3000 items and batch size 1, we get 3000 artifacts. * Each artifact metadata is ~55 bytes, so 3000 * 55 = 165KB > 160KB threshold. */ -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { batchSize: 1, // Batch size of 1 to generate many artifacts isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/event-data-size-limit/size-limit-1.test.ts b/src/tests/event-data-size-limit/size-limit-1.test.ts index 72850e8e..0b1ea7aa 100644 --- a/src/tests/event-data-size-limit/size-limit-1.test.ts +++ b/src/tests/event-data-size-limit/size-limit-1.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; // Increase timeout for this test since we're doing many uploads diff --git a/src/tests/event-data-size-limit/size-limit-1.ts b/src/tests/event-data-size-limit/size-limit-1.ts index aff16624..54f7aa90 100644 --- a/src/tests/event-data-size-limit/size-limit-1.ts +++ b/src/tests/event-data-size-limit/size-limit-1.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; /** * Test worker that generates items to trigger the SQS size limit. @@ -8,7 +8,7 @@ import { ExtractorEventType, processTask } from '../../index'; * Each artifact metadata is ~55 bytes (id, item_type, item_count). * We need ~2857 artifacts to reach 160KB, so generating 3000 items. */ -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // Using external_domain_metadata itemType which doesn't require normalize adapter.initializeRepos([ @@ -20,10 +20,10 @@ processTask({ const repo = adapter.getRepo('external_domain_metadata'); if (!repo) { console.error('Repo not found after init'); - await adapter.emit(ExtractorEventType.DataExtractionError, { + return { + status: 'error', error: { message: 'Repo not found after init!' }, - }); - return; + }; } // Generate 3000 items with batch size 1 = 3000 artifacts @@ -41,15 +41,16 @@ processTask({ ]); if (adapter.isTimeout) { - return; + return { status: 'progress' }; } } console.log('Size limit was NOT triggered, emitting done'); - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { console.log('onTimeout called - emitting progress'); - await adapter.emit(ExtractorEventType.DataExtractionProgress); + return { status: 'progress' }; }, }); diff --git a/src/tests/extract-from-collision.test.ts b/src/tests/extract-from-collision.test.ts index a5e8dbb5..b6106648 100644 --- a/src/tests/extract-from-collision.test.ts +++ b/src/tests/extract-from-collision.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventContext, EventType, TimeValueType } from '../types/extraction'; /** diff --git a/src/tests/jest.setup.ts b/src/tests/jest.setup.ts index 87c86273..8887f19b 100644 --- a/src/tests/jest.setup.ts +++ b/src/tests/jest.setup.ts @@ -1,4 +1,4 @@ -import { MockServer } from '../mock-server/mock-server'; +import { MockServer } from '../testing/mock-server'; // Use port 0 for dynamic port allocation, enabling parallel test execution export const mockServer = new MockServer(0); diff --git a/src/tests/spawn-worker/delete-event-type.test.ts b/src/tests/spawn-worker/delete-event-type.test.ts index b2b2e445..c65f6c77 100644 --- a/src/tests/spawn-worker/delete-event-type.test.ts +++ b/src/tests/spawn-worker/delete-event-type.test.ts @@ -1,6 +1,6 @@ import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/spawn-worker/extraction.ts b/src/tests/spawn-worker/extraction.ts index ed0023b7..8965667a 100644 --- a/src/tests/spawn-worker/extraction.ts +++ b/src/tests/spawn-worker/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,15 +7,18 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath?: string) => { +const run = async (events: AirSyncEvent[], workerPath?: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/spawn-worker/some-cleanup-worker.ts b/src/tests/spawn-worker/some-cleanup-worker.ts index 268c5f07..52934d85 100644 --- a/src/tests/spawn-worker/some-cleanup-worker.ts +++ b/src/tests/spawn-worker/some-cleanup-worker.ts @@ -1,15 +1,18 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { console.log('Some cleanup logic executed.'); - await adapter.emit(ExtractorEventType.ExtractorStateDeletionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExtractorStateDeletionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to execute cleanup logic. Lambda timeout.', }, - }); + }; }, }); diff --git a/src/tests/spawn-worker/unknown-event-type.test.ts b/src/tests/spawn-worker/unknown-event-type.test.ts index 00ca5f0a..58f5204d 100644 --- a/src/tests/spawn-worker/unknown-event-type.test.ts +++ b/src/tests/spawn-worker/unknown-event-type.test.ts @@ -1,6 +1,7 @@ -import { EventType, ExtractorEventType } from '../../types/extraction'; +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; +import { EventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; @@ -18,7 +19,7 @@ describe('Unknown event type', () => { expect(lastRequest?.url).toContain('/callback_url'); expect(lastRequest?.method).toBe('POST'); expect((lastRequest?.body as { event_type: string }).event_type).toBe( - ExtractorEventType.UnknownEventType + UNKNOWN_EVENT_TYPE ); }); }); diff --git a/src/tests/spawn-worker/unknown-event-type.ts b/src/tests/spawn-worker/unknown-event-type.ts index 05284e4e..02c8e7cc 100644 --- a/src/tests/spawn-worker/unknown-event-type.ts +++ b/src/tests/spawn-worker/unknown-event-type.ts @@ -1,12 +1,14 @@ -import { processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { console.log('task should not be called.'); + return { status: 'success' }; }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - onTimeout: async ({ adapter }) => { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { console.log('onTimeout should not be called.'); + return { status: 'success' }; }, }); diff --git a/src/tests/test-helpers.interfaces.ts b/src/tests/test-helpers.interfaces.ts index 085d7c3a..e782ab7a 100644 --- a/src/tests/test-helpers.interfaces.ts +++ b/src/tests/test-helpers.interfaces.ts @@ -1,11 +1,11 @@ -import { DeepPartial } from '../common/test-utils'; -import { AirdropEvent } from '../types/extraction'; +import { DeepPartial } from '../testing/mock-event'; +import { AirSyncEvent } from '../types/extraction'; /** * Internal variant of the createMockEvent overrides — a deep partial of - * {@link AirdropEvent}. The shared test wrapper injects defaults automatically. + * {@link AirSyncEvent}. The shared test wrapper injects defaults automatically. */ -export type CreateMockEventOverrides = DeepPartial; +export type CreateMockEventOverrides = DeepPartial; /** * Options for creating a file stream response. diff --git a/src/tests/timeout-handling/extraction.ts b/src/tests/timeout-handling/extraction.ts index f24232aa..bc9d00d6 100644 --- a/src/tests/timeout-handling/extraction.ts +++ b/src/tests/timeout-handling/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,17 +7,20 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { batchSize: 1000, timeout: 5 * 1000, // 5 seconds isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/timeout-handling/no-timeout.test.ts b/src/tests/timeout-handling/no-timeout.test.ts index 7b9df7dc..06f60cad 100644 --- a/src/tests/timeout-handling/no-timeout.test.ts +++ b/src/tests/timeout-handling/no-timeout.test.ts @@ -1,16 +1,16 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; describe('No timeout', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/no-timeout.ts b/src/tests/timeout-handling/no-timeout.ts index 8eb47e39..2a87475b 100644 --- a/src/tests/timeout-handling/no-timeout.ts +++ b/src/tests/timeout-handling/no-timeout.ts @@ -1,14 +1,16 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { for (let i = 0; i < 10; i++) { console.log('no-timeout iteration', i); } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-blocked.test.ts b/src/tests/timeout-handling/timeout-blocked.test.ts index ec66e516..72faab8c 100644 --- a/src/tests/timeout-handling/timeout-blocked.test.ts +++ b/src/tests/timeout-handling/timeout-blocked.test.ts @@ -1,17 +1,17 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; jest.setTimeout(10000); describe('Timeout blocked', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-blocked.ts b/src/tests/timeout-handling/timeout-blocked.ts index f7c4ee6d..55f9b99e 100644 --- a/src/tests/timeout-handling/timeout-blocked.ts +++ b/src/tests/timeout-handling/timeout-blocked.ts @@ -1,7 +1,8 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { // Simple CPU-intensive nested loops that block the event loop let result = 0; for (let i = 0; i < 100000; i++) { @@ -16,9 +17,10 @@ processTask({ } } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-graceful.test.ts b/src/tests/timeout-handling/timeout-graceful.test.ts index f37a5062..12c6caa7 100644 --- a/src/tests/timeout-handling/timeout-graceful.test.ts +++ b/src/tests/timeout-handling/timeout-graceful.test.ts @@ -1,17 +1,17 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; jest.setTimeout(10000); describe('Timeout graceful', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-graceful.ts b/src/tests/timeout-handling/timeout-graceful.ts index 56ccb396..5293aaef 100644 --- a/src/tests/timeout-handling/timeout-graceful.ts +++ b/src/tests/timeout-handling/timeout-graceful.ts @@ -1,21 +1,22 @@ import { sleep } from '../../common/helpers'; -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // Use async delays that allow the event loop to process timeout messages for (let i = 0; i < 10; i++) { if (adapter.isTimeout) { - return; + return { status: 'progress' }; } console.log('timeout-graceful iteration', i); await sleep(1000); } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-unblocked.test.ts b/src/tests/timeout-handling/timeout-unblocked.test.ts index 8f4fe8b5..0924c6ea 100644 --- a/src/tests/timeout-handling/timeout-unblocked.test.ts +++ b/src/tests/timeout-handling/timeout-unblocked.test.ts @@ -1,17 +1,17 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; jest.setTimeout(10000); // 10 seconds describe('Timeout unblocked', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-unblocked.ts b/src/tests/timeout-handling/timeout-unblocked.ts index 0ac268c1..d4132ddd 100644 --- a/src/tests/timeout-handling/timeout-unblocked.ts +++ b/src/tests/timeout-handling/timeout-unblocked.ts @@ -1,7 +1,7 @@ import { sleep } from '../../common/helpers'; -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // CPU-intensive nested loops that yield control after logging // This allows the event loop to process timeout messages @@ -9,7 +9,7 @@ processTask({ for (let i = 0; i < 100000; i++) { for (let j = 0; j < 10000; j++) { if (adapter.isTimeout) { - return; + return { status: 'progress' }; } result += Math.sqrt(i * j) * Math.sin(i + j); @@ -23,9 +23,10 @@ processTask({ } } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/types/common.ts b/src/types/common.ts index f0428be9..557f8da7 100644 --- a/src/types/common.ts +++ b/src/types/common.ts @@ -1,15 +1,3 @@ -import { Artifact } from '../uploader/uploader.interfaces'; - -/** - * ErrorLevel is an enum that represents the level of an error. - * @deprecated - */ -export enum ErrorLevel { - Warning = 'WARNING', - Error = 'ERROR', - Info = 'INFO', -} - /** * ErrorRecord is an interface that defines the structure of an error record. */ @@ -17,23 +5,6 @@ export interface ErrorRecord { message: string; } -/** - * LogRecord is an interface that defines the structure of a log record. - * @deprecated - */ -export interface LogRecord { - level: ErrorLevel; - message: string; -} - -/** - * AdapterUpdateParams is an interface that defines the structure of the parameters that can be passed to the update adapter. - * @deprecated - */ -export interface AdapterUpdateParams { - artifact?: Artifact; -} - /** * InitialDomainMapping is an interface that defines the structure of the initial domain mapping. */ diff --git a/src/common/errors.ts b/src/types/errors.ts similarity index 100% rename from src/common/errors.ts rename to src/types/errors.ts diff --git a/src/types/extraction.test.ts b/src/types/extraction.test.ts index 1dd9cb9e..5453ee81 100644 --- a/src/types/extraction.test.ts +++ b/src/types/extraction.test.ts @@ -1,4 +1,4 @@ -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; import { EventType, InitialSyncScope, TimeValueType } from './extraction'; diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 0c328de8..600437f9 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -4,62 +4,16 @@ import { Artifact } from '../uploader/uploader.interfaces'; import { ErrorRecord } from './common'; -import { AxiosResponse } from 'axios'; import { NormalizedAttachment } from '../repo/repo.interfaces'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; import { DonV2, LoaderReport, RateLimited } from './loading'; /** - * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS. + * EventType is an enum that defines the different types of events that can be sent to the external extractor from AirSync. * The external extractor can use these events to know what to do next in the extraction process. */ export enum EventType { - // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) - /** - * @deprecated Use StartExtractingExternalSyncUnits instead - */ - ExtractionExternalSyncUnitsStart = 'EXTRACTION_EXTERNAL_SYNC_UNITS_START', - /** - * @deprecated Use StartExtractingMetadata instead - */ - ExtractionMetadataStart = 'EXTRACTION_METADATA_START', - /** - * @deprecated Use StartExtractingData instead - */ - ExtractionDataStart = 'EXTRACTION_DATA_START', - /** - * @deprecated Use ContinueExtractingData instead - */ - ExtractionDataContinue = 'EXTRACTION_DATA_CONTINUE', - /** - * @deprecated Use StartDeletingExtractorState instead - */ - ExtractionDataDelete = 'EXTRACTION_DATA_DELETE', - /** - * @deprecated Use StartExtractingAttachments instead - */ - ExtractionAttachmentsStart = 'EXTRACTION_ATTACHMENTS_START', - /** - * @deprecated Use ContinueExtractingAttachments instead - */ - ExtractionAttachmentsContinue = 'EXTRACTION_ATTACHMENTS_CONTINUE', - /** - * @deprecated Use StartDeletingExtractorAttachmentsState instead - */ - ExtractionAttachmentsDelete = 'EXTRACTION_ATTACHMENTS_DELETE', - - // Loading - StartLoadingData = 'START_LOADING_DATA', - ContinueLoadingData = 'CONTINUE_LOADING_DATA', - StartLoadingAttachments = 'START_LOADING_ATTACHMENTS', - ContinueLoadingAttachments = 'CONTINUE_LOADING_ATTACHMENTS', - StartDeletingLoaderState = 'START_DELETING_LOADER_STATE', - StartDeletingLoaderAttachmentState = 'START_DELETING_LOADER_ATTACHMENT_STATE', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - - // Extraction - New member names with NEW values (preferred) + // Extraction StartExtractingExternalSyncUnits = 'START_EXTRACTING_EXTERNAL_SYNC_UNITS', StartExtractingMetadata = 'START_EXTRACTING_METADATA', StartExtractingData = 'START_EXTRACTING_DATA', @@ -68,83 +22,22 @@ export enum EventType { StartExtractingAttachments = 'START_EXTRACTING_ATTACHMENTS', ContinueExtractingAttachments = 'CONTINUE_EXTRACTING_ATTACHMENTS', StartDeletingExtractorAttachmentsState = 'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE', + + // Loading + StartLoadingData = 'START_LOADING_DATA', + ContinueLoadingData = 'CONTINUE_LOADING_DATA', + StartLoadingAttachments = 'START_LOADING_ATTACHMENTS', + ContinueLoadingAttachments = 'CONTINUE_LOADING_ATTACHMENTS', + StartDeletingLoaderState = 'START_DELETING_LOADER_STATE', + StartDeletingLoaderAttachmentState = 'START_DELETING_LOADER_ATTACHMENT_STATE', } /** - * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS. - * The external extractor can use these events to inform ADaaS about the progress of the extraction process. + * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to AirSync. + * The external extractor can use these events to inform AirSync about the progress of the extraction process. */ export enum ExtractorEventType { - // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) - /** - * @deprecated Use ExternalSyncUnitExtractionDone instead - */ - ExtractionExternalSyncUnitsDone = 'EXTRACTION_EXTERNAL_SYNC_UNITS_DONE', - /** - * @deprecated Use ExternalSyncUnitExtractionError instead - */ - ExtractionExternalSyncUnitsError = 'EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR', - /** - * @deprecated Use MetadataExtractionDone instead - */ - ExtractionMetadataDone = 'EXTRACTION_METADATA_DONE', - /** - * @deprecated Use MetadataExtractionError instead - */ - ExtractionMetadataError = 'EXTRACTION_METADATA_ERROR', - /** - * @deprecated Use DataExtractionProgress instead - */ - ExtractionDataProgress = 'EXTRACTION_DATA_PROGRESS', - /** - * @deprecated Use DataExtractionDelayed instead - */ - ExtractionDataDelay = 'EXTRACTION_DATA_DELAY', - /** - * @deprecated Use DataExtractionDone instead - */ - ExtractionDataDone = 'EXTRACTION_DATA_DONE', - /** - * @deprecated Use DataExtractionError instead - */ - ExtractionDataError = 'EXTRACTION_DATA_ERROR', - /** - * @deprecated Use ExtractorStateDeletionDone instead - */ - ExtractionDataDeleteDone = 'EXTRACTION_DATA_DELETE_DONE', - /** - * @deprecated Use ExtractorStateDeletionError instead - */ - ExtractionDataDeleteError = 'EXTRACTION_DATA_DELETE_ERROR', - /** - * @deprecated Use AttachmentExtractionProgress instead - */ - ExtractionAttachmentsProgress = 'EXTRACTION_ATTACHMENTS_PROGRESS', - /** - * @deprecated Use AttachmentExtractionDelayed instead - */ - ExtractionAttachmentsDelay = 'EXTRACTION_ATTACHMENTS_DELAY', - /** - * @deprecated Use AttachmentExtractionDone instead - */ - ExtractionAttachmentsDone = 'EXTRACTION_ATTACHMENTS_DONE', - /** - * @deprecated Use AttachmentExtractionError instead - */ - ExtractionAttachmentsError = 'EXTRACTION_ATTACHMENTS_ERROR', - /** - * @deprecated Use ExtractorAttachmentsStateDeletionDone instead - */ - ExtractionAttachmentsDeleteDone = 'EXTRACTION_ATTACHMENTS_DELETE_DONE', - /** - * @deprecated Use ExtractorAttachmentsStateDeletionError instead - */ - ExtractionAttachmentsDeleteError = 'EXTRACTION_ATTACHMENTS_DELETE_ERROR', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - - // Extraction - New member names with NEW values (preferred) + // Extraction ExternalSyncUnitExtractionDone = 'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE', ExternalSyncUnitExtractionError = 'EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR', MetadataExtractionDone = 'METADATA_EXTRACTION_DONE', @@ -163,16 +56,6 @@ export enum ExtractorEventType { ExtractorAttachmentsStateDeletionError = 'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR', } -/** - * @deprecated - * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor. - * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs. - */ -export enum ExtractionMode { - INITIAL = 'INITIAL', - INCREMENTAL = 'INCREMENTAL', -} - /** * ExternalSyncUnit is an interface that defines the structure of an external sync unit (repos, projects, ...) that can be extracted. * It must contain an ID, a name, and a description. It can also contain the number of items in the external sync unit. @@ -248,45 +131,7 @@ export interface TimeValue { } /** - * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from ADaaS. - * @deprecated - */ -export interface EventContextIn { - callback_url: string; - dev_org: string; - dev_org_id: string; - dev_user: string; - dev_user_id: string; - external_sync_unit: string; - external_sync_unit_id: string; - external_sync_unit_name: string; - external_system: string; - external_system_type: string; - import_slug: string; - mode: string; - request_id: string; - snap_in_slug: string; - sync_run: string; - sync_run_id: string; - sync_tier: string; - sync_unit: DonV2; - sync_unit_id: string; - uuid: string; - worker_data_url: string; -} - -/** - * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to ADaaS. - * @deprecated - */ -export interface EventContextOut { - uuid: string; - sync_run: string; - sync_unit?: string; -} - -/** - * EventContext is an interface that defines the structure of the event context that is sent to the external connector from Airdrop. + * EventContext is an interface that defines the structure of the event context that is sent to the external connector from AirSync. */ export interface EventContext { callback_url: string; @@ -378,7 +223,7 @@ export interface EventContext { } /** - * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from ADaaS. + * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from AirSync. * It contains the organization ID, organization name, key, and key type. */ export interface ConnectionData { @@ -389,23 +234,15 @@ export interface ConnectionData { } /** - * EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS. + * EventData is an interface that defines the structure of the event data that is sent from the external extractor to AirSync. */ export interface EventData { - /** - * @deprecated This field is deprecated and should not be used. External sync units should be pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo. - * - */ - external_sync_units?: ExternalSyncUnit[]; - /** - * @deprecated This field is deprecated and should not be used. Progress is - * now calculated on the backend. - */ - progress?: number; error?: ErrorRecord; delay?: number; /** - * @deprecated This field is deprecated and should not be used. + * Artifacts produced by the worker's repos, attached to the emitted event by + * the SDK. This includes external sync units, which are pushed to the + * AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo and uploaded as artifacts. */ artifacts?: Artifact[]; @@ -416,41 +253,33 @@ export interface EventData { } /** - * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS. + * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to AirSync. */ export interface WorkerMetadata { adaas_library_version: string; } /** - * DomainObject is an interface that defines the structure of a domain object that can be extracted. - * It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count. - * @deprecated - */ -export interface DomainObjectState { - name: string; - nextChunkId: number; - pages?: { - pages: number[]; - }; - lastModified: string; - isDone: boolean; - count: number; -} - -/** - * AirdropEvent is an interface that defines the structure of the event that is sent to the external extractor from ADaaS. + * AirSyncEvent is an interface that defines the structure of the event that is sent to the external extractor from AirSync. * It contains the context, payload, execution metadata, and input data as common snap-ins. */ -export interface AirdropEvent { +export interface AirSyncEvent { context: { secrets: { service_account_token: string; }; snap_in_version_id: string; snap_in_id: string; + /** DevRev identity of the user who triggered the sync. */ + user_id: string; + /** DevRev org id (don:identity:.../devo/...). */ + dev_oid: string; + /** External source identity, when the platform provides one. */ + source_id: string; + /** DevRev service-account identity used for the sync. */ + service_account_id: string; }; - payload: AirdropMessage; + payload: AirSyncMessage; execution_metadata: { devrev_endpoint: string; }; @@ -458,9 +287,9 @@ export interface AirdropEvent { } /** - * AirdropMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from ADaaS. + * AirSyncMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from AirSync. */ -export interface AirdropMessage { +export interface AirSyncMessage { connection_data: ConnectionData; event_context: EventContext; event_type: EventType; @@ -468,7 +297,7 @@ export interface AirdropMessage { } /** - * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to ADaaS. + * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to AirSync. * It contains the event type, event context, extractor state, and event data. */ export interface ExtractorEvent { @@ -495,11 +324,18 @@ export type ExternalSystemAttachmentStreamingFunction = ({ export interface ExternalSystemAttachmentStreamingParams { item: NormalizedAttachment; - event: AirdropEvent; + event: AirSyncEvent; +} + +export interface HttpStreamResponse { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + headers: Record; } export interface ExternalSystemAttachmentStreamingResponse { - httpStream?: AxiosResponse; + httpStream?: HttpStreamResponse; error?: ErrorRecord; delay?: number; } @@ -534,7 +370,7 @@ export type ExternalSystemAttachmentReducerFunction< batchSize, }: { attachments: Batch; - adapter: WorkerAdapter; + adapter: ExtractionAdapter; batchSize?: number; }) => NewBatch; @@ -553,7 +389,7 @@ export type ExternalSystemAttachmentIteratorFunction = stream, }: { reducedAttachments: NewBatch; - adapter: WorkerAdapter; + adapter: ExtractionAdapter; stream: ExternalSystemAttachmentStreamingFunction; }) => Promise; diff --git a/src/types/index.ts b/src/types/index.ts deleted file mode 100644 index f49ef309..00000000 --- a/src/types/index.ts +++ /dev/null @@ -1,126 +0,0 @@ -// Common -export { - AdapterUpdateParams, - ErrorLevel, - ErrorRecord, - InitialDomainMapping, - LogRecord, - SyncMode, -} from './common'; - -// Extraction -export { - AirdropEvent, - AirdropMessage, - ConnectionData, - DomainObjectState, - EventContextIn, - EventContextOut, - EventContext, - EventData, - EventType, - ExternalProcessAttachmentFunction, - ExternalSyncUnit, - ExternalSystemAttachmentIteratorFunction, - ExternalSystemAttachmentReducerFunction, - ExternalSystemAttachmentStreamingFunction, - ExternalSystemAttachmentStreamingParams, - ExternalSystemAttachmentStreamingResponse, - ExtractionMode, - ExtractorEvent, - ExtractorEventType, - InitialSyncScope, - ProcessAttachmentReturnType, - TimeUnit, - TimeValue, - TimeValueType, -} from './extraction'; - -// Loading -export { - ExternalSystemAttachment, - ExternalSystemItem, - ExternalSystemItemLoadingParams, - ExternalSystemItemLoadingResponse, - LoaderEventType, -} from './loading'; - -// Repo -export { - NormalizedAttachment, - NormalizedItem, - RepoInterface, -} from '../repo/repo.interfaces'; - -// State -export { AdapterState } from '../state/state.interfaces'; - -export { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; - -// Uploader -export { - Artifact, - ArtifactsPrepareResponse, - SsorAttachment, - StreamAttachmentsResponse, - StreamResponse, - UploadResponse, -} from '../uploader/uploader.interfaces'; - -// Mappers -export type { - MappersCreateParams, - MappersGetByExternalIdParams, - MappersGetByTargetIdParams, - MappersUpdateParams, -} from '../mappers/mappers.interface'; - -export { - SyncMapperRecordStatus, - SyncMapperRecordTargetType, -} from '../mappers/mappers.interface'; - -// External Domain Metadata -export type { - CollectionData, - ConditionalPrivilegeData, - CustomLinkData, - CustomLinkNames, - CustomStage, - CustomState, - EnumData, - EnumValue, - EnumValueKey, - ExternalDomainMetadata, - Field, - FieldCondition, - FieldConditionComparator, - FieldConditionEffect, - FieldConditions, - FieldKey, - FieldPrivilegeData, - FieldReferenceData, - FieldType, - FloatData, - IntData, - PermissionData, - RecordType, - RecordTypeCategory, - RecordTypeCategoryKey, - RecordTypeKey, - RecordTypePrivilegeData, - RecordTypeScope, - ReferenceData, - ReferenceDetail, - ReferenceType, - SchemaVersion, - StageKey, - StageDiagram, - StateKey, - StructData, - StructTypeKey, - StructType, - TargetTypeKeyData, - TextData, - TypedReferenceData, -} from './external-domain-metadata'; diff --git a/src/types/loading.ts b/src/types/loading.ts index b08fb1cb..834b02a9 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -1,6 +1,6 @@ import { Mappers } from '../mappers/mappers'; import { ErrorRecord } from './common'; -import { AirdropEvent } from './extraction'; +import { AirSyncEvent } from './extraction'; export interface StatsFileObject { id: string; @@ -60,7 +60,7 @@ export interface ExternalSystemItem { export interface ExternalSystemItemLoadingParams { item: Type; mappers: Mappers; - event: AirdropEvent; + event: AirSyncEvent; } export interface ExternalSystemItemLoadingResponse { @@ -135,13 +135,9 @@ export type SyncMapperRecord = { input_file?: string; }; -/* eslint-disable @typescript-eslint/no-duplicate-enum-values */ export enum LoaderEventType { DataLoadingProgress = 'DATA_LOADING_PROGRESS', - /** - * @deprecated This was a typo. Use DataLoadingDelayed for the corrected spelling - */ - DataLoadingDelay = 'DATA_LOADING_DELAYED', + DataLoadingDelayed = 'DATA_LOADING_DELAYED', DataLoadingDone = 'DATA_LOADING_DONE', DataLoadingError = 'DATA_LOADING_ERROR', @@ -155,24 +151,4 @@ export enum LoaderEventType { LoaderAttachmentStateDeletionDone = 'LOADER_ATTACHMENT_STATE_DELETION_DONE', LoaderAttachmentStateDeletionError = 'LOADER_ATTACHMENT_STATE_DELETION_ERROR', - - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - DataLoadingDelayed = 'DATA_LOADING_DELAYED', - - /** - * @deprecated Use AttachmentsLoadingProgress instead (note: singular changed to plural) - */ - AttachmentsLoadingProgress = 'ATTACHMENT_LOADING_PROGRESS', - /** - * @deprecated Use AttachmentsLoadingDelayed instead (note: singular changed to plural) - */ - AttachmentsLoadingDelayed = 'ATTACHMENT_LOADING_DELAYED', - /** - * @deprecated Use AttachmentsLoadingDone instead (note: singular changed to plural) - */ - AttachmentsLoadingDone = 'ATTACHMENT_LOADING_DONE', - /** - * @deprecated Use AttachmentsLoadingError instead (note: singular changed to plural) - */ - AttachmentsLoadingError = 'ATTACHMENT_LOADING_ERROR', } diff --git a/src/types/workers.ts b/src/types/workers.ts index 4123c291..2d328c8d 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -1,26 +1,25 @@ import { Worker } from 'worker_threads'; import type { LogLevel } from '../logger/logger.interfaces'; -import { State } from '../state/state'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { BaseState } from '../state/state'; -import { AirdropEvent, EventType, ExtractorEventType } from './extraction'; +import { AirSyncEvent, EventType, ExtractorEventType } from './extraction'; import { LoaderEventType } from './loading'; -import { InitialDomainMapping } from './common'; +import { ErrorRecord, InitialDomainMapping } from './common'; /** * WorkerAdapterInterface is an interface for WorkerAdapter class. * @interface WorkerAdapterInterface * @constructor - * @param {AirdropEvent} event - The event object received from the platform + * @param {AirSyncEvent} event - The event object received from the platform * @param {object=} initialState - The initial state of the adapter * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class */ export interface WorkerAdapterInterface { - event: AirdropEvent; - adapterState: State; + event: AirSyncEvent; + adapterState: BaseState; options?: WorkerAdapterOptions; } @@ -51,11 +50,11 @@ export interface WorkerAdapterOptions { * SpawnInterface is an interface for Spawn class. * @interface SpawnInterface * @constructor - * @param {AirdropEvent} event - The event object received from the platform + * @param {AirSyncEvent} event - The event object received from the platform * @param {Worker} worker - The worker thread */ export interface SpawnInterface { - event: AirdropEvent; + event: AirSyncEvent; worker: Worker; options?: WorkerAdapterOptions; resolve: (value: void | PromiseLike) => void; @@ -69,44 +68,81 @@ export interface SpawnInterface { * In case of lambda timeout, the class emits a lambda timeout event to the platform. * @interface SpawnFactoryInterface * @constructor - * @param {AirdropEvent} event - The event object received from the platform + * @param {AirSyncEvent} event - The event object received from the platform * @param {object=} initialState - The initial state of the adapter - * @param {string} workerPath - The path to the worker file * @param {string} initialDomainMapping - The initial domain mapping * @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class * @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname` */ export interface SpawnFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; - - /** @deprecated Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath */ - workerPath?: string; options?: WorkerAdapterOptions; initialDomainMapping?: InitialDomainMapping; baseWorkerPath?: string; } /** - * TaskAdapterInterface is an interface for TaskAdapter class. - * @interface TaskAdapterInterface - * @constructor - * @param {WorkerAdapter} adapter - The adapter object - */ -export interface TaskAdapterInterface { - adapter: WorkerAdapter; + * TaskResult is the value a worker's `task` (and optional `onTimeout`) callback + * returns to tell the SDK how the current phase ended. The SDK — not the + * connector — maps this status to the phase-appropriate platform event and + * emits it exactly once. Connectors never call `emit` directly. + * + * One lambda invocation = one worker process = exactly one emitted event = + * terminal. Any continuation (CONTINUE_*, next phase, retry after delay) + * happens in a fresh invocation driven by the platform. + * + * The discriminant is a bare string literal, so connectors write e.g. + * `return { status: 'delay', delaySeconds: 60 }` with no import. + * + * Status -> emitted event, per phase: + * + * | status | resumable phases | non-resumable (ESU / metadata) | + * |------------|------------------|--------------------------------| + * | 'success' | *_DONE | *_DONE | + * | 'progress' | *_PROGRESS | *_ERROR (illegal; descriptive) | + * | 'delay' | *_DELAYED | *_ERROR (illegal; descriptive) | + * | 'error' | *_ERROR | *_ERROR | + * + * Resumable phases: data/attachment extraction, data/attachment loading. + * Non-resumable phases: external sync units, metadata. + */ +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } + | { status: 'error'; error: ErrorRecord }; + +/** + * Discriminant string of a {@link TaskResult}. + */ +export type TaskStatus = TaskResult['status']; + +/** + * TaskAdapterInterface is the parameter shape passed to a worker's task and + * onTimeout callbacks. + * @param adapter - The mode-specific adapter for the worker. + */ +export interface TaskAdapterInterface { + adapter: Adapter; } /** - * ProcessTaskInterface is an interface for ProcessTask class. - * @interface ProcessTaskInterface - * @constructor - * @param {function} task - The task to be executed, returns exit code - * @param {function} onTimeout - The task to be executed on timeout, returns exit code - */ -export interface ProcessTaskInterface { - task: (params: TaskAdapterInterface) => Promise; - onTimeout: (params: TaskAdapterInterface) => Promise; + * ProcessTaskInterface is the parameter shape for the process-task entry points. + * + * Both callbacks return a {@link TaskResult}; the SDK — not the connector — + * maps that status to the phase-appropriate platform event and emits it exactly + * once. Connectors never call `emit` directly. + * + * `onTimeout` is optional: if omitted, the SDK emits a phase-appropriate default + * on timeout (progress for resumable phases, error for ESU/metadata). + * + * @param task - Runs the phase; returns how it ended. + * @param onTimeout - Runs only on timeout; returns how to hand off. + */ +export interface ProcessTaskInterface { + task: (params: TaskAdapterInterface) => Promise; + onTimeout?: (params: TaskAdapterInterface) => Promise; } /** @@ -181,7 +217,7 @@ export type WorkerMessage = * WorkerData represents the structure of the worker data object. */ export interface WorkerData { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; workerPath: string; initialDomainMapping?: InitialDomainMapping; @@ -192,7 +228,7 @@ export interface WorkerData { * GetWorkerPathInterface is an interface for getting the worker path. */ export interface GetWorkerPathInterface { - event: AirdropEvent; + event: AirSyncEvent; workerBasePath?: string | null; } diff --git a/src/uploader/uploader.interfaces.ts b/src/uploader/uploader.interfaces.ts index ce995a75..2224bed7 100644 --- a/src/uploader/uploader.interfaces.ts +++ b/src/uploader/uploader.interfaces.ts @@ -1,10 +1,10 @@ import { ErrorRecord } from '../types/common'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { ExternalSystemItem, StatsFileObject } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; export interface UploaderFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } @@ -17,7 +17,7 @@ export type UploaderResult = | { response?: never; error: unknown }; /** - * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS. + * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to AirSync. */ export interface Artifact { id: string; diff --git a/src/uploader/uploader.test.ts b/src/uploader/uploader.test.ts index 41e97775..a2546ee8 100644 --- a/src/uploader/uploader.test.ts +++ b/src/uploader/uploader.test.ts @@ -3,7 +3,7 @@ import FormData from 'form-data'; import { jsonl } from 'js-jsonl'; import zlib from 'zlib'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { callPrivateMethod, createArtifact, @@ -14,13 +14,13 @@ import { spyOnPrivateMethod, } from '../tests/test-helpers'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { compressGzip, downloadToLocal } from './uploader.helpers'; import { ArtifactToUpload, UploaderResult } from './uploader.interfaces'; import { Uploader } from './uploader'; -jest.mock('../http/axios-client-internal'); +jest.mock('../http/client'); jest.mock('./uploader.helpers', () => ({ ...jest.requireActual('./uploader.helpers'), downloadToLocal: jest.fn(), diff --git a/src/uploader/uploader.ts b/src/uploader/uploader.ts index fa9833af..f729cbd9 100644 --- a/src/uploader/uploader.ts +++ b/src/uploader/uploader.ts @@ -1,10 +1,11 @@ import { AxiosResponse } from 'axios'; import FormData from 'form-data'; import { jsonl } from 'js-jsonl'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { MAX_DEVREV_ARTIFACT_SIZE } from '../common/constants'; import { NormalizedAttachment } from '../repo/repo.interfaces'; +import { HttpStreamResponse } from '../types/extraction'; import { serializeError } from '../logger/logger'; import { @@ -193,14 +194,14 @@ export class Uploader { } /** - * Streams an artifact file from an axios response to the upload URL. + * Streams an artifact file from an HTTP streaming response to the upload URL. * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data - * @param {AxiosResponse} fileStream - The axios response stream containing the file data + * @param {HttpStreamResponse} fileStream - The HTTP streaming response containing the file data * @returns {Promise} The axios response or undefined on error */ async streamArtifact( artifact: ArtifactToUpload, - fileStream: AxiosResponse + fileStream: HttpStreamResponse ): Promise> { const formData = new FormData(); for (const field in artifact.form_data) { @@ -274,9 +275,9 @@ export class Uploader { /** * Destroys a stream to prevent resource leaks. - * @param {any} fileStream - The axios response stream to destroy + * @param {HttpStreamResponse} fileStream - The HTTP streaming response to destroy */ - private destroyStream(fileStream: AxiosResponse): void { + private destroyStream(fileStream: HttpStreamResponse): void { try { if (fileStream && fileStream.data) { // For axios response streams, the data property contains the actual stream diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 00000000..81df2d6a --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["node_modules", "dist", "**/*.test.ts", "src/tests"] +}