From 1141334d723b539faade110d3fcf9036709df02e Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:01:42 +0200 Subject: [PATCH 01/30] =?UTF-8?q?docs:=20add=20V2=5FPROGRESS.md=20?= =?UTF-8?q?=E2=80=94=20v2=20refactor=20plan=20&=20source=20of=20truth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-contained plan for the v2 rebuild: branch strategy, 12-commit sequence, hard rules, and baked-in reference data (enum old->new tables, deprecated-file list, connector import surface). Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 184 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 V2_PROGRESS.md diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md new file mode 100644 index 00000000..2d47dc43 --- /dev/null +++ b/V2_PROGRESS.md @@ -0,0 +1,184 @@ +# ADaaS SDK v2 Refactor — Progress & Plan (source of truth) + +> This file is the **single source of truth** for the v2 refactor. It is self-contained: +> any session (or subagent) can read ONLY this file + the named git oracles and have +> everything needed. Do NOT rely on chat history. Update the **Status** table after every commit. + +## TL;DR +Rebuild the v2 branch cleanly from `main` as a sequence of small, single-purpose, reviewable +commits. Mechanical/structural transforms first (Phase 1), polish + surface-defining work last +(Phase 2). `npm run build` stays green every commit; the test suite + api report are intentionally +**left broken** until the final steps. No npm publishing during the work. + +## Git facts +- **Working branch:** `v2` (already hard-reset to `origin/main`). +- **Base commit:** `origin/main` = `5b81ef2` (feat: Add new common error enums #204). +- **Oracle (target shape):** `origin/v2` / tag `v2-old-backup` = `9202e47`. This is the PREVIOUS + v2 attempt — it already implemented the rename, deletions, adapter split, state split+envelope, + and emit-from-return, but bundled into huge unreviewable commits built on a stale base. **Use it + as a structural reference / oracle only. Never copy wholesale. Re-author cleanly.** +- **Safety:** old work preserved at tag `v2-old-backup`. Force-push of `v2` is approved by Rado. + +## Hard rules (apply to EVERY Phase-1 commit) +1. **`npm run build` must stay green.** Achieved by commit 1 adding a build tsconfig that excludes + `**/*.test.ts`. (ts-jest still transpiles tests independently, so tests still *run* — and will + fail on old names — that is expected and accepted.) +2. **Never touch `*.test.ts` files or any api-extractor report** (`*.api.md`, `*.api.json`, + `latest.json`, backwards-compatibility fixtures) until Phase 2. Reviewer rejects any commit that does. +3. **Do NOT rename DevRev backend API route strings.** Only SDK-owned identifiers/types/classes are + renamed Airdrop→AirSync. Route strings like `airdrop.sync-mapper-record.get-by-target` and any + `/internal/airdrop.*` endpoints stay verbatim (they are platform API, not SDK naming). +4. **Every deletion must be grep-justified** (zero live references in SDK `src/` non-test + the 3 + inspectable connectors). Record the justification in the commit body. +5. Each commit is **single-purpose**. If a change belongs to a later commit, defer it. +6. Keep `multithreading/` directory name. No logging/console changes. Both out of scope. + +## Commit sequence + +### Phase 1 — structural (review commit-by-commit) +- **C0 — Package rename** `@devrev/ts-adaas` → `@devrev/airsync-sdk` (scoped, stays under @devrev). + Touch: `package.json` `name`; README references; api-extractor config (entry point / package name); + rename the report file `*/ts-adaas.api.md` → `airsync-sdk.api.md` IF trivial, else defer report to Phase 2. + Do NOT publish. Version → `2.0.0-beta.0` placeholder. +- **C1 — Delete dead/deprecated code + add build tsconfig.** + - Delete `src/deprecated/**` (see list below) and its exports from `src/index.ts`. + - Delete `src/common/event-type-translation.ts` + `.test.ts` (the old↔new event-type shim). + - Delete other `@deprecated`-tagged symbols / provably-unused code (grep-justified). + - Add `tsconfig.build.json` (`include: ["src"]`, `exclude: ["**/*.test.ts","node_modules","dist"]`) + and point `build` script at it. This is the "build stays green" enabler. +- **C2 — Airdrop→AirSync identifier rename.** SDK identifiers/types/classes/comments only. + NOT API route strings (rule 3). e.g. `AirdropEvent`→`AirSyncEvent`, `AirdropMessage`→`AirSyncMessage` + (verify exact target names against `origin/v2`). Provide back-compat type aliases ONLY if origin/v2 did. +- **C3 — Delete deprecated enum members** (NOT a rename — main carries old+new side by side; drop old). + Leave only the new members. See enum tables below. Files: `src/types/extraction.ts`, + `src/types/loading.ts`, plus any `case`/reference cleanups in `control-protocol.ts`, `spawn.helpers.ts`, adapters. +- **C4a — State split (structural only).** Introduce `BaseState` + `ExtractionState` + `LoadingState`. + KEEP the flat `AdapterState = ConnectorState & SdkState` shape (behavior identical). + Author fresh; origin/v2 `src/state/base-state.ts` etc. are structural reference only. +- **C4b — State envelope + migration.** Change on-disk shape to `{ connectorState, sdkState }`. + Add migration shim: read legacy flat v1 blob → split SDK-owned keys into `sdkState` → persist envelope. + (origin/v2 `base-state.ts` has the reference impl incl. `V1_SDK_STATE_KEYS`.) +- **C5 — Adapter split (structural only).** `BaseAdapter` + `ExtractionAdapter` + `LoadingAdapter`. + KEEP existing `emit`-based contract working (behavior identical). Author fresh intermediate form + (this exact form exists in NO branch — origin/v2's split already assumes emit-from-return). +- **C6 — Emit-from-return contract.** `task`/`onTimeout` return a `TaskResult` + (`{ status: 'success'|'progress'|'delay'|'error', ... }`); the SDK maps status→phase event and emits + exactly once; `emit` removed from public surface. `processTask` → `processExtractionTask` + + `processLoadingTask`. Reference: origin/v2 `process-task.ts`, `base-adapter.ts` (mapping keys off + event_type/phase, NOT off state shape — so C4b and C6 are independent). + +### Phase 2 — closing / interactive (batched, done at the end) +- **C7 — JSDoc pass.** Bar = `src/mappers/mappers.ts` style (class block: what+when; method block: + one-line what, "Used to/for…" usage, `@param` w/ type, `@returns`). Public surface + non-obvious + internals (state migration, emit-from-return mapping, attachment streaming pool). Fan out per module, + squash to one `docs:` commit. +- **C8 — Regenerate api report** (`airsync-sdk.api.md`). +- **C9 — Exposure audit (INTERACTIVE with Rado).** Read the regenerated report; decide per-symbol what + to keep public vs hide. Empirical floor = anything imported by the 3 connectors (table below). +- **C10 — Fix tests + bw-compat baseline.** Update test files to new names/contract; decide re-baseline + vs remove the backwards-compatibility gate (v2 is an intentional break, so a v1-comparison gate is wrong). +- **C11 — Migration deliverable.** Scan full `main..v2` diff → derive v1→v2 change catalog → build a + **dedicated `migrate-v2` skill in `adaas-sdk`** (`.claude/skills/migrate-v2/`), later ported to the + `connectors-codegen` repo (owns the `connector-dev` plugin). Mechanical changes auto-applied; semantic + (emit-from-return, state access) flagged for review; ambiguous → `MIGRATION_TODO.md`. Validate against + the 3 inspectable connectors. Skill philosophy mirrors existing `update-sdk` (autonomous + defer-on-ambiguity). + +## Orchestration model +- Per Phase-1 commit, in the main session across multiple sittings: + 1. **Implementer subagent** — does the one commit's work; obeys all Hard rules; build stays green. + 2. **Reviewer subagent** (read-only) — verifies diff against that commit's contract + Hard rules + (esp. "no test/report files touched", "deletions grep-justified", "structure-only vs behavior-only"). + 3. Rado eyeballs → commit → next. +- Mini-workflows for parallel sub-steps: deletion grep-verification (C1), JSDoc-by-module (C7), + exposure-by-symbol (C9). + +--- + +## Reference data (baked in so future sessions don't re-derive) + +### `src/deprecated/**` files to delete (C1) +``` +src/deprecated/adapter/index.ts +src/deprecated/common/helpers.ts +src/deprecated/demo-extractor/external_domain_metadata.json +src/deprecated/demo-extractor/index.ts +src/deprecated/http/client.ts +src/deprecated/uploader/index.ts +``` +Also delete `src/common/event-type-translation.ts` (+ `.test.ts`). +`src/index.ts` on main exports these deprecated barrels — remove those export lines: +`./deprecated/adapter`, `./deprecated/demo-extractor`, `./deprecated/http/client`, `./deprecated/uploader`, +and the `formatAxiosError` export (origin/v2 dropped it — confirm against connector usage; azure-boards imports it, so this is a migration note). + +### C3 — EventType (incoming): DELETE these deprecated members, keep the new ones +| DELETE (old member = old VALUE) | KEEP (new member = new VALUE) | +|--------------------------------------------------------------|--------------------------------------------------------------------------| +| ExtractionExternalSyncUnitsStart = EXTRACTION_EXTERNAL_SYNC_UNITS_START | StartExtractingExternalSyncUnits = START_EXTRACTING_EXTERNAL_SYNC_UNITS | +| ExtractionMetadataStart = EXTRACTION_METADATA_START | StartExtractingMetadata = START_EXTRACTING_METADATA | +| ExtractionDataStart = EXTRACTION_DATA_START | StartExtractingData = START_EXTRACTING_DATA | +| ExtractionDataContinue = EXTRACTION_DATA_CONTINUE | ContinueExtractingData = CONTINUE_EXTRACTING_DATA | +| ExtractionDataDelete = EXTRACTION_DATA_DELETE | StartDeletingExtractorState = START_DELETING_EXTRACTOR_STATE | +| ExtractionAttachmentsStart = EXTRACTION_ATTACHMENTS_START | StartExtractingAttachments = START_EXTRACTING_ATTACHMENTS | +| ExtractionAttachmentsContinue = EXTRACTION_ATTACHMENTS_CONTINUE | ContinueExtractingAttachments = CONTINUE_EXTRACTING_ATTACHMENTS | +| ExtractionAttachmentsDelete = EXTRACTION_ATTACHMENTS_DELETE | StartDeletingExtractorAttachmentsState = START_DELETING_EXTRACTOR_ATTACHMENTS_STATE | +Loading members (StartLoadingData…StartDeletingLoaderAttachmentState) + UnknownEventType are unchanged. + +### C3 — ExtractorEventType (outgoing): DELETE deprecated, keep new +| DELETE (old) | KEEP (new) | +|---------------------------------------|-----------------------------------------| +| ExtractionExternalSyncUnitsDone | ExternalSyncUnitExtractionDone | +| ExtractionExternalSyncUnitsError | ExternalSyncUnitExtractionError | +| ExtractionMetadataDone | MetadataExtractionDone | +| ExtractionMetadataError | MetadataExtractionError | +| ExtractionDataProgress | DataExtractionProgress | +| ExtractionDataDelay | DataExtractionDelayed | +| ExtractionDataDone | DataExtractionDone | +| ExtractionDataError | DataExtractionError | +| ExtractionDataDeleteDone | ExtractorStateDeletionDone | +| ExtractionDataDeleteError | ExtractorStateDeletionError | +| ExtractionAttachmentsProgress | AttachmentExtractionProgress | +| ExtractionAttachmentsDelay | AttachmentExtractionDelayed | +| ExtractionAttachmentsDone | AttachmentExtractionDone | +| ExtractionAttachmentsError | AttachmentExtractionError | +| ExtractionAttachmentsDeleteDone | ExtractorAttachmentsStateDeletionDone | +| ExtractionAttachmentsDeleteError | ExtractorAttachmentsStateDeletionError | +(values for new members are the *_EXTRACTION_* / *_DELETION_* strings — see origin/v2 extraction.ts.) + +### C3 — LoaderEventType: DELETE deprecated typo/plural members +DELETE: `DataLoadingDelay` (typo), `AttachmentsLoadingProgress/Delayed/Done/Error` (the plural-typo dupes). +KEEP: `DataLoadingProgress, DataLoadingDelayed, DataLoadingDone, DataLoadingError, +AttachmentLoadingProgress/Delayed/Done/Error, LoaderStateDeletionDone/Error, +LoaderAttachmentStateDeletionDone/Error, UnknownEventType`. + +### Connector import surface (empirical floor for C9 exposure audit + C11 migration) +Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: +- **asana-internal:** AirSyncDefaultItemTypes, AirdropEvent, ErrorRecord, EventType, ExternalDomainMetadata, + ExternalSyncUnit, ExtractorEventType, LoaderEventType, NormalizedAttachment, NormalizedItem, RepoInterface, + SyncMode, WorkerAdapter, axios, processTask, spawn +- **azure-boards:** AirSyncDefaultItemTypes, AirdropEvent, AirdropMessage, EventType, ExternalSyncUnit, + ExternalSystemAttachment, ExternalSystemItem, ExternalSystemItemLoadingParams, ExtractorEventType, + LoaderEventType, NormalizedAttachment, NormalizedItem, SyncMode, WorkerAdapter, formatAxiosError, + installInitialDomainMapping, processTask, spawn +- **google-drive:** AirdropEvent, EventType, ExternalSystemAttachmentStreamingParams, ExtractorEventType, + NormalizedAttachment, NormalizedItem, SyncMode, WorkerAdapter, processTask, spawn, axios, axiosClient + +**Migration-relevant removals these connectors will hit:** `WorkerAdapter` (removed → use processExtraction/LoadingTask ++ return-based contract), `processTask` (split), `formatAxiosError` (dropped from index), `AirdropEvent`/`AirdropMessage` +(renamed AirSync*), all old `EXTRACTION_*` enum members (deleted). + +## Status +| Commit | State | Notes | +|--------|-------|-------| +| C0 package rename | ☐ todo | | +| C1 delete + tsconfig | ☐ todo | | +| C2 AirSync rename | ☐ todo | | +| C3 enum cleanup | ☐ todo | | +| C4a state split | ☐ todo | | +| C4b state envelope | ☐ todo | | +| C5 adapter split | ☐ todo | | +| C6 emit-from-return | ☐ todo | | +| C7 JSDoc | ☐ todo | Phase 2 | +| C8 api report | ☐ todo | Phase 2 | +| C9 exposure audit | ☐ todo | Phase 2, interactive | +| C10 tests + baseline | ☐ todo | Phase 2 | +| C11 migrate-v2 skill | ☐ todo | Phase 2 | From 8ddeb87a3bfa63425dd6b89a1efef9a0c188ad2b Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:03:59 +0200 Subject: [PATCH 02/30] refactor(v2): rename package @devrev/ts-adaas -> @devrev/airsync-sdk Package identity change for the v2 AirSync rebrand. Scoped to package identity only: - package.json name, version (2.0.0-beta.0), description - package-lock.json (regenerated) - README install command - release workflow references Deferred to later commits: README Airdrop->AirSync prose (C2), the api-extractor report filename + config under backwards-compatibility (C8, test/report territory). Ref: V2_PROGRESS.md C0 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release.yaml | 4 ++-- README.md | 2 +- package-lock.json | 8 ++++---- package.json | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 31452734..13d7658e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -# This workflow is used to release a new version of @devrev/ts-adaas package to +# This workflow is used to release a new version of @devrev/airsync-sdk package to # npm registry and generate release notes using softprops/action-gh-release # action. It consists of two jobs: # @@ -93,7 +93,7 @@ jobs: id: version run: | # Get the latest version including prereleases - LATEST_VERSION=$(npm view @devrev/ts-adaas versions --json 2>/dev/null | jq -r '.[-1]' || echo "0.0.0") + LATEST_VERSION=$(npm view @devrev/airsync-sdk versions --json 2>/dev/null | jq -r '.[-1]' || echo "0.0.0") echo "Latest published version: $LATEST_VERSION" echo "LATEST_PUBLISHED_VERSION=$LATEST_VERSION" >> $GITHUB_ENV diff --git a/README.md b/README.md index 11f7b2c5..e2fc2c9b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ It provides features such as: ## Installation ```bash -npm install @devrev/ts-adaas +npm install @devrev/airsync-sdk ``` ## Reference diff --git a/package-lock.json b/package-lock.json index 9d56f7ed..d0038c23 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "@devrev/ts-adaas", - "version": "1.19.7", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@devrev/ts-adaas", - "version": "1.19.7", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.0", "license": "ISC", "dependencies": { "@devrev/typescript-sdk": "^1.1.74", diff --git a/package.json b/package.json index 27d1e7fc..0310fb58 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { - "name": "@devrev/ts-adaas", - "version": "1.19.7", - "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.0", + "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.", "type": "commonjs", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From 0760f0b99e7a9ab64fe6cd8c05af7e29891ad39a Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:04:18 +0200 Subject: [PATCH 03/30] docs: mark C0 done in V2_PROGRESS Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 2d47dc43..e9df7826 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -169,7 +169,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: ## Status | Commit | State | Notes | |--------|-------|-------| -| C0 package rename | ☐ todo | | +| C0 package rename | ☑ done | 8ddeb87. @devrev/ts-adaas→@devrev/airsync-sdk, v2.0.0-beta.0. Report filename rename deferred to C8. | | C1 delete + tsconfig | ☐ todo | | | C2 AirSync rename | ☐ todo | | | C3 enum cleanup | ☐ todo | | From d573cb6a1abff762b78b85c1647b8af99493431e Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:11:04 +0200 Subject: [PATCH 04/30] refactor(v2)!: delete deprecated module + add build tsconfig Remove the entire src/deprecated/ tree (Adapter, DemoExtractor, HTTPClient, deprecated Uploader, demo metadata) and its four barrel re-exports from src/index.ts. These were unused by production code and any of the inspectable connectors (zero live references). Add tsconfig.build.json that extends tsconfig.json and excludes test files (**/*.test.ts, src/tests). Point the 'build' script at it so 'npm run build' compiles only shippable source. This keeps the build green across subsequent v2 commits even while the test suite (run separately via ts-jest) lags behind the rename/contract changes until it is fixed in Phase 2. BREAKING CHANGE: removed deprecated exports Adapter, DemoExtractor, HTTPClient, and the legacy Uploader from the public API. Ref: V2_PROGRESS.md C1 Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 2 +- src/deprecated/adapter/index.ts | 209 ---------------- src/deprecated/common/helpers.ts | 55 ---- .../external_domain_metadata.json | 38 --- src/deprecated/demo-extractor/index.ts | 235 ------------------ src/deprecated/http/client.ts | 149 ----------- src/deprecated/uploader/index.ts | 156 ------------ src/index.ts | 4 - tsconfig.build.json | 4 + 9 files changed, 5 insertions(+), 847 deletions(-) delete mode 100644 src/deprecated/adapter/index.ts delete mode 100644 src/deprecated/common/helpers.ts delete mode 100644 src/deprecated/demo-extractor/external_domain_metadata.json delete mode 100644 src/deprecated/demo-extractor/index.ts delete mode 100644 src/deprecated/http/client.ts delete mode 100644 src/deprecated/uploader/index.ts create mode 100644 tsconfig.build.json diff --git a/package.json b/package.json index 0310fb58..6858c359 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "main": "./dist/index.js", "typings": "./dist/index.d.ts", "scripts": { - "build": "tsc -p ./tsconfig.json", + "build": "tsc -p ./tsconfig.build.json", "prepare": "npm run build", "start": "ts-node src/index.ts", "lint": "eslint .", diff --git a/src/deprecated/adapter/index.ts b/src/deprecated/adapter/index.ts deleted file mode 100644 index 69547684..00000000 --- a/src/deprecated/adapter/index.ts +++ /dev/null @@ -1,209 +0,0 @@ -import axios from 'axios'; - -import { - AirdropEvent, - EventData, - ExtractorEvent, - ExtractorEventType, -} from '../../types/extraction'; -import { Artifact } from '../../uploader/uploader.interfaces'; - -import { AdapterState } from '../../state/state.interfaces'; - -import { STATELESS_EVENT_TYPES } from '../../common/constants'; -import { getTimeoutExtractorEventType } from '../common/helpers'; -// import { Logger } from '../../logger/logger'; -import { State, createAdapterState } from '../../state/state'; -import { translateIncomingEventType } from '../../common/event-type-translation'; -import { runWithSdkLogContext } from '../../logger/logger.context'; - -/** - * Adapter class is used to interact with Airdrop platform. The class provides - * utilities to - * - emit control events to the platform - * - update the state of the extractor - * - set the last saved state in case of a timeout - * - * @class Adapter - * @constructor - * @deprecated - * @param {AirdropEvent} event - The event object received from the platform - * @param {object=} initialState - The initial state of the adapter - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - */ - -/** - * Creates an adapter instance. - * - * @param {AirdropEvent} event - The event object received from the platform - * @param initialState - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @return The adapter instance - */ - -export async function createAdapter( - event: AirdropEvent, - initialState: ConnectorState, - isLocalDevelopment: boolean = false -) { - event.payload.event_type = translateIncomingEventType(event.payload.event_type); - - const newInitialState = structuredClone(initialState); - const adapterState: State = await createAdapterState({ - event, - initialState: newInitialState, - }); - - const a = new Adapter( - event, - adapterState, - isLocalDevelopment - ); - - return a; -} - -export class Adapter { - private adapterState: State; - private _artifacts: Artifact[]; - - private event: AirdropEvent; - private callbackUrl: string; - private devrevToken: string; - private startTime: number; - private heartBeatFn: ReturnType | undefined; - private exit: boolean = false; - private lambdaTimeout: number = 10 * 60 * 1000; // 10 minutes in milliseconds - private heartBeatInterval: number = 30 * 1000; // 30 seconds in milliseconds - - constructor( - event: AirdropEvent, - adapterState: State, - isLocalDevelopment: boolean = false - ) { - if (!isLocalDevelopment) { - // Logger.init(event); - } - - this.adapterState = adapterState; - this._artifacts = []; - - this.event = event; - this.callbackUrl = event.payload.event_context.callback_url; - this.devrevToken = event.context.secrets.service_account_token; - - this.startTime = Date.now(); - - // Run heartbeat every 30 seconds - this.heartBeatFn = setInterval(async () => { - const b = await this.heartbeat(); - if (b) { - this.exitAdapter(); - } - }, this.heartBeatInterval); - } - - get state(): AdapterState { - return this.adapterState.state; - } - - set state(value: AdapterState) { - this.adapterState.state = value; - } - - get artifacts(): Artifact[] { - return this._artifacts; - } - - set artifacts(value: Artifact[]) { - this._artifacts = value; - } - - /** - * Emits an event to the platform. - * - * @param {ExtractorEventType} newEventType - The event type to be emitted - * @param {EventData=} data - The data to be sent with the event - */ - async emit(newEventType: ExtractorEventType, data?: EventData) { - if (this.exit) { - console.warn( - 'Adapter is already in exit state. No more events can be emitted.' - ); - return; - } - - // We want to save the state every time we emit an event, except for the start and delete events - if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { - runWithSdkLogContext(() => - console.log(`Saving state before emitting event`) - ); - await this.adapterState.postState(this.state); - } - - const newEvent: ExtractorEvent = { - event_type: newEventType, - event_context: this.event.payload.event_context, - event_data: { - ...data, - }, - }; - - try { - await axios.post( - this.callbackUrl, - { ...newEvent }, - { - headers: { - Accept: 'application/json, text/plain, */*', - Authorization: this.devrevToken, - 'Content-Type': 'application/json', - }, - } - ); - - console.log('Successfully emitted event: ' + JSON.stringify(newEvent)); - } catch (error) { - // If this request fails the extraction will be stuck in loop and - // we need to stop it through UI or think about retrying this request - console.log( - 'Failed to emit event: ' + - JSON.stringify(newEvent) + - ', error: ' + - error - ); - } finally { - this.exitAdapter(); - } - } - - /** - * Exit the adapter. This will stop the heartbeat and no - * further events will be emitted. - */ - private exitAdapter() { - this.exit = true; - } - - /** - * Heartbeat function to check if the lambda is about to timeout. - * @returns true if 10 minutes have passed since the start of the lambda. - */ - private async heartbeat(): Promise { - if (this.exit) { - return true; - } - if (Date.now() - this.startTime > this.lambdaTimeout) { - const timeoutEventType = getTimeoutExtractorEventType( - this.event.payload.event_type - ); - if (timeoutEventType !== null) { - const { eventType, isError } = timeoutEventType; - const err = isError ? { message: 'Lambda Timeout' } : undefined; - await this.emit(eventType, { error: err, artifacts: this._artifacts }); - return true; - } - } - return false; - } -} diff --git a/src/deprecated/common/helpers.ts b/src/deprecated/common/helpers.ts deleted file mode 100644 index 41eb9477..00000000 --- a/src/deprecated/common/helpers.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { jsonl } from 'js-jsonl'; - -import { EventType, ExtractorEventType } from '../../types/extraction'; - -export function createFormData( - //eslint-disable-next-line @typescript-eslint/no-explicit-any - preparedArtifact: any, - fetchedObjects: object[] | object -): FormData { - const formData = new FormData(); - for (const item of preparedArtifact.form_data) { - formData.append(item.key, item.value); - } - - const output = jsonl.stringify(fetchedObjects); - formData.append('file', output); - - return formData; -} - -export function getTimeoutExtractorEventType(eventType: EventType): { - eventType: ExtractorEventType; - isError: boolean; -} | null { - switch (eventType) { - case EventType.ExtractionMetadataStart: - return { - eventType: ExtractorEventType.ExtractionMetadataError, - isError: true, - }; - case EventType.ExtractionDataStart: - case EventType.ExtractionDataContinue: - return { - eventType: ExtractorEventType.ExtractionDataProgress, - isError: false, - }; - case EventType.ExtractionAttachmentsStart: - case EventType.ExtractionAttachmentsContinue: - return { - eventType: ExtractorEventType.ExtractionAttachmentsProgress, - isError: false, - }; - case EventType.ExtractionExternalSyncUnitsStart: - return { - eventType: ExtractorEventType.ExtractionExternalSyncUnitsError, - isError: true, - }; - default: - console.log( - 'Event type not recognized in getTimeoutExtractorEventType function: ' + - eventType - ); - return null; - } -} diff --git a/src/deprecated/demo-extractor/external_domain_metadata.json b/src/deprecated/demo-extractor/external_domain_metadata.json deleted file mode 100644 index ea876481..00000000 --- a/src/deprecated/demo-extractor/external_domain_metadata.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "record_types": { - "users": { - "fields": { - "name": { - "is_required": true, - "type": "text", - "name": "Name", - "text": { - "min_length": 1 - } - }, - "email": { - "type": "text", - "name": "Email", - "is_required": true - } - } - }, - "contacts": { - "fields": { - "name": { - "is_required": true, - "type": "text", - "name": "Name", - "text": { - "min_length": 1 - } - }, - "email": { - "type": "text", - "name": "Email", - "is_required": true - } - } - } - } -} diff --git a/src/deprecated/demo-extractor/index.ts b/src/deprecated/demo-extractor/index.ts deleted file mode 100644 index 30e937ff..00000000 --- a/src/deprecated/demo-extractor/index.ts +++ /dev/null @@ -1,235 +0,0 @@ -import { - AirdropEvent, - EventType, - ExternalSyncUnit, - ExtractorEventType, -} from '../../types/extraction'; -import { Adapter } from '../adapter'; -import { Uploader } from '../uploader'; -import externalDomainMetadata from './external_domain_metadata.json'; - -type ConnectorState = object; - -/** - * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS. - * - * @class DemoExtractor - * @deprecated - **/ -export class DemoExtractor { - private event: AirdropEvent; - private adapter: Adapter; - private uploader: Uploader; - - constructor(event: AirdropEvent, adapter: Adapter) { - this.event = event; - this.adapter = adapter; - this.uploader = new Uploader( - this.event.execution_metadata.devrev_endpoint, - this.event.context.secrets.service_account_token - ); - } - - async run() { - switch (this.event.payload.event_type) { - case EventType.ExtractionExternalSyncUnitsStart: { - const externalSyncUnits: ExternalSyncUnit[] = [ - { - id: 'devrev', - name: 'devrev', - description: 'Demo external sync unit', - }, - ]; - - await this.adapter.emit( - ExtractorEventType.ExtractionExternalSyncUnitsDone, - { - external_sync_units: externalSyncUnits, - } - ); - - break; - } - - case EventType.ExtractionMetadataStart: { - const { artifact, error } = await this.uploader.upload( - 'metadata_1.jsonl', - 'external_domain_metadata', - externalDomainMetadata - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionMetadataError, { - error, - }); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionMetadataDone, { - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataStart: { - const contacts = [ - { - id: 'contact-1', - created_date: '1999-12-25T01:00:03+01:00', - modified_date: '1999-12-25T01:00:03+01:00', - data: { - email: 'johnsmith@test.com', - name: 'John Smith', - }, - }, - { - id: 'contact-2', - created_date: '1999-12-27T15:31:34+01:00', - modified_date: '2002-04-09T01:55:31+02:00', - data: { - email: 'janesmith@test.com', - name: 'Jane Smith', - }, - }, - ]; - - const { artifact, error } = await this.uploader.upload( - 'contacts_1.json', - 'contacts', - contacts - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionDataError, { - error, - }); - - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionDataProgress, { - progress: 50, - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataContinue: { - const users = [ - { - id: 'user-1', - created_date: '1999-12-25T01:00:03+01:00', - modified_date: '1999-12-25T01:00:03+01:00', - data: { - email: 'johndoe@test.com', - name: 'John Doe', - }, - }, - { - id: 'user-2', - created_date: '1999-12-27T15:31:34+01:00', - modified_date: '2002-04-09T01:55:31+02:00', - data: { - email: 'janedoe@test.com', - name: 'Jane Doe', - }, - }, - ]; - - const { artifact, error } = await this.uploader.upload( - 'users_1.json', - 'users', - users - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionDataError, { - error, - }); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionDataDone, { - progress: 100, - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataDelete: { - await this.adapter.emit(ExtractorEventType.ExtractionDataDeleteDone); - break; - } - - case EventType.ExtractionAttachmentsStart: { - const attachment1 = ['This is attachment1.txt content']; - const { artifact, error } = await this.uploader.upload( - 'attachment1.txt', - 'attachment', - attachment1 - ); - - if (error || !artifact) { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsError, - { - error, - } - ); - return; - } - - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsProgress, - { - artifacts: [artifact], - } - ); - - break; - } - - case EventType.ExtractionAttachmentsContinue: { - const attachment2 = ['This is attachment2.txt content']; - const { artifact, error } = await this.uploader.upload( - 'attachment2.txt', - 'attachment', - attachment2 - ); - - if (error || !artifact) { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsError, - { - error, - } - ); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDone, { - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionAttachmentsDelete: { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsDeleteDone - ); - break; - } - - default: { - console.error( - 'Event in DemoExtractor run not recognized: ' + - JSON.stringify(this.event.payload.event_type) - ); - } - } - } -} diff --git a/src/deprecated/http/client.ts b/src/deprecated/http/client.ts deleted file mode 100644 index e4430437..00000000 --- a/src/deprecated/http/client.ts +++ /dev/null @@ -1,149 +0,0 @@ -import axios, { - InternalAxiosRequestConfig, - isAxiosError, - RawAxiosRequestHeaders, -} from 'axios'; -import { - RATE_LIMIT_EXCEEDED, - RATE_LIMIT_EXCEEDED_STATUS_CODE, -} from '../../http/constants'; -import { HTTPResponse } from '../../http/types'; - -export const defaultResponse: HTTPResponse = { - data: { - delay: 0, - nextPage: 1, - records: [], - }, - message: '', - success: false, -}; - -/** - * HTTPClient class to make HTTP requests - * @deprecated - */ -export class HTTPClient { - private retryAfter = 0; - private retryAt = 0; - private axiosInstance = axios.create(); - - constructor() { - // Add request interceptor to check for retryAfter before making a request - this.axiosInstance.interceptors.request.use( - (config: InternalAxiosRequestConfig) => { - // Check if retryAfter is not 0 and return a LIMIT_EXCEEDED error - if (this.retryAfter !== 0) { - // check if the current time is greater than the retryAt time - const currentTime = new Date().getTime(); - if (currentTime < this.retryAt) { - console.error( - 'Rate limit exceeded. Interceptor has retryAfter: ' + - this.retryAfter - ); - // Rate limit exceeded. - return Promise.reject(RATE_LIMIT_EXCEEDED); - } else { - // Reset the retryAfter - this.retryAfter = 0; - } - } - return config; - }, - (error) => { - return Promise.reject(error); - } - ); - } - - /** - * - * Function to make a GET call to the endpoint. - * There is special handling for rate limit exceeded error. - * In case of rate limit exceeded, the function returns success as true and the delay time in seconds - * In case of any other error, the function returns success as false and the error message - */ - async getCall( - endpoint: string, - headers: RawAxiosRequestHeaders, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - params?: any - ): Promise { - // Return the LIMIT_EXCEEDED error if the retryAfter is not 0 - try { - const res = await this.axiosInstance.get(endpoint, { - headers: headers, - params: params, - }); - return { - ...defaultResponse, - data: { - delay: 0, - records: res.data, - }, - success: true, - }; - } catch (error: unknown) { - console.log('Error in getCall: ' + JSON.stringify(error)); - // send error to adapter - if (isAxiosError(error)) { - if (error.response?.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) { - this.retryAfter = error.response.headers['retry-after'] - ? error.response.headers['retry-after'] - : 0; - this.retryAt = new Date().getTime() + this.retryAfter * 1000; - console.warn( - 'Rate limit exceeded. Error code: ' + - error.response.status + - ' RetryAfter: ' + - this.retryAfter + - ' RetryAt: ' + - this.retryAt - ); - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: RATE_LIMIT_EXCEEDED, - success: true, - }; - } - if (error.response) { - return { ...defaultResponse, message: error.response.data }; - } else { - return { ...defaultResponse, message: error.message }; - } - } else { - if (this.retryAfter !== 0) { - console.warn( - 'Rate limit exceeded. Going to return the following response: ' + - JSON.stringify(error) - ); - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: - typeof error === 'string' - ? error - : JSON.stringify(error, Object.getOwnPropertyNames(error)), - success: true, - }; - } - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: - typeof error === 'string' - ? error - : JSON.stringify(error, Object.getOwnPropertyNames(error)), - success: false, - }; - } - } - } -} diff --git a/src/deprecated/uploader/index.ts b/src/deprecated/uploader/index.ts deleted file mode 100644 index eca7203b..00000000 --- a/src/deprecated/uploader/index.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { betaSDK, client } from '@devrev/typescript-sdk'; -import fs, { promises as fsPromises } from 'fs'; -import { axiosClient } from '../../http/axios-client-internal'; -import { Artifact, UploadResponse } from '../../uploader/uploader.interfaces'; -import { createFormData } from '../common/helpers'; - -/** - * Uploader class is used to upload files to the DevRev platform. - * The class provides utilities to - * - prepare artifact - * - upload artifact - * - return the artifact information to the platform - * - * @class Uploader - * @constructor - * @param {string} endpoint - The endpoint of the DevRev platform - * @param {string} token - The token to authenticate with the DevRev platform - * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system. - */ -export class Uploader { - private betaDevrevSdk: betaSDK.Api; - private local: boolean; - constructor(endpoint: string, token: string, local = false) { - this.betaDevrevSdk = client.setupBeta({ - endpoint, - token, - }); - this.local = local; - } - - /** - * - * Uploads the file to the DevRev platform. The file is uploaded to the platform - * and the artifact information is returned. - * - * @param {string} filename - The name of the file to be uploaded - * @param {string} entity - The entity type of the file to be uploaded - * @param {object[] | object} fetchedObjects - The objects to be uploaded - * @param filetype - The type of the file to be uploaded - * @returns {Promise} - The response object containing the artifact information - */ - async upload( - filename: string, - entity: string, - fetchedObjects: object[] | object, - filetype: string = 'application/jsonl+json' - ): Promise { - if (this.local) { - await this.downloadToLocal(filename, fetchedObjects); - } - - const preparedArtifact = await this.prepareArtifact(filename, filetype); - - if (!preparedArtifact) { - return { - artifact: undefined, - error: { message: 'Error while preparing artifact' }, - }; - } - - const uploadedArtifact = await this.uploadToArtifact( - preparedArtifact, - fetchedObjects - ); - - if (!uploadedArtifact) { - return { - artifact: undefined, - error: { message: 'Error while uploading artifact' }, - }; - } - - // If file was successfully uploaded we want to post data about that file when emitting - const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1; - const artifact: Artifact = { - id: preparedArtifact.id, - item_type: entity, - item_count: itemCount, - }; - - console.log(`Artifact uploaded successfully: ${artifact.id}`); - - return { artifact, error: undefined }; - } - - private async prepareArtifact( - filename: string, - filetype: string - ): Promise { - try { - const response = await this.betaDevrevSdk.artifactsPrepare({ - file_name: filename, - file_type: filetype, - }); - - return response.data; - } catch (error) { - console.error('Error while preparing artifact: ' + error); - return null; - } - } - - private async uploadToArtifact( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - preparedArtifact: any, - fetchedObjects: object[] | object - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): Promise { - const formData = createFormData(preparedArtifact, fetchedObjects); - try { - const response = await axiosClient.post(preparedArtifact.url, formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }); - - return response; - } catch (error) { - console.error('Error while uploading artifact: ' + error); - return null; - } - } - - private async downloadToLocal( - filePath: string, - fetchedObjects: object | object[] - ) { - console.log(`Uploading ${filePath} to local file system`); - try { - if (!fs.existsSync('extracted_files')) { - fs.mkdirSync('extracted_files'); - } - - const timestamp = new Date().getTime(); - const fileHandle = await fsPromises.open( - `extracted_files/${timestamp}_${filePath}`, - 'w' - ); - let objArray = []; - if (!Array.isArray(fetchedObjects)) { - objArray.push(fetchedObjects); - } else { - objArray = fetchedObjects; - } - for (const jsonObject of objArray) { - const jsonLine = JSON.stringify(jsonObject) + '\n'; - await fileHandle.write(jsonLine); - } - await fileHandle.close(); - console.log('Data successfully written to', filePath); - } catch (error) { - console.error('Error writing data to file:', error); - return Promise.reject(error); - } - } -} diff --git a/src/index.ts b/src/index.ts index 26dd8ff5..b4b94dc3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,6 @@ export { AirSyncDefaultItemTypes } from './common/constants'; export { ExtractionCommonError } from './common/errors'; export * from './common/install-initial-domain-mapping'; -export * from './deprecated/adapter'; -export * from './deprecated/demo-extractor'; -export * from './deprecated/http/client'; -export * from './deprecated/uploader'; export * from './http'; export { formatAxiosError, serializeAxiosError } from './logger/logger'; export { MockServer } from './mock-server/mock-server'; diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 00000000..81df2d6a --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["node_modules", "dist", "**/*.test.ts", "src/tests"] +} From 5c2aecd1d7aa3b8ea6f69553d52fa48feb44b711 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:11:15 +0200 Subject: [PATCH 05/30] docs: mark C1 done; move event-type-translation deletion to C3 Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index e9df7826..6c9ab35f 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -40,18 +40,28 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi Touch: `package.json` `name`; README references; api-extractor config (entry point / package name); rename the report file `*/ts-adaas.api.md` → `airsync-sdk.api.md` IF trivial, else defer report to Phase 2. Do NOT publish. Version → `2.0.0-beta.0` placeholder. -- **C1 — Delete dead/deprecated code + add build tsconfig.** - - Delete `src/deprecated/**` (see list below) and its exports from `src/index.ts`. - - Delete `src/common/event-type-translation.ts` + `.test.ts` (the old↔new event-type shim). - - Delete other `@deprecated`-tagged symbols / provably-unused code (grep-justified). +- **C1 — Delete deprecated dir + add build tsconfig.** + - Delete `src/deprecated/**` (see list below) and its 4 `export * from './deprecated/...'` lines from `src/index.ts`. - Add `tsconfig.build.json` (`include: ["src"]`, `exclude: ["**/*.test.ts","node_modules","dist"]`) and point `build` script at it. This is the "build stays green" enabler. + - NOTE: `event-type-translation.ts` deletion MOVED to C3 — it is NOT dead; production code + (`process-task.ts`, `spawn.ts`, `worker-adapter.ts`, `control-protocol.ts`) imports it, so it + can only be removed alongside the old enum members it translates (C3). Deleting it here would + break the build. - **C2 — Airdrop→AirSync identifier rename.** SDK identifiers/types/classes/comments only. NOT API route strings (rule 3). e.g. `AirdropEvent`→`AirSyncEvent`, `AirdropMessage`→`AirSyncMessage` (verify exact target names against `origin/v2`). Provide back-compat type aliases ONLY if origin/v2 did. -- **C3 — Delete deprecated enum members** (NOT a rename — main carries old+new side by side; drop old). - Leave only the new members. See enum tables below. Files: `src/types/extraction.ts`, - `src/types/loading.ts`, plus any `case`/reference cleanups in `control-protocol.ts`, `spawn.helpers.ts`, adapters. +- **C3 — Remove old event-type compatibility layer** (NOT a rename — main carries old+new side by side; drop old). + - Delete deprecated enum members, leaving only the new ones (tables below). Files: `src/types/extraction.ts`, + `src/types/loading.ts`. + - Delete `src/common/event-type-translation.ts` + `.test.ts`. + - Rewire its 4 production callers to use event types directly (no translation; backend now sends/accepts + only new types): `src/multithreading/process-task.ts` (translateIncomingEventType), + `src/multithreading/spawn/spawn.ts` (translateIncomingEventType), + `src/multithreading/worker-adapter/worker-adapter.ts` (translateOutgoingEventType), + `src/common/control-protocol.ts` (translateOutgoingEventType). + - Plus any `case`/reference cleanups in `spawn.helpers.ts`, adapters. + - Exported translation fns are NOT in index.ts (internal) — safe to delete. - **C4a — State split (structural only).** Introduce `BaseState` + `ExtractionState` + `LoadingState`. KEEP the flat `AdapterState = ConnectorState & SdkState` shape (behavior identical). Author fresh; origin/v2 `src/state/base-state.ts` etc. are structural reference only. @@ -170,7 +180,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | Commit | State | Notes | |--------|-------|-------| | C0 package rename | ☑ done | 8ddeb87. @devrev/ts-adaas→@devrev/airsync-sdk, v2.0.0-beta.0. Report filename rename deferred to C8. | -| C1 delete + tsconfig | ☐ todo | | +| C1 delete + tsconfig | ☑ done | d573cb6. Deleted src/deprecated/ (6 files) + 4 index exports; added tsconfig.build.json (excludes tests), build script points to it. Reviewer-approved. event-type-translation deletion moved to C3. | | C2 AirSync rename | ☐ todo | | | C3 enum cleanup | ☐ todo | | | C4a state split | ☐ todo | | From 1fa9afc8a08b2204a14c07dd48074e779ba44cea Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:20:18 +0200 Subject: [PATCH 06/30] refactor(v2)!: rename AirdropEvent/AirdropMessage -> AirSync*; rebrand prose Rename the public types AirdropEvent -> AirSyncEvent and AirdropMessage -> AirSyncMessage across all production code (no back-compat alias), and update stale 'ADaaS'/'Airdrop' branding in comments and JSDoc to 'AirSync'. Left untouched (platform contracts): the /internal/airdrop.* API route strings, the AIRDROP_* mapping enum members and their 'airdrop_*' values, and the external_system_type: 'ADaaS' string literal. BREAKING CHANGE: AirdropEvent and AirdropMessage are renamed to AirSyncEvent and AirSyncMessage. Connectors must update their imports and type references. Ref: V2_PROGRESS.md C2 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/common/control-protocol.ts | 4 +-- src/common/install-initial-domain-mapping.ts | 4 +-- src/common/test-utils.ts | 14 ++++---- src/logger/logger.interfaces.ts | 4 +-- src/mappers/mappers.interface.ts | 4 +-- src/multithreading/spawn/spawn.ts | 8 ++--- .../worker-adapter/worker-adapter.ts | 6 ++-- src/repo/repo.interfaces.ts | 4 +-- src/state/state.interfaces.ts | 4 +-- src/types/extraction.ts | 32 +++++++++---------- src/types/index.ts | 4 +-- src/types/loading.ts | 4 +-- src/types/workers.ts | 18 +++++------ src/uploader/uploader.interfaces.ts | 6 ++-- 14 files changed, 58 insertions(+), 58 deletions(-) diff --git a/src/common/control-protocol.ts b/src/common/control-protocol.ts index c372c91d..6116de9b 100644 --- a/src/common/control-protocol.ts +++ b/src/common/control-protocol.ts @@ -1,7 +1,7 @@ import { AxiosResponse } from 'axios'; import { axiosClient } from '../http/axios-client-internal'; import { - AirdropEvent, + AirSyncEvent, EventData, ExtractorEvent, ExtractorEventType, @@ -12,7 +12,7 @@ import { LIBRARY_VERSION } from './constants'; import { translateOutgoingEventType } from './event-type-translation'; export interface EmitInterface { - event: AirdropEvent; + event: AirSyncEvent; eventType: ExtractorEventType | LoaderEventType; data?: EventData; } diff --git a/src/common/install-initial-domain-mapping.ts b/src/common/install-initial-domain-mapping.ts index f65026ea..5a8ec436 100644 --- a/src/common/install-initial-domain-mapping.ts +++ b/src/common/install-initial-domain-mapping.ts @@ -1,11 +1,11 @@ import { axiosClient } from '../http/axios-client-internal'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { serializeError } from '../logger/logger'; import { InitialDomainMapping } from '../types/common'; export async function installInitialDomainMapping( - event: AirdropEvent, + event: AirSyncEvent, initialDomainMappingJson: InitialDomainMapping ): Promise { const devrevEndpoint = event.execution_metadata.devrev_endpoint; diff --git a/src/common/test-utils.ts b/src/common/test-utils.ts index 1584a345..c9b8b2ba 100644 --- a/src/common/test-utils.ts +++ b/src/common/test-utils.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, EventType } from '../types/extraction'; +import { AirSyncEvent, EventType } from '../types/extraction'; export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; @@ -44,19 +44,19 @@ function deepMerge>( } /** - * Creates a mock AirdropEvent for testing. + * Creates a mock AirSyncEvent for testing. * * @param mockServerUrl - Base URL for the mock server. Defaults to {@link MOCK_SERVER_DEFAULT_URL}. * The `callback_url`, `worker_data_url`, and `devrev_endpoint` fields are * derived from this value unless explicitly overridden. - * @param overrides - Deep partial of AirdropEvent. Any provided fields are + * @param overrides - Deep partial of AirSyncEvent. Any provided fields are * deep-merged on top of the defaults. */ export function createMockEvent( mockServerUrl: string = MOCK_SERVER_DEFAULT_URL, - overrides: DeepPartial = {} -): AirdropEvent { - const base: AirdropEvent = { + overrides: DeepPartial = {} +): AirSyncEvent { + const base: AirSyncEvent = { context: { secrets: { service_account_token: 'test_token', @@ -118,7 +118,7 @@ export function createMockEvent( const merged = deepMerge( base as unknown as Record, overrides as DeepPartial> - ) as unknown as AirdropEvent; + ) as unknown as AirSyncEvent; // Ensure mock server URLs always win over overrides, unless the caller // explicitly provided them. diff --git a/src/logger/logger.interfaces.ts b/src/logger/logger.interfaces.ts index 9c39bde9..b8f24944 100644 --- a/src/logger/logger.interfaces.ts +++ b/src/logger/logger.interfaces.ts @@ -1,9 +1,9 @@ import type { RawAxiosResponseHeaders } from 'axios'; -import type { AirdropEvent, EventContext } from '../types/extraction'; +import type { AirSyncEvent, EventContext } from '../types/extraction'; import type { WorkerAdapterOptions } from '../types/workers'; export interface LoggerFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } diff --git a/src/mappers/mappers.interface.ts b/src/mappers/mappers.interface.ts index b37272a1..7d03c60b 100644 --- a/src/mappers/mappers.interface.ts +++ b/src/mappers/mappers.interface.ts @@ -1,4 +1,4 @@ -import { AirdropEvent } from '../types'; +import { AirSyncEvent } from '../types'; import { DonV2 } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; @@ -6,7 +6,7 @@ import { WorkerAdapterOptions } from '../types/workers'; * Configuration interface for creating a Mappers instance. */ export interface MappersFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 0350d34d..77432688 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -5,7 +5,7 @@ import { emit } from '../../common/control-protocol'; import { translateIncomingEventType } from '../../common/event-type-translation'; import { getMemoryUsage } from '../../common/helpers'; import { Logger, serializeError } from '../../logger/logger'; -import { AirdropEvent, EventType } from '../../types/extraction'; +import { AirSyncEvent, EventType } from '../../types/extraction'; import { GetWorkerPathInterface, SpawnFactoryInterface, @@ -65,7 +65,7 @@ function getWorkerPath({ * The class provides utilities to emit control events to the platform and exit the worker gracefully. * In case of lambda timeout, the class emits a lambda timeout event to the platform. * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class - * @param {AirdropEvent} options.event - The event object received from the platform + * @param {AirSyncEvent} options.event - The event object received from the platform * @param {object} options.initialState - The initial state of the adapter * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname` @@ -169,7 +169,7 @@ export async function spawn({ } export class Spawn { - private event: AirdropEvent; + private event: AirSyncEvent; private alreadyEmitted: boolean; private softTimeoutSent: boolean; private defaultLambdaTimeout: number = DEFAULT_LAMBDA_TIMEOUT; @@ -269,7 +269,7 @@ export class Spawn { // If worker sends a message that it has emitted an event, then set alreadyEmitted to true. if (message?.subject === WorkerMessageSubject.WorkerMessageEmitted) { - console.info('Worker has emitted message to ADaaS.'); + console.info('Worker has emitted message to AirSync.'); this.alreadyEmitted = true; } diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts index 81491292..d938287e 100644 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ b/src/multithreading/worker-adapter/worker-adapter.ts @@ -27,7 +27,7 @@ import { import { State } from '../../state/state'; import { AdapterState } from '../../state/state.interfaces'; import { - AirdropEvent, + AirSyncEvent, EventData, EventType, ExternalSystemAttachmentProcessors, @@ -74,7 +74,7 @@ export function createWorkerAdapter({ } /** - * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in + * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in * as parameter in processTask and onTimeout functions. The class provides * utilities to emit control events to the platform, update the state of the connector, * and upload artifacts to the platform. @@ -89,7 +89,7 @@ export function createWorkerAdapter({ * @public */ export class WorkerAdapter { - readonly event: AirdropEvent; + readonly event: AirSyncEvent; readonly options?: WorkerAdapterOptions; isTimeout: boolean; hasWorkerEmitted: boolean; diff --git a/src/repo/repo.interfaces.ts b/src/repo/repo.interfaces.ts index 4573a3a0..f790be2a 100644 --- a/src/repo/repo.interfaces.ts +++ b/src/repo/repo.interfaces.ts @@ -1,6 +1,6 @@ import { Artifact } from '../uploader/uploader.interfaces'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; /** @@ -16,7 +16,7 @@ export interface RepoInterface { * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo. */ export interface RepoFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; itemType: string; normalize?: (record: object) => NormalizedItem | NormalizedAttachment; onUpload: (artifact: Artifact) => void; diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index f3c05929..6c28cf34 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -1,5 +1,5 @@ import { InitialDomainMapping } from '../types/common'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { FileToLoad } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; @@ -56,7 +56,7 @@ export interface FromDevRev { } export interface StateInterface { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; initialDomainMapping?: InitialDomainMapping; options?: WorkerAdapterOptions; diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 0c328de8..c902e0bd 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -10,7 +10,7 @@ import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; import { DonV2, LoaderReport, RateLimited } from './loading'; /** - * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS. + * EventType is an enum that defines the different types of events that can be sent to the external extractor from AirSync. * The external extractor can use these events to know what to do next in the extraction process. */ export enum EventType { @@ -71,8 +71,8 @@ export enum EventType { } /** - * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS. - * The external extractor can use these events to inform ADaaS about the progress of the extraction process. + * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to AirSync. + * The external extractor can use these events to inform AirSync about the progress of the extraction process. */ export enum ExtractorEventType { // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) @@ -248,7 +248,7 @@ export interface TimeValue { } /** - * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from ADaaS. + * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from AirSync. * @deprecated */ export interface EventContextIn { @@ -276,7 +276,7 @@ export interface EventContextIn { } /** - * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to ADaaS. + * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to AirSync. * @deprecated */ export interface EventContextOut { @@ -286,7 +286,7 @@ export interface EventContextOut { } /** - * EventContext is an interface that defines the structure of the event context that is sent to the external connector from Airdrop. + * EventContext is an interface that defines the structure of the event context that is sent to the external connector from AirSync. */ export interface EventContext { callback_url: string; @@ -378,7 +378,7 @@ export interface EventContext { } /** - * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from ADaaS. + * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from AirSync. * It contains the organization ID, organization name, key, and key type. */ export interface ConnectionData { @@ -389,7 +389,7 @@ export interface ConnectionData { } /** - * EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS. + * EventData is an interface that defines the structure of the event data that is sent from the external extractor to AirSync. */ export interface EventData { /** @@ -416,7 +416,7 @@ export interface EventData { } /** - * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS. + * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to AirSync. */ export interface WorkerMetadata { adaas_library_version: string; @@ -439,10 +439,10 @@ export interface DomainObjectState { } /** - * AirdropEvent is an interface that defines the structure of the event that is sent to the external extractor from ADaaS. + * AirSyncEvent is an interface that defines the structure of the event that is sent to the external extractor from AirSync. * It contains the context, payload, execution metadata, and input data as common snap-ins. */ -export interface AirdropEvent { +export interface AirSyncEvent { context: { secrets: { service_account_token: string; @@ -450,7 +450,7 @@ export interface AirdropEvent { snap_in_version_id: string; snap_in_id: string; }; - payload: AirdropMessage; + payload: AirSyncMessage; execution_metadata: { devrev_endpoint: string; }; @@ -458,9 +458,9 @@ export interface AirdropEvent { } /** - * AirdropMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from ADaaS. + * AirSyncMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from AirSync. */ -export interface AirdropMessage { +export interface AirSyncMessage { connection_data: ConnectionData; event_context: EventContext; event_type: EventType; @@ -468,7 +468,7 @@ export interface AirdropMessage { } /** - * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to ADaaS. + * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to AirSync. * It contains the event type, event context, extractor state, and event data. */ export interface ExtractorEvent { @@ -495,7 +495,7 @@ export type ExternalSystemAttachmentStreamingFunction = ({ export interface ExternalSystemAttachmentStreamingParams { item: NormalizedAttachment; - event: AirdropEvent; + event: AirSyncEvent; } export interface ExternalSystemAttachmentStreamingResponse { diff --git a/src/types/index.ts b/src/types/index.ts index f49ef309..f98c7251 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -10,8 +10,8 @@ export { // Extraction export { - AirdropEvent, - AirdropMessage, + AirSyncEvent, + AirSyncMessage, ConnectionData, DomainObjectState, EventContextIn, diff --git a/src/types/loading.ts b/src/types/loading.ts index b08fb1cb..3ab48aef 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -1,6 +1,6 @@ import { Mappers } from '../mappers/mappers'; import { ErrorRecord } from './common'; -import { AirdropEvent } from './extraction'; +import { AirSyncEvent } from './extraction'; export interface StatsFileObject { id: string; @@ -60,7 +60,7 @@ export interface ExternalSystemItem { export interface ExternalSystemItemLoadingParams { item: Type; mappers: Mappers; - event: AirdropEvent; + event: AirSyncEvent; } export interface ExternalSystemItemLoadingResponse { diff --git a/src/types/workers.ts b/src/types/workers.ts index 4123c291..9e3ed259 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -4,7 +4,7 @@ import type { LogLevel } from '../logger/logger.interfaces'; import { State } from '../state/state'; import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; -import { AirdropEvent, EventType, ExtractorEventType } from './extraction'; +import { AirSyncEvent, EventType, ExtractorEventType } from './extraction'; import { LoaderEventType } from './loading'; @@ -14,12 +14,12 @@ import { InitialDomainMapping } from './common'; * WorkerAdapterInterface is an interface for WorkerAdapter class. * @interface WorkerAdapterInterface * @constructor - * @param {AirdropEvent} event - The event object received from the platform + * @param {AirSyncEvent} event - The event object received from the platform * @param {object=} initialState - The initial state of the adapter * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class */ export interface WorkerAdapterInterface { - event: AirdropEvent; + event: AirSyncEvent; adapterState: State; options?: WorkerAdapterOptions; } @@ -51,11 +51,11 @@ export interface WorkerAdapterOptions { * SpawnInterface is an interface for Spawn class. * @interface SpawnInterface * @constructor - * @param {AirdropEvent} event - The event object received from the platform + * @param {AirSyncEvent} event - The event object received from the platform * @param {Worker} worker - The worker thread */ export interface SpawnInterface { - event: AirdropEvent; + event: AirSyncEvent; worker: Worker; options?: WorkerAdapterOptions; resolve: (value: void | PromiseLike) => void; @@ -69,7 +69,7 @@ export interface SpawnInterface { * In case of lambda timeout, the class emits a lambda timeout event to the platform. * @interface SpawnFactoryInterface * @constructor - * @param {AirdropEvent} event - The event object received from the platform + * @param {AirSyncEvent} event - The event object received from the platform * @param {object=} initialState - The initial state of the adapter * @param {string} workerPath - The path to the worker file * @param {string} initialDomainMapping - The initial domain mapping @@ -77,7 +77,7 @@ export interface SpawnInterface { * @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname` */ export interface SpawnFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; /** @deprecated Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath */ @@ -181,7 +181,7 @@ export type WorkerMessage = * WorkerData represents the structure of the worker data object. */ export interface WorkerData { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; workerPath: string; initialDomainMapping?: InitialDomainMapping; @@ -192,7 +192,7 @@ export interface WorkerData { * GetWorkerPathInterface is an interface for getting the worker path. */ export interface GetWorkerPathInterface { - event: AirdropEvent; + event: AirSyncEvent; workerBasePath?: string | null; } diff --git a/src/uploader/uploader.interfaces.ts b/src/uploader/uploader.interfaces.ts index ce995a75..2224bed7 100644 --- a/src/uploader/uploader.interfaces.ts +++ b/src/uploader/uploader.interfaces.ts @@ -1,10 +1,10 @@ import { ErrorRecord } from '../types/common'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { ExternalSystemItem, StatsFileObject } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; export interface UploaderFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } @@ -17,7 +17,7 @@ export type UploaderResult = | { response?: never; error: unknown }; /** - * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS. + * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to AirSync. */ export interface Artifact { id: string; From 8d091594b5e64be4729981f3f288d55a9aa20445 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:20:28 +0200 Subject: [PATCH 07/30] docs: mark C2 done Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 6c9ab35f..ed65b4bb 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -48,9 +48,15 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi (`process-task.ts`, `spawn.ts`, `worker-adapter.ts`, `control-protocol.ts`) imports it, so it can only be removed alongside the old enum members it translates (C3). Deleting it here would break the build. -- **C2 — Airdrop→AirSync identifier rename.** SDK identifiers/types/classes/comments only. - NOT API route strings (rule 3). e.g. `AirdropEvent`→`AirSyncEvent`, `AirdropMessage`→`AirSyncMessage` - (verify exact target names against `origin/v2`). Provide back-compat type aliases ONLY if origin/v2 did. +- **C2 — Airdrop→AirSync identifier rename.** DECIDED with Rado: + - HARD rename, NO back-compat alias: `AirdropEvent`→`AirSyncEvent`, `AirdropMessage`→`AirSyncMessage`. + (origin/v2 did NOT do this rename — oracle unreliable here; do it properly.) + - Update stale branding in comments/prose: bare "Airdrop" + "ADaaS" → "AirSync". + - MUST NOT touch: `/internal/airdrop.*` API routes; the `AIRDROP_*` mapping enum members AND their + `'airdrop_*'` string values (mappers.interface.ts); the `external_system_type: 'ADaaS'` string LITERAL + (platform contract — keep quotes-protected). + - PRODUCTION scope only: exclude `**/*.test.ts` AND `src/tests/**` (test scaffolding renamed later in C10). + - Connectors all import AirdropEvent → breaking; captured for migrate-v2 skill. - **C3 — Remove old event-type compatibility layer** (NOT a rename — main carries old+new side by side; drop old). - Delete deprecated enum members, leaving only the new ones (tables below). Files: `src/types/extraction.ts`, `src/types/loading.ts`. @@ -181,7 +187,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: |--------|-------|-------| | C0 package rename | ☑ done | 8ddeb87. @devrev/ts-adaas→@devrev/airsync-sdk, v2.0.0-beta.0. Report filename rename deferred to C8. | | C1 delete + tsconfig | ☑ done | d573cb6. Deleted src/deprecated/ (6 files) + 4 index exports; added tsconfig.build.json (excludes tests), build script points to it. Reviewer-approved. event-type-translation deletion moved to C3. | -| C2 AirSync rename | ☐ todo | | +| C2 AirSync rename | ☑ done | 1fa9afc. AirdropEvent→AirSyncEvent, AirdropMessage→AirSyncMessage (hard, no alias) + prose ADaaS/Airdrop→AirSync. Protected: airdrop.* routes, AIRDROP_* enum, 'ADaaS' literal. Reviewer-approved. | | C3 enum cleanup | ☐ todo | | | C4a state split | ☐ todo | | | C4b state envelope | ☐ todo | | From cc05f41fdcc74548fcac43c33d6cd2f971b9e4ae Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:55:30 +0200 Subject: [PATCH 08/30] refactor(v2)!: remove old event-type compatibility layer The backend now sends and accepts only the new event-type values, so the old->new translation indirection and the deprecated enum members are no longer needed. - Delete deprecated members from EventType and ExtractorEventType (the EXTRACTION_* values) and the typo/plural dupes from LoaderEventType (DataLoadingDelay, AttachmentsLoading*), keeping only the new members. - Delete src/common/event-type-translation.ts and its test. - Rewire the four callers to use event types directly with no translation: process-task.ts, spawn.ts (incoming), control-protocol.ts emit() and worker-adapter.ts emit() (outgoing). - Drop the now-dead old-member case arms in spawn.helpers.ts (return values unchanged). Behavior is unchanged: translation was identity for new values, which are now the only values on the wire. BREAKING CHANGE: removed deprecated enum members. Connectors must use the new event-type members (e.g. EventType.StartExtractingMetadata instead of EventType.ExtractionMetadataStart; LoaderEventType.DataLoadingDelayed instead of DataLoadingDelay). Ref: V2_PROGRESS.md C3 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/common/control-protocol.ts | 7 +- src/common/event-type-translation.test.ts | 193 ------------------ src/common/event-type-translation.ts | 163 --------------- src/multithreading/process-task.ts | 6 - src/multithreading/spawn/spawn.helpers.ts | 20 +- src/multithreading/spawn/spawn.ts | 18 +- .../worker-adapter/worker-adapter.ts | 3 - src/types/extraction.ts | 126 ++---------- src/types/loading.ts | 24 +-- 9 files changed, 23 insertions(+), 537 deletions(-) delete mode 100644 src/common/event-type-translation.test.ts delete mode 100644 src/common/event-type-translation.ts diff --git a/src/common/control-protocol.ts b/src/common/control-protocol.ts index 6116de9b..0d13be13 100644 --- a/src/common/control-protocol.ts +++ b/src/common/control-protocol.ts @@ -9,7 +9,6 @@ import { } from '../types/extraction'; import { LoaderEventType } from '../types/loading'; import { LIBRARY_VERSION } from './constants'; -import { translateOutgoingEventType } from './event-type-translation'; export interface EmitInterface { event: AirSyncEvent; @@ -22,12 +21,8 @@ export const emit = async ({ eventType, data, }: EmitInterface): Promise => { - // Translate outgoing event type to ensure we always send new event types - // TODO: Remove when the old types are completely phased out - const translatedEventType = translateOutgoingEventType(eventType); - const newEvent: ExtractorEvent | LoaderEvent = { - event_type: translatedEventType, + event_type: eventType, event_context: event.payload.event_context, event_data: { ...data, diff --git a/src/common/event-type-translation.test.ts b/src/common/event-type-translation.test.ts deleted file mode 100644 index 85b4825b..00000000 --- a/src/common/event-type-translation.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; -import { - translateExtractorEventType, - translateIncomingEventType, - translateLoaderEventType, - translateOutgoingEventType, -} from './event-type-translation'; - -describe(translateIncomingEventType.name, () => { - it.each([ - [ - EventType.ExtractionExternalSyncUnitsStart, - EventType.StartExtractingExternalSyncUnits, - ], - [EventType.ExtractionMetadataStart, EventType.StartExtractingMetadata], - [EventType.ExtractionDataStart, EventType.StartExtractingData], - [EventType.ExtractionDataContinue, EventType.ContinueExtractingData], - [EventType.ExtractionDataDelete, EventType.StartDeletingExtractorState], - [ - EventType.ExtractionAttachmentsStart, - EventType.StartExtractingAttachments, - ], - [ - EventType.ExtractionAttachmentsContinue, - EventType.ContinueExtractingAttachments, - ], - [ - EventType.ExtractionAttachmentsDelete, - EventType.StartDeletingExtractorAttachmentsState, - ], - ])('maps legacy extraction event %s to %s', (legacy, modern) => { - expect(translateIncomingEventType(legacy)).toBe(modern); - }); - - it.each([ - [EventType.StartExtractingExternalSyncUnits], - [EventType.StartExtractingMetadata], - [EventType.StartExtractingData], - [EventType.ContinueExtractingData], - [EventType.StartDeletingExtractorState], - [EventType.StartExtractingAttachments], - [EventType.ContinueExtractingAttachments], - [EventType.StartDeletingExtractorAttachmentsState], - [EventType.StartLoadingData], - [EventType.ContinueLoadingData], - [EventType.StartLoadingAttachments], - [EventType.ContinueLoadingAttachments], - [EventType.StartDeletingLoaderState], - [EventType.StartDeletingLoaderAttachmentState], - [EventType.UnknownEventType], - ])('is a no-op for already-modern event type %s', (eventType) => { - expect(translateIncomingEventType(eventType)).toBe(eventType); - }); - - it('returns the input verbatim for an unrecognised event type', () => { - const result = translateIncomingEventType('NONSENSE_EVENT' as EventType); - - expect(result).toBe('NONSENSE_EVENT'); - }); -}); - -describe(translateExtractorEventType.name, () => { - it.each([ - [ - ExtractorEventType.ExtractionExternalSyncUnitsDone, - ExtractorEventType.ExternalSyncUnitExtractionDone, - ], - [ - ExtractorEventType.ExtractionExternalSyncUnitsError, - ExtractorEventType.ExternalSyncUnitExtractionError, - ], - [ - ExtractorEventType.ExtractionMetadataDone, - ExtractorEventType.MetadataExtractionDone, - ], - [ - ExtractorEventType.ExtractionMetadataError, - ExtractorEventType.MetadataExtractionError, - ], - [ - ExtractorEventType.ExtractionDataProgress, - ExtractorEventType.DataExtractionProgress, - ], - [ - ExtractorEventType.ExtractionDataDelay, - ExtractorEventType.DataExtractionDelayed, - ], - [ - ExtractorEventType.ExtractionDataDone, - ExtractorEventType.DataExtractionDone, - ], - [ - ExtractorEventType.ExtractionDataError, - ExtractorEventType.DataExtractionError, - ], - [ - ExtractorEventType.ExtractionDataDeleteDone, - ExtractorEventType.ExtractorStateDeletionDone, - ], - [ - ExtractorEventType.ExtractionDataDeleteError, - ExtractorEventType.ExtractorStateDeletionError, - ], - [ - ExtractorEventType.ExtractionAttachmentsProgress, - ExtractorEventType.AttachmentExtractionProgress, - ], - [ - ExtractorEventType.ExtractionAttachmentsDelay, - ExtractorEventType.AttachmentExtractionDelayed, - ], - [ - ExtractorEventType.ExtractionAttachmentsDone, - ExtractorEventType.AttachmentExtractionDone, - ], - [ - ExtractorEventType.ExtractionAttachmentsError, - ExtractorEventType.AttachmentExtractionError, - ], - [ - ExtractorEventType.ExtractionAttachmentsDeleteDone, - ExtractorEventType.ExtractorAttachmentsStateDeletionDone, - ], - [ - ExtractorEventType.ExtractionAttachmentsDeleteError, - ExtractorEventType.ExtractorAttachmentsStateDeletionError, - ], - ])('maps legacy extractor event %s to %s', (legacy, modern) => { - expect(translateExtractorEventType(legacy)).toBe(modern); - }); - - it.each([ - [ExtractorEventType.DataExtractionDone], - [ExtractorEventType.DataExtractionProgress], - [ExtractorEventType.AttachmentExtractionDone], - [ExtractorEventType.MetadataExtractionDone], - [ExtractorEventType.UnknownEventType], - ])('is a no-op for already-modern extractor event %s', (eventType) => { - expect(translateExtractorEventType(eventType)).toBe(eventType); - }); -}); - -describe(translateLoaderEventType.name, () => { - it.each([ - [LoaderEventType.DataLoadingDelay, LoaderEventType.DataLoadingDelayed], - [ - LoaderEventType.AttachmentsLoadingProgress, - LoaderEventType.AttachmentLoadingProgress, - ], - [ - LoaderEventType.AttachmentsLoadingDelayed, - LoaderEventType.AttachmentLoadingDelayed, - ], - [ - LoaderEventType.AttachmentsLoadingDone, - LoaderEventType.AttachmentLoadingDone, - ], - [ - LoaderEventType.AttachmentsLoadingError, - LoaderEventType.AttachmentLoadingError, - ], - ])('maps legacy loader event %s to %s', (legacy, modern) => { - expect(translateLoaderEventType(legacy)).toBe(modern); - }); - - it.each([ - [LoaderEventType.DataLoadingDone], - [LoaderEventType.DataLoadingProgress], - [LoaderEventType.AttachmentLoadingDone], - ])('is a no-op for already-modern loader event %s', (eventType) => { - expect(translateLoaderEventType(eventType)).toBe(eventType); - }); -}); - -describe(translateOutgoingEventType.name, () => { - it('routes extractor events through translateExtractorEventType', () => { - expect( - translateOutgoingEventType(ExtractorEventType.ExtractionDataDone) - ).toBe(ExtractorEventType.DataExtractionDone); - }); - - it('routes loader events through translateLoaderEventType', () => { - expect( - translateOutgoingEventType(LoaderEventType.AttachmentsLoadingDone) - ).toBe(LoaderEventType.AttachmentLoadingDone); - }); - - it('passes through unknown event types unchanged', () => { - const unknown = 'SOME_UNKNOWN_EVENT' as ExtractorEventType; - expect(translateOutgoingEventType(unknown)).toBe(unknown); - }); -}); diff --git a/src/common/event-type-translation.ts b/src/common/event-type-translation.ts deleted file mode 100644 index f1a32abe..00000000 --- a/src/common/event-type-translation.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; - -/** - * Maps old incoming event type strings to new EventType enum values. - * This ensures backwards compatibility when the platform sends old event types. - * @param eventTypeString The raw event type string from the platform - * @returns The translated EventType enum value - */ -export function translateIncomingEventType(eventTypeString: string): EventType { - // Create a reverse mapping from OLD string values to NEW enum member names - const eventTypeMap: Record = { - // Old extraction event types from platform -> New enum members - [EventType.ExtractionExternalSyncUnitsStart]: - EventType.StartExtractingExternalSyncUnits, - [EventType.ExtractionMetadataStart]: EventType.StartExtractingMetadata, - [EventType.ExtractionDataStart]: EventType.StartExtractingData, - [EventType.ExtractionDataContinue]: EventType.ContinueExtractingData, - [EventType.ExtractionDataDelete]: EventType.StartDeletingExtractorState, - [EventType.ExtractionAttachmentsStart]: - EventType.StartExtractingAttachments, - [EventType.ExtractionAttachmentsContinue]: - EventType.ContinueExtractingAttachments, - [EventType.ExtractionAttachmentsDelete]: - EventType.StartDeletingExtractorAttachmentsState, - - // New extraction event types (already correct, map to new enum members) - [EventType.StartExtractingExternalSyncUnits]: - EventType.StartExtractingExternalSyncUnits, - [EventType.StartExtractingMetadata]: EventType.StartExtractingMetadata, - [EventType.StartExtractingData]: EventType.StartExtractingData, - [EventType.ContinueExtractingData]: EventType.ContinueExtractingData, - [EventType.StartDeletingExtractorState]: - EventType.StartDeletingExtractorState, - [EventType.StartExtractingAttachments]: - EventType.StartExtractingAttachments, - [EventType.ContinueExtractingAttachments]: - EventType.ContinueExtractingAttachments, - [EventType.StartDeletingExtractorAttachmentsState]: - EventType.StartDeletingExtractorAttachmentsState, - - // Loading events - [EventType.StartLoadingData]: EventType.StartLoadingData, - [EventType.ContinueLoadingData]: EventType.ContinueLoadingData, - [EventType.StartLoadingAttachments]: EventType.StartLoadingAttachments, - [EventType.ContinueLoadingAttachments]: - EventType.ContinueLoadingAttachments, - [EventType.StartDeletingLoaderState]: EventType.StartDeletingLoaderState, - [EventType.StartDeletingLoaderAttachmentState]: - EventType.StartDeletingLoaderAttachmentState, - - // Unknown - [EventType.UnknownEventType]: EventType.UnknownEventType, - }; - - const translated = eventTypeMap[eventTypeString]; - if (!translated) { - console.warn( - `Unknown event type received: ${eventTypeString}. This may indicate a new event type or a typo.` - ); - // Return the original string cast as EventType as a fallback - return eventTypeString as EventType; - } - - return translated; -} - -/** - * Translates ExtractorEventType enum values by converting old enum members to new ones. - * Old enum members are deprecated and should be replaced with new ones. - */ -export function translateExtractorEventType( - eventType: ExtractorEventType -): ExtractorEventType { - // Map old enum members to new enum members - const stringValue = eventType as string; - - const mapping: Record = { - // Old string values -> New enum members - [ExtractorEventType.ExtractionExternalSyncUnitsDone]: - ExtractorEventType.ExternalSyncUnitExtractionDone, - [ExtractorEventType.ExtractionExternalSyncUnitsError]: - ExtractorEventType.ExternalSyncUnitExtractionError, - [ExtractorEventType.ExtractionMetadataDone]: - ExtractorEventType.MetadataExtractionDone, - [ExtractorEventType.ExtractionMetadataError]: - ExtractorEventType.MetadataExtractionError, - [ExtractorEventType.ExtractionDataProgress]: - ExtractorEventType.DataExtractionProgress, - [ExtractorEventType.ExtractionDataDelay]: - ExtractorEventType.DataExtractionDelayed, - [ExtractorEventType.ExtractionDataDone]: - ExtractorEventType.DataExtractionDone, - [ExtractorEventType.ExtractionDataError]: - ExtractorEventType.DataExtractionError, - [ExtractorEventType.ExtractionDataDeleteDone]: - ExtractorEventType.ExtractorStateDeletionDone, - [ExtractorEventType.ExtractionDataDeleteError]: - ExtractorEventType.ExtractorStateDeletionError, - [ExtractorEventType.ExtractionAttachmentsProgress]: - ExtractorEventType.AttachmentExtractionProgress, - [ExtractorEventType.ExtractionAttachmentsDelay]: - ExtractorEventType.AttachmentExtractionDelayed, - [ExtractorEventType.ExtractionAttachmentsDone]: - ExtractorEventType.AttachmentExtractionDone, - [ExtractorEventType.ExtractionAttachmentsError]: - ExtractorEventType.AttachmentExtractionError, - [ExtractorEventType.ExtractionAttachmentsDeleteDone]: - ExtractorEventType.ExtractorAttachmentsStateDeletionDone, - [ExtractorEventType.ExtractionAttachmentsDeleteError]: - ExtractorEventType.ExtractorAttachmentsStateDeletionError, - }; - - // If there's a mapping, use it; otherwise return original (already new) - return mapping[stringValue] ?? eventType; -} - -/** - * Translates LoaderEventType enum values by converting old enum members to new ones. - * Old enum members are deprecated and should be replaced with new ones. - */ -export function translateLoaderEventType( - eventType: LoaderEventType -): LoaderEventType { - // Map old enum members to new enum members - const stringValue = eventType as string; - - const mapping: Record = { - // Old string values -> New enum members - [LoaderEventType.DataLoadingDelay]: LoaderEventType.DataLoadingDelayed, - [LoaderEventType.AttachmentsLoadingProgress]: - LoaderEventType.AttachmentLoadingProgress, - [LoaderEventType.AttachmentsLoadingDelayed]: - LoaderEventType.AttachmentLoadingDelayed, - [LoaderEventType.AttachmentsLoadingDone]: - LoaderEventType.AttachmentLoadingDone, - [LoaderEventType.AttachmentsLoadingError]: - LoaderEventType.AttachmentLoadingError, - }; - - // If there's a mapping, use it; otherwise return original (already new) - return mapping[stringValue] ?? eventType; -} - -/** - * Translates any outgoing event type (Extractor or Loader) to ensure new event types are used. - */ -export function translateOutgoingEventType( - eventType: ExtractorEventType | LoaderEventType -): ExtractorEventType | LoaderEventType { - // Check if it's an ExtractorEventType by checking if the value exists in ExtractorEventType - if ( - Object.values(ExtractorEventType).includes(eventType as ExtractorEventType) - ) { - return translateExtractorEventType(eventType as ExtractorEventType); - } - // Otherwise treat as LoaderEventType - if (Object.values(LoaderEventType).includes(eventType as LoaderEventType)) { - return translateLoaderEventType(eventType as LoaderEventType); - } - // If neither, return as-is - return eventType; -} diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 4f3e75a3..0818f55f 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -1,5 +1,4 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads'; -import { translateIncomingEventType } from '../common/event-type-translation'; import { Logger, serializeError } from '../logger/logger'; import { runWithSdkLogContext, @@ -26,11 +25,6 @@ export function processTask({ try { const event = workerData.event; - // TODO: Remove when the old types are completely phased out - event.payload.event_type = translateIncomingEventType( - event.payload.event_type - ); - const initialState = workerData.initialState as ConnectorState; const initialDomainMapping = workerData.initialDomainMapping; const options = workerData.options; diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index abbe9908..3db03dfb 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -10,48 +10,40 @@ export function getTimeoutErrorEventType(eventType: EventType): { eventType: ExtractorEventType | LoaderEventType; } { switch (eventType) { - // Metadata extraction (handles both old and new enum members) + // Metadata extraction case EventType.StartExtractingMetadata: - case EventType.ExtractionMetadataStart: return { eventType: ExtractorEventType.MetadataExtractionError, }; - // Data extraction (handles both old and new enum members) + // Data extraction case EventType.StartExtractingData: case EventType.ContinueExtractingData: - case EventType.ExtractionDataStart: - case EventType.ExtractionDataContinue: return { eventType: ExtractorEventType.DataExtractionError, }; - // Data deletion (handles both old and new enum members) + // Data deletion case EventType.StartDeletingExtractorState: - case EventType.ExtractionDataDelete: return { eventType: ExtractorEventType.ExtractorStateDeletionError, }; - // Attachments extraction (handles both old and new enum members) + // Attachments extraction case EventType.StartExtractingAttachments: case EventType.ContinueExtractingAttachments: - case EventType.ExtractionAttachmentsStart: - case EventType.ExtractionAttachmentsContinue: return { eventType: ExtractorEventType.AttachmentExtractionError, }; - // Attachments deletion (handles both old and new enum members) + // Attachments deletion case EventType.StartDeletingExtractorAttachmentsState: - case EventType.ExtractionAttachmentsDelete: return { eventType: ExtractorEventType.ExtractorAttachmentsStateDeletionError, }; - // External sync units (handles both old and new enum members) + // External sync units case EventType.StartExtractingExternalSyncUnits: - case EventType.ExtractionExternalSyncUnitsStart: return { eventType: ExtractorEventType.ExternalSyncUnitExtractionError, }; diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 77432688..02176ff0 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -2,7 +2,6 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import { emit } from '../../common/control-protocol'; -import { translateIncomingEventType } from '../../common/event-type-translation'; import { getMemoryUsage } from '../../common/helpers'; import { Logger, serializeError } from '../../logger/logger'; import { AirSyncEvent, EventType } from '../../types/extraction'; @@ -79,14 +78,6 @@ export async function spawn({ options, baseWorkerPath, }: SpawnFactoryInterface): Promise { - // Translate incoming event type for backwards compatibility. This allows the - // SDK to accept both old and new event type formats. Then update the event with the translated event type. - const originalEventType = event.payload.event_type; - const translatedEventType = translateIncomingEventType( - event.payload.event_type as string - ); - event.payload.event_type = translatedEventType; - // Read the command line arguments to check if the local flag is passed. const argv = await yargs(hideBin(process.argv)).argv; if (argv._.includes('local') || argv.local) { @@ -100,11 +91,6 @@ export async function spawn({ // eslint-disable-next-line no-global-assign console = new Logger({ event, options }); - if (translatedEventType !== originalEventType) { - console.log( - `Event type translated from ${originalEventType} to ${translatedEventType}.` - ); - } if (options?.isLocalDevelopment) { console.log('Snap-in is running in local development mode.'); } @@ -115,11 +101,11 @@ export async function spawn({ } else if ( baseWorkerPath != null && options?.workerPathOverrides != null && - options.workerPathOverrides[translatedEventType as EventType] != null + options.workerPathOverrides[event.payload.event_type as EventType] != null ) { script = baseWorkerPath + - options.workerPathOverrides[translatedEventType as EventType]; + options.workerPathOverrides[event.payload.event_type as EventType]; } else { script = getWorkerPath({ event, diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts index d938287e..b89d889c 100644 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ b/src/multithreading/worker-adapter/worker-adapter.ts @@ -58,7 +58,6 @@ import { } from '../../types/workers'; import { Uploader } from '../../uploader/uploader'; import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; -import { translateOutgoingEventType } from '../../common/event-type-translation'; import { truncateMessage } from '../../common/helpers'; export function createWorkerAdapter({ @@ -244,8 +243,6 @@ export class WorkerAdapter { data?: EventData ): Promise { return runWithSdkLogContext(async () => { - newEventType = translateOutgoingEventType(newEventType); - if (this.hasWorkerEmitted) { console.warn( `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` diff --git a/src/types/extraction.ts b/src/types/extraction.ts index c902e0bd..34189d04 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -14,39 +14,15 @@ import { DonV2, LoaderReport, RateLimited } from './loading'; * The external extractor can use these events to know what to do next in the extraction process. */ export enum EventType { - // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) - /** - * @deprecated Use StartExtractingExternalSyncUnits instead - */ - ExtractionExternalSyncUnitsStart = 'EXTRACTION_EXTERNAL_SYNC_UNITS_START', - /** - * @deprecated Use StartExtractingMetadata instead - */ - ExtractionMetadataStart = 'EXTRACTION_METADATA_START', - /** - * @deprecated Use StartExtractingData instead - */ - ExtractionDataStart = 'EXTRACTION_DATA_START', - /** - * @deprecated Use ContinueExtractingData instead - */ - ExtractionDataContinue = 'EXTRACTION_DATA_CONTINUE', - /** - * @deprecated Use StartDeletingExtractorState instead - */ - ExtractionDataDelete = 'EXTRACTION_DATA_DELETE', - /** - * @deprecated Use StartExtractingAttachments instead - */ - ExtractionAttachmentsStart = 'EXTRACTION_ATTACHMENTS_START', - /** - * @deprecated Use ContinueExtractingAttachments instead - */ - ExtractionAttachmentsContinue = 'EXTRACTION_ATTACHMENTS_CONTINUE', - /** - * @deprecated Use StartDeletingExtractorAttachmentsState instead - */ - ExtractionAttachmentsDelete = 'EXTRACTION_ATTACHMENTS_DELETE', + // Extraction + StartExtractingExternalSyncUnits = 'START_EXTRACTING_EXTERNAL_SYNC_UNITS', + StartExtractingMetadata = 'START_EXTRACTING_METADATA', + StartExtractingData = 'START_EXTRACTING_DATA', + ContinueExtractingData = 'CONTINUE_EXTRACTING_DATA', + StartDeletingExtractorState = 'START_DELETING_EXTRACTOR_STATE', + StartExtractingAttachments = 'START_EXTRACTING_ATTACHMENTS', + ContinueExtractingAttachments = 'CONTINUE_EXTRACTING_ATTACHMENTS', + StartDeletingExtractorAttachmentsState = 'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE', // Loading StartLoadingData = 'START_LOADING_DATA', @@ -58,16 +34,6 @@ export enum EventType { // Unknown UnknownEventType = 'UNKNOWN_EVENT_TYPE', - - // Extraction - New member names with NEW values (preferred) - StartExtractingExternalSyncUnits = 'START_EXTRACTING_EXTERNAL_SYNC_UNITS', - StartExtractingMetadata = 'START_EXTRACTING_METADATA', - StartExtractingData = 'START_EXTRACTING_DATA', - ContinueExtractingData = 'CONTINUE_EXTRACTING_DATA', - StartDeletingExtractorState = 'START_DELETING_EXTRACTOR_STATE', - StartExtractingAttachments = 'START_EXTRACTING_ATTACHMENTS', - ContinueExtractingAttachments = 'CONTINUE_EXTRACTING_ATTACHMENTS', - StartDeletingExtractorAttachmentsState = 'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE', } /** @@ -75,76 +41,7 @@ export enum EventType { * The external extractor can use these events to inform AirSync about the progress of the extraction process. */ export enum ExtractorEventType { - // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) - /** - * @deprecated Use ExternalSyncUnitExtractionDone instead - */ - ExtractionExternalSyncUnitsDone = 'EXTRACTION_EXTERNAL_SYNC_UNITS_DONE', - /** - * @deprecated Use ExternalSyncUnitExtractionError instead - */ - ExtractionExternalSyncUnitsError = 'EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR', - /** - * @deprecated Use MetadataExtractionDone instead - */ - ExtractionMetadataDone = 'EXTRACTION_METADATA_DONE', - /** - * @deprecated Use MetadataExtractionError instead - */ - ExtractionMetadataError = 'EXTRACTION_METADATA_ERROR', - /** - * @deprecated Use DataExtractionProgress instead - */ - ExtractionDataProgress = 'EXTRACTION_DATA_PROGRESS', - /** - * @deprecated Use DataExtractionDelayed instead - */ - ExtractionDataDelay = 'EXTRACTION_DATA_DELAY', - /** - * @deprecated Use DataExtractionDone instead - */ - ExtractionDataDone = 'EXTRACTION_DATA_DONE', - /** - * @deprecated Use DataExtractionError instead - */ - ExtractionDataError = 'EXTRACTION_DATA_ERROR', - /** - * @deprecated Use ExtractorStateDeletionDone instead - */ - ExtractionDataDeleteDone = 'EXTRACTION_DATA_DELETE_DONE', - /** - * @deprecated Use ExtractorStateDeletionError instead - */ - ExtractionDataDeleteError = 'EXTRACTION_DATA_DELETE_ERROR', - /** - * @deprecated Use AttachmentExtractionProgress instead - */ - ExtractionAttachmentsProgress = 'EXTRACTION_ATTACHMENTS_PROGRESS', - /** - * @deprecated Use AttachmentExtractionDelayed instead - */ - ExtractionAttachmentsDelay = 'EXTRACTION_ATTACHMENTS_DELAY', - /** - * @deprecated Use AttachmentExtractionDone instead - */ - ExtractionAttachmentsDone = 'EXTRACTION_ATTACHMENTS_DONE', - /** - * @deprecated Use AttachmentExtractionError instead - */ - ExtractionAttachmentsError = 'EXTRACTION_ATTACHMENTS_ERROR', - /** - * @deprecated Use ExtractorAttachmentsStateDeletionDone instead - */ - ExtractionAttachmentsDeleteDone = 'EXTRACTION_ATTACHMENTS_DELETE_DONE', - /** - * @deprecated Use ExtractorAttachmentsStateDeletionError instead - */ - ExtractionAttachmentsDeleteError = 'EXTRACTION_ATTACHMENTS_DELETE_ERROR', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - - // Extraction - New member names with NEW values (preferred) + // Extraction ExternalSyncUnitExtractionDone = 'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE', ExternalSyncUnitExtractionError = 'EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR', MetadataExtractionDone = 'METADATA_EXTRACTION_DONE', @@ -161,6 +58,9 @@ export enum ExtractorEventType { AttachmentExtractionError = 'ATTACHMENT_EXTRACTION_ERROR', ExtractorAttachmentsStateDeletionDone = 'EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE', ExtractorAttachmentsStateDeletionError = 'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR', + + // Unknown + UnknownEventType = 'UNKNOWN_EVENT_TYPE', } /** diff --git a/src/types/loading.ts b/src/types/loading.ts index 3ab48aef..37d2d305 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -135,13 +135,9 @@ export type SyncMapperRecord = { input_file?: string; }; -/* eslint-disable @typescript-eslint/no-duplicate-enum-values */ export enum LoaderEventType { DataLoadingProgress = 'DATA_LOADING_PROGRESS', - /** - * @deprecated This was a typo. Use DataLoadingDelayed for the corrected spelling - */ - DataLoadingDelay = 'DATA_LOADING_DELAYED', + DataLoadingDelayed = 'DATA_LOADING_DELAYED', DataLoadingDone = 'DATA_LOADING_DONE', DataLoadingError = 'DATA_LOADING_ERROR', @@ -157,22 +153,4 @@ export enum LoaderEventType { LoaderAttachmentStateDeletionError = 'LOADER_ATTACHMENT_STATE_DELETION_ERROR', UnknownEventType = 'UNKNOWN_EVENT_TYPE', - DataLoadingDelayed = 'DATA_LOADING_DELAYED', - - /** - * @deprecated Use AttachmentsLoadingProgress instead (note: singular changed to plural) - */ - AttachmentsLoadingProgress = 'ATTACHMENT_LOADING_PROGRESS', - /** - * @deprecated Use AttachmentsLoadingDelayed instead (note: singular changed to plural) - */ - AttachmentsLoadingDelayed = 'ATTACHMENT_LOADING_DELAYED', - /** - * @deprecated Use AttachmentsLoadingDone instead (note: singular changed to plural) - */ - AttachmentsLoadingDone = 'ATTACHMENT_LOADING_DONE', - /** - * @deprecated Use AttachmentsLoadingError instead (note: singular changed to plural) - */ - AttachmentsLoadingError = 'ATTACHMENT_LOADING_ERROR', } From 1c15f885a9f0eee1603e7c9e101558874676e90e Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 08:55:40 +0200 Subject: [PATCH 09/30] docs: mark C3 done Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index ed65b4bb..108de424 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -188,7 +188,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C0 package rename | ☑ done | 8ddeb87. @devrev/ts-adaas→@devrev/airsync-sdk, v2.0.0-beta.0. Report filename rename deferred to C8. | | C1 delete + tsconfig | ☑ done | d573cb6. Deleted src/deprecated/ (6 files) + 4 index exports; added tsconfig.build.json (excludes tests), build script points to it. Reviewer-approved. event-type-translation deletion moved to C3. | | C2 AirSync rename | ☑ done | 1fa9afc. AirdropEvent→AirSyncEvent, AirdropMessage→AirSyncMessage (hard, no alias) + prose ADaaS/Airdrop→AirSync. Protected: airdrop.* routes, AIRDROP_* enum, 'ADaaS' literal. Reviewer-approved. | -| C3 enum cleanup | ☐ todo | | +| C3 enum cleanup | ☑ done | cc05f41. Deleted deprecated enum members (EventType, ExtractorEventType, LoaderEventType) + event-type-translation.ts/.test; rewired 4 callers (process-task, spawn, control-protocol, worker-adapter) + spawn.helpers cases. Behavior-equivalent. Reviewer-approved. | | C4a state split | ☐ todo | | | C4b state envelope | ☐ todo | | | C5 adapter split | ☐ todo | | From b63f3aba0db87a1304b19e61407147adea91d160 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 09:10:14 +0200 Subject: [PATCH 10/30] refactor(v2): split State into BaseState + ExtractionState + LoadingState Structural split of the monolithic State class into a shared abstract BaseState plus mode-specific ExtractionState and LoadingState subclasses. The on-disk state shape is unchanged: still the flat AdapterState = ConnectorState & SdkState (the connector/SDK envelope split comes in a later commit). - base-state.ts: abstract BaseState owns the shared lifecycle (init, fetch, postState) and installInitialDomainMappingIfNeeded, extracted from the old createAdapterState factory. - extraction-state.ts: ExtractionState seeds extractionSdkState and adds resolveExtractionWindow (time-value resolution, pending-boundary reuse, lastSyncStarted, window validation); createExtractionState factory. - loading-state.ts: LoadingState seeds loadingSdkState; createLoadingState factory. - state.ts: thin module re-exporting the classes/factories; createAdapterState is now a dispatcher selecting the mode-specific state by event_context.mode. - Consumers (types/workers.ts, worker-adapter.ts) reference BaseState. Behavior is preserved. Loading mode previously ran extraction-window code that was inert for loading events (no matching event types, no pending boundaries in loadingSdkState); routing it to LoadingState only drops those inert log lines. Ref: V2_PROGRESS.md C4a Co-Authored-By: Claude Opus 4.8 (1M context) --- .../worker-adapter/worker-adapter.ts | 4 +- src/state/base-state.ts | 246 ++++++++++++ src/state/extraction-state.ts | 150 ++++++++ src/state/loading-state.ts | 45 +++ src/state/state.ts | 362 ++---------------- src/types/workers.ts | 4 +- 6 files changed, 468 insertions(+), 343 deletions(-) create mode 100644 src/state/base-state.ts create mode 100644 src/state/extraction-state.ts create mode 100644 src/state/loading-state.ts diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts index b89d889c..23e2550a 100644 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ b/src/multithreading/worker-adapter/worker-adapter.ts @@ -24,7 +24,7 @@ import { NormalizedAttachment, RepoInterface, } from '../../repo/repo.interfaces'; -import { State } from '../../state/state'; +import { BaseState } from '../../state/state'; import { AdapterState } from '../../state/state.interfaces'; import { AirSyncEvent, @@ -93,7 +93,7 @@ export class WorkerAdapter { isTimeout: boolean; hasWorkerEmitted: boolean; - private adapterState: State; + private adapterState: BaseState; private _artifacts: Artifact[]; private repos: Repo[] = []; private currentEventDataLength: number = 0; diff --git a/src/state/base-state.ts b/src/state/base-state.ts new file mode 100644 index 00000000..3c9f8a9c --- /dev/null +++ b/src/state/base-state.ts @@ -0,0 +1,246 @@ +import axios from 'axios'; +import { parentPort } from 'node:worker_threads'; + +import { installInitialDomainMapping } from '../common/install-initial-domain-mapping'; +import { axiosClient } from '../http/axios-client-internal'; +import { getPrintableState, serializeError } from '../logger/logger'; +import { InitialDomainMapping } from '../types/common'; +import { AirSyncEvent } from '../types/extraction'; +import { WorkerMessageSubject } from '../types/workers'; +import { ExtractionScope } from '../types/workers'; + +import { AdapterState, SdkState, StateInterface } from './state.interfaces'; + +/** + * BaseState owns the state lifecycle shared by every sync mode: holding the + * adapter state, fetching/initializing/posting it against the platform, and the + * snap-in-version-gated initial domain mapping install. + * + * Mode-specific subclasses (`ExtractionState`, `LoadingState`) seed the + * SDK-owned portion of the state and add mode-specific setup in their factories. + * + * @typeParam ConnectorState - the connector-owned state shape + */ +export abstract class BaseState { + protected _state: AdapterState; + protected _extractionScope: ExtractionScope = {}; + protected readonly initialSdkState: SdkState; + protected readonly event: AirSyncEvent; + private workerUrl: string; + private devrevToken: string; + private syncUnitId: string; + private requestId: string; + + constructor( + { event, initialState }: StateInterface, + initialSdkState: SdkState + ) { + this.event = event; + this.initialSdkState = initialSdkState; + this._state = { + ...initialState, + ...initialSdkState, + } as AdapterState; + this.workerUrl = event.payload.event_context.worker_data_url; + this.devrevToken = event.context.secrets.service_account_token; + this.syncUnitId = event.payload.event_context.sync_unit_id; + this.requestId = event.payload.event_context.request_id_adaas; + } + + get state(): AdapterState { + return this._state; + } + + set state(value: AdapterState) { + this._state = value; + } + + get extractionScope(): ExtractionScope { + return this._extractionScope; + } + + /** + * Installs the initial domain mapping when the snap-in version in state does + * not match the version in the event context. Shared by all modes so that a + * loading run still installs the mapping if extraction has not done so. + * @param initialDomainMapping The initial domain mapping passed to spawn + */ + async installInitialDomainMappingIfNeeded( + initialDomainMapping?: InitialDomainMapping + ): Promise { + const snapInVersionId = this.event.context.snap_in_version_id; + const hasSnapInVersionInState = 'snapInVersionId' in this.state; + const shouldUpdateIDM = + !hasSnapInVersionInState || + this.state.snapInVersionId !== snapInVersionId; + + if (!shouldUpdateIDM) { + console.log( + `Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.` + ); + return; + } + + try { + console.log( + `Snap-in version in state "${this.state.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` + ); + + if (initialDomainMapping) { + await installInitialDomainMapping(this.event, initialDomainMapping); + this.state.snapInVersionId = snapInVersionId; + } else { + throw new Error( + 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' + ); + } + } catch (error) { + const errorMessage = `Error while installing initial domain mapping. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + + /** + * Initializes the state for this adapter instance by fetching from API + * or creating an initial state if none exists (404). + * @param initialState The initial connector state provided by the spawn function + */ + async init(initialState: ConnectorState): Promise { + try { + const { state: stringifiedState, objects } = await this.fetchState(); + if (!stringifiedState) { + throw new Error('No state found in response.'); + } + + let parsedState: AdapterState; + try { + parsedState = JSON.parse(stringifiedState); + } catch (error) { + throw new Error(`Failed to parse state. ${error}`); + } + + this.state = parsedState; + console.log( + 'State fetched successfully. Current state', + getPrintableState(this.state) + ); + + if (objects) { + try { + this._extractionScope = JSON.parse(objects); + } catch (error) { + console.warn(`Failed to parse extractionScope. ${error}`); + } + } + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 404) { + console.log('State not found. Initializing state with initial state.'); + const initialAdapterState: AdapterState = { + ...initialState, + ...this.initialSdkState, + }; + + this.state = initialAdapterState; + await this.postState(initialAdapterState); + } else { + const errorMessage = `Failed to init state. ${serializeError(error)}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } + + /** + * Updates the state of the adapter by posting to API. + * @param {object} state - The state to be updated + */ + async postState(state?: AdapterState) { + const url = this.workerUrl + '.update'; + this.state = state || this.state; + + let stringifiedState: string; + try { + stringifiedState = JSON.stringify(this.state); + } catch (error) { + const errorMessage = `Failed to stringify state. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + + try { + await axiosClient.post( + url, + { + state: stringifiedState, + }, + { + headers: { + Authorization: this.devrevToken, + }, + params: { + sync_unit: this.syncUnitId, + request_id: this.requestId, + }, + } + ); + + console.log( + 'State updated successfully to', + getPrintableState(this.state) + ); + } catch (error) { + const errorMessage = `Failed to update the state. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + + /** + * Fetches the state of the adapter from API. + * @return The raw state data from API + */ + async fetchState(): Promise<{ state: string; objects?: string }> { + console.log( + `Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.` + ); + + const url = this.workerUrl + '.get'; + const response = await axiosClient.get(url, { + headers: { + Authorization: this.devrevToken, + }, + params: { + sync_unit: this.syncUnitId, + request_id: this.requestId, + }, + }); + + return { + state: response.data?.state, + objects: response.data?.objects, + }; + } +} diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts new file mode 100644 index 00000000..bb99a0f7 --- /dev/null +++ b/src/state/extraction-state.ts @@ -0,0 +1,150 @@ +import { parentPort } from 'node:worker_threads'; + +import { STATELESS_EVENT_TYPES } from '../common/constants'; +import { resolveTimeValue } from '../common/time-value-resolver'; +import { serializeError } from '../logger/logger'; +import { EventType } from '../types/extraction'; +import { WorkerMessageSubject } from '../types/workers'; + +import { BaseState } from './base-state'; +import { extractionSdkState, StateInterface } from './state.interfaces'; + +/** + * ExtractionState is the per-mode state for extraction workers. It seeds the + * extraction SDK state (extraction boundaries + attachments bookkeeping) on top + * of the shared lifecycle provided by `BaseState` and adds extraction-window + * resolution. + */ +export class ExtractionState extends BaseState { + constructor(params: StateInterface) { + super(params, extractionSdkState); + } + + /** + * Resolves the extraction window onto the event context. + * + * On StartExtractingData: stamp `lastSyncStarted` if not already set. + * On StartExtractingMetadata: resolve fresh from the TimeValue objects in the + * event context and cache them as pending boundaries (always overwrite). + * On all other events: reuse the pending boundaries cached during + * StartExtractingMetadata. Finally, validate that extract_from < extract_to. + */ + resolveExtractionWindow(): void { + // Set lastSyncStarted if the event type is StartExtractingData + if ( + this.event.payload.event_type === EventType.StartExtractingData && + !this.state.lastSyncStarted + ) { + this.state.lastSyncStarted = new Date().toISOString(); + console.log(`Setting lastSyncStarted to ${this.state.lastSyncStarted}.`); + } + + const eventContext = this.event.payload.event_context; + + if (this.event.payload.event_type === EventType.StartExtractingMetadata) { + const timeFields = [ + { + source: 'extraction_start_time', + target: 'extract_from', + pending: 'pendingWorkersOldest', + }, + { + source: 'extraction_end_time', + target: 'extract_to', + pending: 'pendingWorkersNewest', + }, + ] as const; + + for (const { source, target, pending } of timeFields) { + const timeValue = eventContext[source]; + if (timeValue && timeValue.type) { + try { + const resolved = resolveTimeValue(timeValue, this.state); + eventContext[target] = resolved; + this.state[pending] = resolved; + console.log( + `Resolved ${target} to ${resolved}. Stored in ${pending}.` + ); + } catch (error) { + const errorMessage = `Failed to resolve ${source}: ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } + } else { + // Non-StartExtractingMetadata events: reuse pending values from state + if (this.state.pendingWorkersOldest) { + eventContext.extract_from = this.state.pendingWorkersOldest; + console.log( + `Reusing pendingWorkersOldest as extract_from: ${this.state.pendingWorkersOldest}.` + ); + } else { + console.log( + 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' + ); + } + if (this.state.pendingWorkersNewest) { + eventContext.extract_to = this.state.pendingWorkersNewest; + console.log( + `Reusing pendingWorkersNewest as extract_to: ${this.state.pendingWorkersNewest}.` + ); + } else { + console.log( + 'pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.' + ); + } + } + + // Validate that extract_from is before extract_to + if (eventContext.extract_from && eventContext.extract_to) { + if (eventContext.extract_from >= eventContext.extract_to) { + const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } +} + +/** + * Creates and initializes an `ExtractionState` for an extraction worker. + * + * For non-stateless events this fetches persisted state, installs the initial + * domain mapping if the snap-in version changed, then resolves the extraction + * window (time-value resolution + pending boundary reuse) and validates it. + */ +export async function createExtractionState({ + event, + initialState, + initialDomainMapping, + options, +}: StateInterface): Promise> { + // Deep clone the initial state to avoid mutating the original state + const deepCloneInitialState: ConnectorState = structuredClone(initialState); + + const state = new ExtractionState({ + event, + initialState: deepCloneInitialState, + initialDomainMapping, + options, + }); + + if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { + await state.init(deepCloneInitialState); + await state.installInitialDomainMappingIfNeeded(initialDomainMapping); + state.resolveExtractionWindow(); + } + + return state; +} diff --git a/src/state/loading-state.ts b/src/state/loading-state.ts new file mode 100644 index 00000000..151f5bd9 --- /dev/null +++ b/src/state/loading-state.ts @@ -0,0 +1,45 @@ +import { STATELESS_EVENT_TYPES } from '../common/constants'; + +import { BaseState } from './base-state'; +import { loadingSdkState, StateInterface } from './state.interfaces'; + +/** + * LoadingState is the per-mode state for loading workers. It seeds the loading + * SDK state (files-to-load bookkeeping) on top of the shared lifecycle provided + * by `BaseState`. Loading has no extraction-window resolution. + */ +export class LoadingState extends BaseState { + constructor(params: StateInterface) { + super(params, loadingSdkState); + } +} + +/** + * Creates and initializes a `LoadingState` for a loading worker. + * + * For non-stateless events this fetches persisted state and installs the + * initial domain mapping if the snap-in version changed. + */ +export async function createLoadingState({ + event, + initialState, + initialDomainMapping, + options, +}: StateInterface): Promise> { + // Deep clone the initial state to avoid mutating the original state + const deepCloneInitialState: ConnectorState = structuredClone(initialState); + + const state = new LoadingState({ + event, + initialState: deepCloneInitialState, + initialDomainMapping, + options, + }); + + if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { + await state.init(deepCloneInitialState); + await state.installInitialDomainMappingIfNeeded(initialDomainMapping); + } + + return state; +} diff --git a/src/state/state.ts b/src/state/state.ts index 6f13224c..ab7dbb5c 100644 --- a/src/state/state.ts +++ b/src/state/state.ts @@ -1,342 +1,26 @@ -import axios from 'axios'; -import { parentPort } from 'node:worker_threads'; - -import { STATELESS_EVENT_TYPES } from '../common/constants'; -import { installInitialDomainMapping } from '../common/install-initial-domain-mapping'; -import { resolveTimeValue } from '../common/time-value-resolver'; -import { axiosClient } from '../http/axios-client-internal'; -import { getPrintableState, serializeError } from '../logger/logger'; import { SyncMode } from '../types/common'; -import { EventType } from '../types/extraction'; -import { WorkerMessageSubject } from '../types/workers'; - -import { - AdapterState, - extractionSdkState, - loadingSdkState, - SdkState, - StateInterface, -} from './state.interfaces'; -import { ExtractionScope } from '../types/workers'; - -export async function createAdapterState({ - event, - initialState, - initialDomainMapping, - options, -}: StateInterface): Promise> { - // Deep clone the initial state to avoid mutating the original state - const deepCloneInitialState: ConnectorState = structuredClone(initialState); - - const as = new State({ - event, - initialState: deepCloneInitialState, - initialDomainMapping, - options, - }); - - if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { - await as.init(deepCloneInitialState); - - // Check if IDM needs to be updated - const snapInVersionId = event.context.snap_in_version_id; - const hasSnapInVersionInState = 'snapInVersionId' in as.state; - const shouldUpdateIDM = - !hasSnapInVersionInState || as.state.snapInVersionId !== snapInVersionId; - - if (!shouldUpdateIDM) { - console.log( - `Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.` - ); - } else { - try { - console.log( - `Snap-in version in state "${as.state.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` - ); - - if (initialDomainMapping) { - await installInitialDomainMapping(event, initialDomainMapping); - as.state.snapInVersionId = snapInVersionId; - } else { - throw new Error( - 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' - ); - } - } catch (error) { - const errorMessage = `Error while installing initial domain mapping. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - - // Set lastSyncStarted if the event type is StartExtractingData - if ( - event.payload.event_type === EventType.StartExtractingData && - !as.state.lastSyncStarted - ) { - as.state.lastSyncStarted = new Date().toISOString(); - console.log(`Setting lastSyncStarted to ${as.state.lastSyncStarted}.`); - } - - // Resolve extraction timestamps from TimeValue objects, or reuse pending values from a prior invocation. - // On StartExtractingMetadata: resolve fresh from TimeValue objects and store in pending state (always overwrite). - // On all other events: reuse the pending values cached during StartExtractingMetadata. - const eventContext = event.payload.event_context; - if (event.payload.event_type === EventType.StartExtractingMetadata) { - const timeFields = [ - { - source: 'extraction_start_time', - target: 'extract_from', - pending: 'pendingWorkersOldest', - }, - { - source: 'extraction_end_time', - target: 'extract_to', - pending: 'pendingWorkersNewest', - }, - ] as const; - - for (const { source, target, pending } of timeFields) { - const timeValue = eventContext[source]; - if (timeValue && timeValue.type) { - try { - const resolved = resolveTimeValue(timeValue, as.state); - eventContext[target] = resolved; - as.state[pending] = resolved; - console.log( - `Resolved ${target} to ${resolved}. Stored in ${pending}.` - ); - } catch (error) { - const errorMessage = `Failed to resolve ${source}: ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - } else { - // Non-StartExtractingMetadata events: reuse pending values from state - if (as.state.pendingWorkersOldest) { - eventContext.extract_from = as.state.pendingWorkersOldest; - console.log( - `Reusing pendingWorkersOldest as extract_from: ${as.state.pendingWorkersOldest}.` - ); - } else { - console.log( - 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' - ); - } - if (as.state.pendingWorkersNewest) { - eventContext.extract_to = as.state.pendingWorkersNewest; - console.log( - `Reusing pendingWorkersNewest as extract_to: ${as.state.pendingWorkersNewest}.` - ); - } else { - console.log( - 'pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.' - ); - } - } - - // Validate that extract_from is before extract_to - if (eventContext.extract_from && eventContext.extract_to) { - if (eventContext.extract_from >= eventContext.extract_to) { - const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - - return as; -} - -export class State { - private _state: AdapterState; - private _extractionScope: ExtractionScope = {}; - private initialSdkState: SdkState; - private workerUrl: string; - private devrevToken: string; - private syncUnitId: string; - private requestId: string; - - constructor({ event, initialState }: StateInterface) { - this.initialSdkState = - event.payload.event_context.mode === SyncMode.LOADING - ? loadingSdkState - : extractionSdkState; - this._state = { - ...initialState, - ...this.initialSdkState, - } as AdapterState; - this.workerUrl = event.payload.event_context.worker_data_url; - this.devrevToken = event.context.secrets.service_account_token; - this.syncUnitId = event.payload.event_context.sync_unit_id; - this.requestId = event.payload.event_context.request_id_adaas; - } - - get state(): AdapterState { - return this._state; - } - - set state(value: AdapterState) { - this._state = value; - } - - get extractionScope(): ExtractionScope { - return this._extractionScope; - } - - /** - * Initializes the state for this adapter instance by fetching from API - * or creating an initial state if none exists (404). - * @param initialState The initial connector state provided by the spawn function - */ - async init(initialState: ConnectorState): Promise { - try { - const { state: stringifiedState, objects } = await this.fetchState(); - if (!stringifiedState) { - throw new Error('No state found in response.'); - } - - let parsedState: AdapterState; - try { - parsedState = JSON.parse(stringifiedState); - } catch (error) { - throw new Error(`Failed to parse state. ${error}`); - } - - this.state = parsedState; - console.log( - 'State fetched successfully. Current state', - getPrintableState(this.state) - ); - - if (objects) { - try { - this._extractionScope = JSON.parse(objects); - } catch (error) { - console.warn(`Failed to parse extractionScope. ${error}`); - } - } - } catch (error) { - if (axios.isAxiosError(error) && error.response?.status === 404) { - console.log('State not found. Initializing state with initial state.'); - const initialAdapterState: AdapterState = { - ...initialState, - ...this.initialSdkState, - }; - - this.state = initialAdapterState; - await this.postState(initialAdapterState); - } else { - const errorMessage = `Failed to init state. ${serializeError(error)}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - - /** - * Updates the state of the adapter by posting to API. - * @param {object} state - The state to be updated - */ - async postState(state?: AdapterState) { - const url = this.workerUrl + '.update'; - this.state = state || this.state; - - let stringifiedState: string; - try { - stringifiedState = JSON.stringify(this.state); - } catch (error) { - const errorMessage = `Failed to stringify state. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - - try { - await axiosClient.post( - url, - { - state: stringifiedState, - }, - { - headers: { - Authorization: this.devrevToken, - }, - params: { - sync_unit: this.syncUnitId, - request_id: this.requestId, - }, - } - ); - - console.log( - 'State updated successfully to', - getPrintableState(this.state) - ); - } catch (error) { - const errorMessage = `Failed to update the state. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - - /** - * Fetches the state of the adapter from API. - * @return The raw state data from API - */ - async fetchState(): Promise<{ state: string; objects?: string }> { - console.log( - `Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.` - ); - - const url = this.workerUrl + '.get'; - const response = await axiosClient.get(url, { - headers: { - Authorization: this.devrevToken, - }, - params: { - sync_unit: this.syncUnitId, - request_id: this.requestId, - }, - }); - - return { - state: response.data?.state, - objects: response.data?.objects, - }; - } +import { BaseState } from './base-state'; +import { createExtractionState } from './extraction-state'; +import { createLoadingState } from './loading-state'; +import { StateInterface } from './state.interfaces'; + +export { BaseState } from './base-state'; +export { ExtractionState, createExtractionState } from './extraction-state'; +export { LoadingState, createLoadingState } from './loading-state'; + +/** + * Creates and initializes the adapter state for the current worker, dispatching + * to the extraction or loading state based on the event's sync mode. + * + * @param params The state factory parameters (event, initial state, options) + * @returns The initialized mode-specific state + */ +export async function createAdapterState( + params: StateInterface +): Promise> { + if (params.event.payload.event_context.mode === SyncMode.LOADING) { + return createLoadingState(params); + } + return createExtractionState(params); } diff --git a/src/types/workers.ts b/src/types/workers.ts index 9e3ed259..8be46ef5 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -1,7 +1,7 @@ import { Worker } from 'worker_threads'; import type { LogLevel } from '../logger/logger.interfaces'; -import { State } from '../state/state'; +import { BaseState } from '../state/state'; import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; import { AirSyncEvent, EventType, ExtractorEventType } from './extraction'; @@ -20,7 +20,7 @@ import { InitialDomainMapping } from './common'; */ export interface WorkerAdapterInterface { event: AirSyncEvent; - adapterState: State; + adapterState: BaseState; options?: WorkerAdapterOptions; } From 958bc3542b4043ad86a8535babcd73723ce11eb3 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 09:10:24 +0200 Subject: [PATCH 11/30] docs: mark C4a done; fix oracle reference (tag v2-old-backup, not origin/v2) Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 108de424..2cdf8ac3 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -13,10 +13,13 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi ## Git facts - **Working branch:** `v2` (already hard-reset to `origin/main`). - **Base commit:** `origin/main` = `5b81ef2` (feat: Add new common error enums #204). -- **Oracle (target shape):** `origin/v2` / tag `v2-old-backup` = `9202e47`. This is the PREVIOUS +- **Oracle (target shape):** tag `v2-old-backup` = `9202e47`. This is the PREVIOUS v2 attempt — it already implemented the rename, deletions, adapter split, state split+envelope, and emit-from-return, but bundled into huge unreviewable commits built on a stale base. **Use it as a structural reference / oracle only. Never copy wholesale. Re-author cleanly.** + Read oracle files with `git show v2-old-backup:src/...`. +- **IMPORTANT:** `origin/v2` is NOW our working branch (post-reset), NOT the oracle. The oracle is the + TAG `v2-old-backup`. Don't confuse them. - **Safety:** old work preserved at tag `v2-old-backup`. Force-push of `v2` is approved by Rado. ## Hard rules (apply to EVERY Phase-1 commit) @@ -50,7 +53,7 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi break the build. - **C2 — Airdrop→AirSync identifier rename.** DECIDED with Rado: - HARD rename, NO back-compat alias: `AirdropEvent`→`AirSyncEvent`, `AirdropMessage`→`AirSyncMessage`. - (origin/v2 did NOT do this rename — oracle unreliable here; do it properly.) + (v2-old-backup (oracle tag) did NOT do this rename — oracle unreliable here; do it properly.) - Update stale branding in comments/prose: bare "Airdrop" + "ADaaS" → "AirSync". - MUST NOT touch: `/internal/airdrop.*` API routes; the `AIRDROP_*` mapping enum members AND their `'airdrop_*'` string values (mappers.interface.ts); the `external_system_type: 'ADaaS'` string LITERAL @@ -70,17 +73,31 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi - Exported translation fns are NOT in index.ts (internal) — safe to delete. - **C4a — State split (structural only).** Introduce `BaseState` + `ExtractionState` + `LoadingState`. KEEP the flat `AdapterState = ConnectorState & SdkState` shape (behavior identical). - Author fresh; origin/v2 `src/state/base-state.ts` etc. are structural reference only. + Author fresh; oracle `src/state/base-state.ts` etc. guide STRUCTURE only (the oracle is already + envelope-based = C4b; do NOT copy its envelope split here). + DESIGN (decided after reading current state.ts + oracle): + - `base-state.ts`: `abstract class BaseState` holds flat `_state: AdapterState`, + `_extractionScope`, shared fields, `state` get/set, `extractionScope` get, `init()`, `fetchState()`, + `postState()`, and `installInitialDomainMappingIfNeeded()` (extracted from current factory). Ctor takes + `initialSdkState`. + - `extraction-state.ts`: `ExtractionState extends BaseState` (ctor passes `extractionSdkState`) + + `resolveExtractionWindow()` + lastSyncStarted logic; factory `createExtractionState()`. + - `loading-state.ts`: `LoadingState extends BaseState` (ctor passes `loadingSdkState`); factory + `createLoadingState()`. + - `state.ts`: keep a thin `createAdapterState()` DISPATCHER picking Extraction/Loading by + `event_context.mode === SyncMode.LOADING` (process-task stays unchanged until C6). Re-export classes. + - `state.interfaces.ts`: UNCHANGED (flat shape kept; type reshaping is C4b). + - Consumers: `types/workers.ts` + `worker-adapter.ts` change `State` type ref → `BaseState`. - **C4b — State envelope + migration.** Change on-disk shape to `{ connectorState, sdkState }`. Add migration shim: read legacy flat v1 blob → split SDK-owned keys into `sdkState` → persist envelope. - (origin/v2 `base-state.ts` has the reference impl incl. `V1_SDK_STATE_KEYS`.) + (v2-old-backup (oracle tag) `base-state.ts` has the reference impl incl. `V1_SDK_STATE_KEYS`.) - **C5 — Adapter split (structural only).** `BaseAdapter` + `ExtractionAdapter` + `LoadingAdapter`. KEEP existing `emit`-based contract working (behavior identical). Author fresh intermediate form - (this exact form exists in NO branch — origin/v2's split already assumes emit-from-return). + (this exact form exists in NO branch — v2-old-backup (oracle tag)'s split already assumes emit-from-return). - **C6 — Emit-from-return contract.** `task`/`onTimeout` return a `TaskResult` (`{ status: 'success'|'progress'|'delay'|'error', ... }`); the SDK maps status→phase event and emits exactly once; `emit` removed from public surface. `processTask` → `processExtractionTask` + - `processLoadingTask`. Reference: origin/v2 `process-task.ts`, `base-adapter.ts` (mapping keys off + `processLoadingTask`. Reference: v2-old-backup (oracle tag) `process-task.ts`, `base-adapter.ts` (mapping keys off event_type/phase, NOT off state shape — so C4b and C6 are independent). ### Phase 2 — closing / interactive (batched, done at the end) @@ -124,7 +141,7 @@ src/deprecated/uploader/index.ts Also delete `src/common/event-type-translation.ts` (+ `.test.ts`). `src/index.ts` on main exports these deprecated barrels — remove those export lines: `./deprecated/adapter`, `./deprecated/demo-extractor`, `./deprecated/http/client`, `./deprecated/uploader`, -and the `formatAxiosError` export (origin/v2 dropped it — confirm against connector usage; azure-boards imports it, so this is a migration note). +and the `formatAxiosError` export (v2-old-backup (oracle tag) dropped it — confirm against connector usage; azure-boards imports it, so this is a migration note). ### C3 — EventType (incoming): DELETE these deprecated members, keep the new ones | DELETE (old member = old VALUE) | KEEP (new member = new VALUE) | @@ -158,7 +175,7 @@ Loading members (StartLoadingData…StartDeletingLoaderAttachmentState) + Unknow | ExtractionAttachmentsError | AttachmentExtractionError | | ExtractionAttachmentsDeleteDone | ExtractorAttachmentsStateDeletionDone | | ExtractionAttachmentsDeleteError | ExtractorAttachmentsStateDeletionError | -(values for new members are the *_EXTRACTION_* / *_DELETION_* strings — see origin/v2 extraction.ts.) +(values for new members are the *_EXTRACTION_* / *_DELETION_* strings — see v2-old-backup (oracle tag) extraction.ts.) ### C3 — LoaderEventType: DELETE deprecated typo/plural members DELETE: `DataLoadingDelay` (typo), `AttachmentsLoadingProgress/Delayed/Done/Error` (the plural-typo dupes). @@ -189,7 +206,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C1 delete + tsconfig | ☑ done | d573cb6. Deleted src/deprecated/ (6 files) + 4 index exports; added tsconfig.build.json (excludes tests), build script points to it. Reviewer-approved. event-type-translation deletion moved to C3. | | C2 AirSync rename | ☑ done | 1fa9afc. AirdropEvent→AirSyncEvent, AirdropMessage→AirSyncMessage (hard, no alias) + prose ADaaS/Airdrop→AirSync. Protected: airdrop.* routes, AIRDROP_* enum, 'ADaaS' literal. Reviewer-approved. | | C3 enum cleanup | ☑ done | cc05f41. Deleted deprecated enum members (EventType, ExtractorEventType, LoaderEventType) + event-type-translation.ts/.test; rewired 4 callers (process-task, spawn, control-protocol, worker-adapter) + spawn.helpers cases. Behavior-equivalent. Reviewer-approved. | -| C4a state split | ☐ todo | | +| C4a state split | ☑ done | b63f3ab. BaseState + ExtractionState + LoadingState, flat shape preserved; state.ts is dispatcher by mode. Reviewer-confirmed behavior-equivalent (loading only loses inert logs). | | C4b state envelope | ☐ todo | | | C5 adapter split | ☐ todo | | | C6 emit-from-return | ☐ todo | | From 30ba1b370504daa34439f7a4db0f14aed259dd2b Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 09:24:25 +0200 Subject: [PATCH 12/30] refactor(v2)!: persist state as { connectorState, sdkState } envelope Separate connector-owned state from SDK bookkeeping on disk. State is now persisted as a v2 envelope { connectorState, sdkState } instead of a single flat blob that merged both, so SDK internals stay encapsulated and can never collide with connector keys. - state.interfaces.ts: add AdapterStateEnvelope and V1_SDK_STATE_KEYS (the union of the SDK-owned initial-state keys); mark the flat AdapterState deprecated. SdkState stays a single combined interface (narrowing into per-mode variants is deferred to the adapter split). - base-state.ts: hold _connectorState and _sdkState separately; the state getter/setter now exposes connector state only; add an sdkState getter/setter for SDK internals; init() routes fetched state through a normalizeFetchedState() migration shim; postState() persists the envelope. - Migration shim: a v2 envelope is used as-is; a legacy flat v1 blob is split by V1_SDK_STATE_KEYS (SDK keys to sdkState, the rest to connectorState); a half-envelope or non-object fails loud. Existing syncs migrate on first read. - extraction-state.ts: resolve the extraction window against sdkState. - worker-adapter.ts: adapter.state now returns ConnectorState; add an adapter.sdkState getter; all internal SDK-field access goes through sdkState. - attachments-streaming-pool.ts: read toDevRev via adapter.sdkState. BREAKING CHANGE: adapter.state no longer exposes SDK bookkeeping fields (lastSyncStarted, workersOldest, toDevRev, fromDevRev, snapInVersionId, ...). Connector state is now disjoint from SDK state. Persisted v1 state is migrated automatically on read. Ref: V2_PROGRESS.md C4b Co-Authored-By: Claude Opus 4.8 (1M context) --- .../attachments-streaming-pool.ts | 18 +-- .../worker-adapter/worker-adapter.ts | 62 +++---- src/state/base-state.ts | 153 +++++++++++++----- src/state/extraction-state.ts | 24 +-- src/state/state.interfaces.ts | 30 +++- 5 files changed, 199 insertions(+), 88 deletions(-) diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index 1690386b..4dbc1caa 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -74,7 +74,7 @@ export class AttachmentsStreamingPool { `Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.` ); - if (!this.adapter.state.toDevRev) { + if (!this.adapter.sdkState.toDevRev) { const error = new Error('toDevRev state is not initialized'); console.error(error); return { error }; @@ -83,17 +83,17 @@ export class AttachmentsStreamingPool { // Get the list of successfully processed attachments in previous (possibly incomplete) batch extraction. // If no such list exists, create an empty one. if ( - !this.adapter.state.toDevRev.attachmentsMetadata + !this.adapter.sdkState.toDevRev.attachmentsMetadata .lastProcessedAttachmentsIdsList ) { - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = []; } // Migrate old processed attachments to the new format. - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = this.migrateProcessedAttachments( - this.adapter.state.toDevRev.attachmentsMetadata + this.adapter.sdkState.toDevRev.attachmentsMetadata .lastProcessedAttachmentsIdsList ); @@ -139,8 +139,8 @@ export class AttachmentsStreamingPool { } if ( - this.adapter.state.toDevRev && - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList?.some( + this.adapter.sdkState.toDevRev && + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList?.some( (it) => it.id == attachment.id && it.parent_id == attachment.parent_id ) ) { @@ -180,10 +180,10 @@ export class AttachmentsStreamingPool { // No rate limiting, process normally if ( - this.adapter.state.toDevRev?.attachmentsMetadata + this.adapter.sdkState.toDevRev?.attachmentsMetadata ?.lastProcessedAttachmentsIdsList ) { - this.adapter.state.toDevRev?.attachmentsMetadata.lastProcessedAttachmentsIdsList.push( + this.adapter.sdkState.toDevRev?.attachmentsMetadata.lastProcessedAttachmentsIdsList.push( { id: attachment.id, parent_id: attachment.parent_id } ); } diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts index 23e2550a..aae497a5 100644 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ b/src/multithreading/worker-adapter/worker-adapter.ts @@ -25,7 +25,7 @@ import { RepoInterface, } from '../../repo/repo.interfaces'; import { BaseState } from '../../state/state'; -import { AdapterState } from '../../state/state.interfaces'; +import { SdkState } from '../../state/state.interfaces'; import { AirSyncEvent, EventData, @@ -129,14 +129,20 @@ export class WorkerAdapter { }); } - get state(): AdapterState { + /** Connector-owned state exposed to snap-in code. */ + get state(): ConnectorState { return this.adapterState.state; } - set state(value: AdapterState) { + set state(value: ConnectorState) { this.adapterState.state = value; } + /** SDK-internal bookkeeping state. Used by SDK internals; not for connector use. */ + get sdkState(): SdkState { + return this.adapterState.sdkState; + } + get reports(): LoaderReport[] { return this.loaderReports; } @@ -177,7 +183,7 @@ export class WorkerAdapter { onUpload: (artifact: Artifact) => { // We need to store artifacts ids in state for later use when streaming attachments if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { - this.state.toDevRev?.attachmentsMetadata.artifactIds.push( + this.sdkState.toDevRev?.attachmentsMetadata.artifactIds.push( artifact.id ); } @@ -296,15 +302,15 @@ export class WorkerAdapter { // If the extraction is done, we want to save the timestamp of the last successful sync if (newEventType === ExtractorEventType.AttachmentExtractionDone) { console.log( - `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).` + `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.sdkState.lastSyncStarted}).` ); - this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted; - this.state.lastSyncStarted = ''; + this.sdkState.lastSuccessfulSyncStarted = this.sdkState.lastSyncStarted; + this.sdkState.lastSyncStarted = ''; // Clear pending extraction boundaries now that the cycle is complete - this.state.pendingWorkersOldest = ''; - this.state.pendingWorkersNewest = ''; + this.sdkState.pendingWorkersOldest = ''; + this.sdkState.pendingWorkersNewest = ''; // Update workersOldest and workersNewest boundaries from resolved extraction timestamps. // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest. @@ -313,24 +319,24 @@ export class WorkerAdapter { if ( extractionStart && - (!this.state.workersOldest || - extractionStart < this.state.workersOldest) + (!this.sdkState.workersOldest || + extractionStart < this.sdkState.workersOldest) ) { console.log( - `Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.` + `Updating workersOldest from '${this.sdkState.workersOldest}' to '${extractionStart}'.` ); - this.state.workersOldest = extractionStart; + this.sdkState.workersOldest = extractionStart; } if ( extractionEnd && - (!this.state.workersNewest || - extractionEnd > this.state.workersNewest) + (!this.sdkState.workersNewest || + extractionEnd > this.sdkState.workersNewest) ) { console.log( - `Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.` + `Updating workersNewest from '${this.sdkState.workersNewest}' to '${extractionEnd}'.` ); - this.state.workersNewest = extractionEnd; + this.sdkState.workersNewest = extractionEnd; } } @@ -423,14 +429,14 @@ export class WorkerAdapter { const filesToLoad = await this.getLoaderBatches({ supportedItemTypes: itemTypes, }); - this.adapterState.state.fromDevRev = { + this.adapterState.sdkState.fromDevRev = { filesToLoad, }; } if ( - !this.adapterState.state.fromDevRev || - !this.adapterState.state.fromDevRev.filesToLoad.length + !this.adapterState.sdkState.fromDevRev || + !this.adapterState.sdkState.fromDevRev.filesToLoad.length ) { console.warn('No files to load, returning.'); return { @@ -441,12 +447,12 @@ export class WorkerAdapter { console.log( 'Files to load in state', - this.adapterState.state.fromDevRev?.filesToLoad + this.adapterState.sdkState.fromDevRev?.filesToLoad ); try { - outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev - .filesToLoad) { + outerloop: for (const fileToLoad of this.adapterState.sdkState + .fromDevRev.filesToLoad) { const itemTypeToLoad = itemTypesToLoad.find( (itemTypeToLoad: ItemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType @@ -580,7 +586,7 @@ export class WorkerAdapter { }): Promise { return runWithSdkLogContext(async () => { if (this.event.payload.event_type === EventType.StartLoadingAttachments) { - this.adapterState.state.fromDevRev = { + this.adapterState.sdkState.fromDevRev = { filesToLoad: await this.getLoaderBatches({ supportedItemTypes: ['attachment'], }), @@ -588,8 +594,8 @@ export class WorkerAdapter { } if ( - !this.adapterState.state.fromDevRev || - this.adapterState.state.fromDevRev?.filesToLoad.length === 0 + !this.adapterState.sdkState.fromDevRev || + this.adapterState.sdkState.fromDevRev?.filesToLoad.length === 0 ) { console.log('No files to load, returning.'); return { @@ -598,7 +604,7 @@ export class WorkerAdapter { }; } - const filesToLoad = this.adapterState.state.fromDevRev?.filesToLoad; + const filesToLoad = this.adapterState.sdkState.fromDevRev?.filesToLoad; try { outerloop: for (const fileToLoad of filesToLoad) { @@ -1123,7 +1129,7 @@ export class WorkerAdapter { ]; this.initializeRepos(repos); - const attachmentsMetadata = this.state.toDevRev?.attachmentsMetadata; + const attachmentsMetadata = this.sdkState.toDevRev?.attachmentsMetadata; // If there are no attachments metadata artifact IDs in state, finish here if (!attachmentsMetadata?.artifactIds?.length) { diff --git a/src/state/base-state.ts b/src/state/base-state.ts index 3c9f8a9c..06db8959 100644 --- a/src/state/base-state.ts +++ b/src/state/base-state.ts @@ -9,12 +9,17 @@ import { AirSyncEvent } from '../types/extraction'; import { WorkerMessageSubject } from '../types/workers'; import { ExtractionScope } from '../types/workers'; -import { AdapterState, SdkState, StateInterface } from './state.interfaces'; +import { + AdapterStateEnvelope, + SdkState, + StateInterface, + V1_SDK_STATE_KEYS, +} from './state.interfaces'; /** - * BaseState owns the state lifecycle shared by every sync mode: holding the - * adapter state, fetching/initializing/posting it against the platform, and the - * snap-in-version-gated initial domain mapping install. + * BaseState owns the state lifecycle shared by every sync mode: connector vs. + * SDK state separation, fetch/init/post against the platform, the v1->v2 + * migration shim, and the snap-in-version-gated initial domain mapping install. * * Mode-specific subclasses (`ExtractionState`, `LoadingState`) seed the * SDK-owned portion of the state and add mode-specific setup in their factories. @@ -22,7 +27,8 @@ import { AdapterState, SdkState, StateInterface } from './state.interfaces'; * @typeParam ConnectorState - the connector-owned state shape */ export abstract class BaseState { - protected _state: AdapterState; + protected _connectorState: ConnectorState; + protected _sdkState: SdkState; protected _extractionScope: ExtractionScope = {}; protected readonly initialSdkState: SdkState; protected readonly event: AirSyncEvent; @@ -37,22 +43,30 @@ export abstract class BaseState { ) { this.event = event; this.initialSdkState = initialSdkState; - this._state = { - ...initialState, - ...initialSdkState, - } as AdapterState; + this._connectorState = initialState; + this._sdkState = { ...initialSdkState }; this.workerUrl = event.payload.event_context.worker_data_url; this.devrevToken = event.context.secrets.service_account_token; this.syncUnitId = event.payload.event_context.sync_unit_id; this.requestId = event.payload.event_context.request_id_adaas; } - get state(): AdapterState { - return this._state; + /** Connector-owned state. This is what `adapter.state` exposes to snap-in code. */ + get state(): ConnectorState { + return this._connectorState; } - set state(value: AdapterState) { - this._state = value; + set state(value: ConnectorState) { + this._connectorState = value; + } + + /** SDK-internal bookkeeping state. Never exposed to connector code. */ + get sdkState(): SdkState { + return this._sdkState; + } + + set sdkState(value: SdkState) { + this._sdkState = value; } get extractionScope(): ExtractionScope { @@ -69,10 +83,10 @@ export abstract class BaseState { initialDomainMapping?: InitialDomainMapping ): Promise { const snapInVersionId = this.event.context.snap_in_version_id; - const hasSnapInVersionInState = 'snapInVersionId' in this.state; + const hasSnapInVersionInState = 'snapInVersionId' in this.sdkState; const shouldUpdateIDM = !hasSnapInVersionInState || - this.state.snapInVersionId !== snapInVersionId; + this.sdkState.snapInVersionId !== snapInVersionId; if (!shouldUpdateIDM) { console.log( @@ -83,12 +97,12 @@ export abstract class BaseState { try { console.log( - `Snap-in version in state "${this.state.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` + `Snap-in version in state "${this.sdkState.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` ); if (initialDomainMapping) { await installInitialDomainMapping(this.event, initialDomainMapping); - this.state.snapInVersionId = snapInVersionId; + this.sdkState.snapInVersionId = snapInVersionId; } else { throw new Error( 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' @@ -110,6 +124,10 @@ export abstract class BaseState { /** * Initializes the state for this adapter instance by fetching from API * or creating an initial state if none exists (404). + * + * Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat + * v1 blob (connector keys merged with SDK keys), migrating the latter on read. + * Always persists the v2 envelope going forward. * @param initialState The initial connector state provided by the spawn function */ async init(initialState: ConnectorState): Promise { @@ -119,18 +137,23 @@ export abstract class BaseState { throw new Error('No state found in response.'); } - let parsedState: AdapterState; + let parsed: unknown; try { - parsedState = JSON.parse(stringifiedState); + parsed = JSON.parse(stringifiedState); } catch (error) { throw new Error(`Failed to parse state. ${error}`); } - this.state = parsedState; - console.log( - 'State fetched successfully. Current state', - getPrintableState(this.state) - ); + const { connectorState, sdkState } = this.normalizeFetchedState(parsed); + this.state = connectorState; + this.sdkState = sdkState; + + console.log('State fetched successfully. Current state', { + connectorState: getPrintableState( + this.state as Record + ), + sdkState: getPrintableState(this.sdkState), + }); if (objects) { try { @@ -142,13 +165,9 @@ export abstract class BaseState { } catch (error) { if (axios.isAxiosError(error) && error.response?.status === 404) { console.log('State not found. Initializing state with initial state.'); - const initialAdapterState: AdapterState = { - ...initialState, - ...this.initialSdkState, - }; - - this.state = initialAdapterState; - await this.postState(initialAdapterState); + this.state = initialState; + this.sdkState = { ...this.initialSdkState }; + await this.postState(); } else { const errorMessage = `Failed to init state. ${serializeError(error)}`; console.error(errorMessage); @@ -161,17 +180,73 @@ export abstract class BaseState { } } + /** + * Normalizes a parsed on-disk state into the `{ connectorState, sdkState }` + * envelope, migrating a legacy flat v1 blob if needed. + * + * - v2 envelope (`{ connectorState, sdkState }`): used as-is. + * - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`, + * everything else becomes connector state. + * - Malformed envelope (one side present, the other missing) fails loud. + */ + private normalizeFetchedState(parsed: unknown): { + connectorState: ConnectorState; + sdkState: SdkState; + } { + if (parsed === null || typeof parsed !== 'object') { + throw new Error('Fetched state is not a JSON object.'); + } + + const record = parsed as Record; + const hasConnector = 'connectorState' in record; + const hasSdk = 'sdkState' in record; + + if (hasConnector || hasSdk) { + if (!hasConnector || !hasSdk) { + throw new Error( + 'Malformed state envelope: expected both "connectorState" and "sdkState".' + ); + } + return { + connectorState: record.connectorState as ConnectorState, + sdkState: { ...this.initialSdkState, ...(record.sdkState as SdkState) }, + }; + } + + // Legacy flat v1 blob: split known SDK keys out of the connector state. + const connectorState: Record = {}; + const sdkState: Record = {}; + for (const [key, value] of Object.entries(record)) { + if (V1_SDK_STATE_KEYS.has(key)) { + sdkState[key] = value; + } else { + connectorState[key] = value; + } + } + + return { + connectorState: connectorState as ConnectorState, + sdkState: { ...this.initialSdkState, ...(sdkState as SdkState) }, + }; + } + /** * Updates the state of the adapter by posting to API. - * @param {object} state - The state to be updated + * Persists the v2 `{ connectorState, sdkState }` envelope. + * @param {object} state - The connector state to be updated */ - async postState(state?: AdapterState) { + async postState(state?: ConnectorState) { const url = this.workerUrl + '.update'; this.state = state || this.state; + const envelope: AdapterStateEnvelope = { + connectorState: this.state, + sdkState: this.sdkState, + }; + let stringifiedState: string; try { - stringifiedState = JSON.stringify(this.state); + stringifiedState = JSON.stringify(envelope); } catch (error) { const errorMessage = `Failed to stringify state. ${serializeError( error @@ -201,10 +276,12 @@ export abstract class BaseState { } ); - console.log( - 'State updated successfully to', - getPrintableState(this.state) - ); + console.log('State updated successfully to', { + connectorState: getPrintableState( + this.state as Record + ), + sdkState: getPrintableState(this.sdkState), + }); } catch (error) { const errorMessage = `Failed to update the state. ${serializeError( error diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts index bb99a0f7..b78dbbc0 100644 --- a/src/state/extraction-state.ts +++ b/src/state/extraction-state.ts @@ -30,13 +30,15 @@ export class ExtractionState extends BaseState { * StartExtractingMetadata. Finally, validate that extract_from < extract_to. */ resolveExtractionWindow(): void { + const sdkState = this.sdkState; + // Set lastSyncStarted if the event type is StartExtractingData if ( this.event.payload.event_type === EventType.StartExtractingData && - !this.state.lastSyncStarted + !sdkState.lastSyncStarted ) { - this.state.lastSyncStarted = new Date().toISOString(); - console.log(`Setting lastSyncStarted to ${this.state.lastSyncStarted}.`); + sdkState.lastSyncStarted = new Date().toISOString(); + console.log(`Setting lastSyncStarted to ${sdkState.lastSyncStarted}.`); } const eventContext = this.event.payload.event_context; @@ -59,9 +61,9 @@ export class ExtractionState extends BaseState { const timeValue = eventContext[source]; if (timeValue && timeValue.type) { try { - const resolved = resolveTimeValue(timeValue, this.state); + const resolved = resolveTimeValue(timeValue, sdkState); eventContext[target] = resolved; - this.state[pending] = resolved; + sdkState[pending] = resolved; console.log( `Resolved ${target} to ${resolved}. Stored in ${pending}.` ); @@ -80,20 +82,20 @@ export class ExtractionState extends BaseState { } } else { // Non-StartExtractingMetadata events: reuse pending values from state - if (this.state.pendingWorkersOldest) { - eventContext.extract_from = this.state.pendingWorkersOldest; + if (sdkState.pendingWorkersOldest) { + eventContext.extract_from = sdkState.pendingWorkersOldest; console.log( - `Reusing pendingWorkersOldest as extract_from: ${this.state.pendingWorkersOldest}.` + `Reusing pendingWorkersOldest as extract_from: ${sdkState.pendingWorkersOldest}.` ); } else { console.log( 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' ); } - if (this.state.pendingWorkersNewest) { - eventContext.extract_to = this.state.pendingWorkersNewest; + if (sdkState.pendingWorkersNewest) { + eventContext.extract_to = sdkState.pendingWorkersNewest; console.log( - `Reusing pendingWorkersNewest as extract_to: ${this.state.pendingWorkersNewest}.` + `Reusing pendingWorkersNewest as extract_to: ${sdkState.pendingWorkersNewest}.` ); } else { console.log( diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index 6c28cf34..9e105772 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -30,11 +30,25 @@ export interface SdkState { } /** - * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. - * It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, snapInVersionId and attachmentsMetadata. + * AdapterState is the legacy (v1) flat adapter state: connector state merged + * with SDK bookkeeping in a single object. + * + * @deprecated v2 persists the `{ connectorState, sdkState }` envelope + * (see {@link AdapterStateEnvelope}). Connector state is now exposed via + * `adapter.state` and SDK state is kept internal. */ export type AdapterState = ConnectorState & SdkState; +/** + * AdapterStateEnvelope is the v2 on-disk state shape: connector state and SDK + * bookkeeping are stored as disjoint sub-objects so SDK internals stay + * encapsulated and never collide with connector keys. + */ +export interface AdapterStateEnvelope { + connectorState: ConnectorState; + sdkState: SdkState; +} + export interface ToDevRev { attachmentsMetadata: { artifactIds: string[]; @@ -85,3 +99,15 @@ export const loadingSdkState = { filesToLoad: [], }, }; + +/** + * The set of top-level state keys owned by the SDK. Derived from the initial + * SDK state constants so it auto-updates whenever a new SDK field is added. + * Used by the migration shim to split a flat v1 state blob into the + * `{ connectorState, sdkState }` envelope: keys in this set go to `sdkState`, + * everything else is connector state. + */ +export const V1_SDK_STATE_KEYS: ReadonlySet = new Set([ + ...Object.keys(extractionSdkState), + ...Object.keys(loadingSdkState), +]); From 0da1331924c701c877723c9b646892348880ec5e Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 09:25:00 +0200 Subject: [PATCH 13/30] docs: mark C4b done; note SdkState narrowing deferred to C5 Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 2cdf8ac3..609a2efd 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -90,7 +90,22 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi - Consumers: `types/workers.ts` + `worker-adapter.ts` change `State` type ref → `BaseState`. - **C4b — State envelope + migration.** Change on-disk shape to `{ connectorState, sdkState }`. Add migration shim: read legacy flat v1 blob → split SDK-owned keys into `sdkState` → persist envelope. - (v2-old-backup (oracle tag) `base-state.ts` has the reference impl incl. `V1_SDK_STATE_KEYS`.) + (oracle `base-state.ts` has the reference impl incl. `V1_SDK_STATE_KEYS`.) + DESIGN DECISION: keep a SINGLE combined `SdkState` for C4b (do NOT narrow into Base/Extraction/Loading + SdkState — oracle narrows but that's coupled to the adapter split; defer to C5). `BaseState` + keeps ONE type param; gains `_connectorState` + `_sdkState: SdkState`. Keep deprecated lastSyncStarted/ + lastSuccessfulSyncStarted in SdkState (still live: stamped in extraction-state, promoted in worker-adapter, + read by time-value-resolver). Blast radius (SDK-field access moves off `.state` → `.sdkState`): + - state.interfaces.ts: add AdapterStateEnvelope + V1_SDK_STATE_KEYS; keep SdkState/initials/AdapterState. + - base-state.ts: `_connectorState`+`_sdkState`; `state` getter→connector, add `sdkState` getter; + init/postState use envelope; private normalizeFetchedState() (v2 as-is; v1 flat split by V1_SDK_STATE_KEYS; + malformed→throw). installIDM uses this.sdkState.snapInVersionId. + - extraction-state.ts: resolveExtractionWindow this.state.X→this.sdkState.X; pass this.sdkState to resolver. + - worker-adapter.ts: get state()→ConnectorState (KEY public breaking change); add get sdkState(); internal + this.state.→this.sdkState; this.adapterState.state.fromDevRev→this.adapterState.sdkState.fromDevRev. + - attachments-streaming-pool.ts: this.adapter.state.toDevRev→this.adapter.sdkState.toDevRev. + - time-value-resolver.ts: signature unchanged. + BREAKING: connectors reading SDK fields via adapter.state break; on-disk state auto-migrates v1→v2 on read. - **C5 — Adapter split (structural only).** `BaseAdapter` + `ExtractionAdapter` + `LoadingAdapter`. KEEP existing `emit`-based contract working (behavior identical). Author fresh intermediate form (this exact form exists in NO branch — v2-old-backup (oracle tag)'s split already assumes emit-from-return). @@ -207,8 +222,8 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C2 AirSync rename | ☑ done | 1fa9afc. AirdropEvent→AirSyncEvent, AirdropMessage→AirSyncMessage (hard, no alias) + prose ADaaS/Airdrop→AirSync. Protected: airdrop.* routes, AIRDROP_* enum, 'ADaaS' literal. Reviewer-approved. | | C3 enum cleanup | ☑ done | cc05f41. Deleted deprecated enum members (EventType, ExtractorEventType, LoaderEventType) + event-type-translation.ts/.test; rewired 4 callers (process-task, spawn, control-protocol, worker-adapter) + spawn.helpers cases. Behavior-equivalent. Reviewer-approved. | | C4a state split | ☑ done | b63f3ab. BaseState + ExtractionState + LoadingState, flat shape preserved; state.ts is dispatcher by mode. Reviewer-confirmed behavior-equivalent (loading only loses inert logs). | -| C4b state envelope | ☐ todo | | -| C5 adapter split | ☐ todo | | +| C4b state envelope | ☑ done | 30ba1b3. { connectorState, sdkState } envelope + v1->v2 migration shim (normalizeFetchedState). adapter.state→connector-only, new adapter.sdkState; ~28 SDK-field access sites moved. SdkState kept combined (narrowing deferred to C5). Reviewer-approved (migration cases verified). | +| C5 adapter split | ☐ todo | NOTE for C5: narrow SdkState into Base/Extraction/Loading variants here (oracle state.interfaces shows the split). | | C6 emit-from-return | ☐ todo | | | C7 JSDoc | ☐ todo | Phase 2 | | C8 api report | ☐ todo | Phase 2 | From a7a877fa255021a542ee9abd6a1249c29f87dafd Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 09:41:37 +0200 Subject: [PATCH 14/30] refactor(v2)!: split WorkerAdapter into Base/Extraction/Loading adapters Replace the monolithic WorkerAdapter with a BaseAdapter holding the shared state access and the emit() control-protocol flow as a template method, plus ExtractionAdapter and LoadingAdapter subclasses that own their mode-specific surface. - base-adapter.ts: abstract BaseAdapter owns event/options/isTimeout/ hasWorkerEmitted, adapterState, uploader, state/sdkState/extractionScope accessors, postState(), and emit(). emit() is a template method with three hooks: beforeEmit (pre-emit work), buildEmitPayload (mode payload extras), afterEmit (post-emit cleanup). - extraction-adapter.ts: ExtractionAdapter owns repos, artifacts, attachment streaming/processing. beforeEmit uploads repos and updates extraction boundaries (incl. lastSuccessfulSyncStarted promotion); buildEmitPayload adds artifacts; afterEmit clears them. An override emit() handles the inline external-sync-unit upload before delegating to BaseAdapter.emit(). - loading-adapter.ts: LoadingAdapter owns mappers, loader reports, item and attachment loading. buildEmitPayload adds reports + processed_files. - worker-adapter.helpers.ts moved to adapters/loading-adapter.helpers.ts (loader-only). Old worker-adapter.ts removed. - WorkerAdapter is now a type alias ExtractionAdapter | LoadingAdapter. processTask builds the concrete adapter from the event's sync mode and passes it to task/onTimeout. Attachment processor types and the streaming pool are typed to ExtractionAdapter. Behavior is preserved: emit() runs the same steps in the same order, the mode-specific payloads are unchanged, and the extraction-boundary bookkeeping is identical. The emit(eventType, data) signature is kept; the emit-from-return contract change is a separate commit. BREAKING CHANGE: WorkerAdapter is no longer a constructable class. Extraction tasks receive an ExtractionAdapter and loading tasks a LoadingAdapter; the methods available are scoped to the worker's mode. Ref: V2_PROGRESS.md C5 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../attachments-streaming-pool.interfaces.ts | 4 +- .../attachments-streaming-pool.ts | 4 +- src/index.ts | 4 +- src/multithreading/adapters/base-adapter.ts | 185 +++ .../adapters/extraction-adapter.ts | 535 +++++++ .../loading-adapter.helpers.ts} | 0 .../adapters/loading-adapter.ts | 645 +++++++++ src/multithreading/process-task.ts | 22 +- .../worker-adapter/worker-adapter.ts | 1244 ----------------- src/types/extraction.ts | 6 +- src/types/workers.ts | 13 +- 11 files changed, 1403 insertions(+), 1259 deletions(-) create mode 100644 src/multithreading/adapters/base-adapter.ts create mode 100644 src/multithreading/adapters/extraction-adapter.ts rename src/multithreading/{worker-adapter/worker-adapter.helpers.ts => adapters/loading-adapter.helpers.ts} (100%) create mode 100644 src/multithreading/adapters/loading-adapter.ts delete mode 100644 src/multithreading/worker-adapter/worker-adapter.ts diff --git a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts index 9067c4f6..9a22ea08 100644 --- a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts +++ b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts @@ -2,10 +2,10 @@ import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment, } from '../types'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; export interface AttachmentsStreamingPoolParams { - adapter: WorkerAdapter; + adapter: ExtractionAdapter; attachments: NormalizedAttachment[]; batchSize?: number; stream: ExternalSystemAttachmentStreamingFunction; diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index 4dbc1caa..58281e16 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -1,5 +1,5 @@ import { sleep } from '../common/helpers'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; import { ProcessedAttachment } from '../state/state.interfaces'; import { ExternalSystemAttachmentStreamingFunction, @@ -9,7 +9,7 @@ import { import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces'; export class AttachmentsStreamingPool { - private adapter: WorkerAdapter; + private adapter: ExtractionAdapter; private attachments: NormalizedAttachment[]; private batchSize: number; private delay: number | undefined; diff --git a/src/index.ts b/src/index.ts index b4b94dc3..ce506898 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,9 @@ export type { } from './mock-server/mock-server.interfaces'; export { processTask } from './multithreading/process-task'; export { spawn } from './multithreading/spawn/spawn'; -export { WorkerAdapter } from './multithreading/worker-adapter/worker-adapter'; +export { BaseAdapter } from './multithreading/adapters/base-adapter'; +export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter'; +export { LoadingAdapter } from './multithreading/adapters/loading-adapter'; export { createMockEvent, MOCK_SERVER_DEFAULT_URL } from './common/test-utils'; export type { DeepPartial } from './common/test-utils'; export * from './types'; diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts new file mode 100644 index 00000000..e05c5411 --- /dev/null +++ b/src/multithreading/adapters/base-adapter.ts @@ -0,0 +1,185 @@ +import { parentPort } from 'node:worker_threads'; + +import { STATELESS_EVENT_TYPES } from '../../common/constants'; +import { emit } from '../../common/control-protocol'; +import { truncateMessage } from '../../common/helpers'; +import { serializeError } from '../../logger/logger'; +import { runWithSdkLogContext } from '../../logger/logger.context'; +import { BaseState } from '../../state/state'; +import { SdkState } from '../../state/state.interfaces'; +import { + AirSyncEvent, + EventData, + ExtractorEventType, +} from '../../types/extraction'; +import { LoaderEventType } from '../../types/loading'; +import { + WorkerAdapterOptions, + WorkerMessageEmitted, + WorkerMessageSubject, +} from '../../types/workers'; +import { Uploader } from '../../uploader/uploader'; + +/** + * BaseAdapter holds the state and behavior shared by both sync modes and owns + * the `emit` control-protocol flow as a template method. Mode-specific adapters + * (`ExtractionAdapter`, `LoadingAdapter`) implement the abstract hooks to inject + * their own pre-emit work and event payload shaping. + * + * @typeParam ConnectorState - the connector-owned state shape + */ +export abstract class BaseAdapter { + readonly event: AirSyncEvent; + readonly options?: WorkerAdapterOptions; + isTimeout: boolean; + hasWorkerEmitted: boolean; + + protected adapterState: BaseState; + protected uploader: Uploader; + + constructor({ + event, + adapterState, + options, + }: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + this.event = event; + this.options = options; + this.adapterState = adapterState; + this.hasWorkerEmitted = false; + this.isTimeout = false; + this.uploader = new Uploader({ + event, + options, + }); + } + + /** Connector-owned state exposed to snap-in code. */ + get state(): ConnectorState { + return this.adapterState.state; + } + + set state(value: ConnectorState) { + this.adapterState.state = value; + } + + /** SDK-internal bookkeeping state. Used by SDK internals; not for connector use. */ + get sdkState(): SdkState { + return this.adapterState.sdkState; + } + + get extractionScope() { + return this.adapterState.extractionScope; + } + + async postState() { + return runWithSdkLogContext(async () => { + await this.adapterState.postState(); + }); + } + + /** + * Pre-emit hook run before any state is persisted or the event is sent. + * Extraction uploads pending repos and updates extraction boundaries here; + * loading has nothing to do. Throwing aborts the emit (the caller signals + * worker exit). + */ + protected abstract beforeEmit( + newEventType: ExtractorEventType | LoaderEventType + ): Promise; + + /** + * Builds the mode-specific extras merged into the emitted event payload + * (extraction: artifacts; loading: reports + processed files). + */ + protected abstract buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData; + + /** + * Post-emit hook run after the event has been sent successfully. Extraction + * clears its accumulated artifacts here; loading has nothing to do. + */ + protected abstract afterEmit( + newEventType: ExtractorEventType | LoaderEventType + ): void; + + /** + * Emits an event to the platform. + * + * @param newEventType - The event type to be emitted + * @param data - The data to be sent with the event + */ + async emit( + newEventType: ExtractorEventType | LoaderEventType, + data?: EventData + ): Promise { + return runWithSdkLogContext(async () => { + if (this.hasWorkerEmitted) { + console.warn( + `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` + ); + return; + } + + try { + await this.beforeEmit(newEventType); + } catch (error) { + console.error('Error while preparing to emit event', error); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + return; + } + + // We want to save the state every time we emit an event, except for the start and delete events + if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { + console.log( + `Saving state before emitting event with event type: ${newEventType}.` + ); + + try { + await this.adapterState.postState(); + } catch (error) { + console.error('Error while posting state', error); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + return; + } + } + + try { + // Always prune error messages to make them shorter before emit + if (data?.error?.message) { + data.error.message = truncateMessage(data.error.message); + } + + await emit({ + eventType: newEventType, + event: this.event, + data: { + ...data, + ...this.buildEmitPayload(newEventType), + }, + }); + + const message: WorkerMessageEmitted = { + subject: WorkerMessageSubject.WorkerMessageEmitted, + payload: { eventType: newEventType }, + }; + this.afterEmit(newEventType); + parentPort?.postMessage(message); + this.hasWorkerEmitted = true; + } catch (error) { + console.error( + `Error while emitting event with event type: ${newEventType}.`, + serializeError(error) + ); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + } + }); + } +} diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts new file mode 100644 index 00000000..6aba8cc1 --- /dev/null +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -0,0 +1,535 @@ +import { AxiosResponse } from 'axios'; + +import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; +import { + AirSyncDefaultItemTypes, + EVENT_SIZE_THRESHOLD_BYTES, + SSOR_ATTACHMENT, +} from '../../common/constants'; +import { serializeError } from '../../logger/logger'; +import { + runWithSdkLogContext, + runWithUserLogContext, +} from '../../logger/logger.context'; +import { Repo } from '../../repo/repo'; +import { + NormalizedAttachment, + RepoInterface, +} from '../../repo/repo.interfaces'; +import { + AirSyncEvent, + EventData, + ExternalSystemAttachmentProcessors, + ExternalSystemAttachmentStreamingFunction, + ExtractorEventType, + ProcessAttachmentReturnType, + StreamAttachmentsReturnType, +} from '../../types/extraction'; +import { LoaderEventType } from '../../types/loading'; +import { BaseState } from '../../state/state'; +import { WorkerAdapterOptions } from '../../types/workers'; +import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; + +import { BaseAdapter } from './base-adapter'; + +/** + * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the + * extraction surface (repos, artifacts, attachment streaming) and uploads + * pending repos and updates the extraction boundaries before emitting. + * + * @public + */ +export class ExtractionAdapter< + ConnectorState +> extends BaseAdapter { + private _artifacts: Artifact[]; + private repos: Repo[] = []; + private currentEventDataLength: number = 0; + + constructor(params: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + super(params); + this._artifacts = []; + } + + /** + * Returns whether the given item type should be extracted. + * Defaults to true if the scope is empty or the item type is not listed. + */ + shouldExtract(itemType: string): boolean { + const scope = this.extractionScope; + if (Object.keys(scope).length === 0) return true; + if (!(itemType in scope)) return true; + return scope[itemType].extract; + } + + initializeRepos(repos: RepoInterface[]) { + this.repos = repos.map((repo) => { + const shouldNormalize = + repo.itemType !== AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && + repo.itemType !== SSOR_ATTACHMENT; + + return new Repo({ + event: this.event, + itemType: repo.itemType, + ...(shouldNormalize && { normalize: repo.normalize }), + onUpload: (artifact: Artifact) => { + // We need to store artifacts ids in state for later use when streaming attachments + if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { + this.sdkState.toDevRev?.attachmentsMetadata.artifactIds.push( + artifact.id + ); + } + + // Calculate size of the entire artifact object that goes in the SQS message + this.currentEventDataLength += Buffer.byteLength( + JSON.stringify(artifact), + 'utf8' + ); + + if ( + this.currentEventDataLength > EVENT_SIZE_THRESHOLD_BYTES && + !this.isTimeout + ) { + this.isTimeout = true; + } + }, + options: { + ...this.options, + ...repo.overridenOptions, + }, + }); + }); + } + + getRepo(itemType: string): Repo | undefined { + return runWithSdkLogContext(() => { + const repo = this.repos.find((repo) => repo.itemType === itemType); + + if (!repo) { + console.error(`Repo for item type ${itemType} not found.`); + return; + } + + return repo; + }); + } + + get artifacts(): Artifact[] { + return this._artifacts; + } + + set artifacts(artifacts: Artifact[]) { + this._artifacts = this._artifacts + .concat(artifacts) + .filter((value, index, self) => self.indexOf(value) === index); + } + + protected async beforeEmit( + newEventType: ExtractorEventType | LoaderEventType + ): Promise { + // Upload all repos before emitting the event + console.log( + `Uploading all repos before emitting event with event type: ${newEventType}.` + ); + await this.uploadAllRepos(); + + // If the extraction is done, we want to save the timestamp of the last successful sync + if (newEventType === ExtractorEventType.AttachmentExtractionDone) { + const sdkState = this.sdkState; + + console.log( + `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${sdkState.lastSyncStarted}).` + ); + + sdkState.lastSuccessfulSyncStarted = sdkState.lastSyncStarted; + sdkState.lastSyncStarted = ''; + + // Clear pending extraction boundaries now that the cycle is complete + sdkState.pendingWorkersOldest = ''; + sdkState.pendingWorkersNewest = ''; + + // Update workersOldest and workersNewest boundaries from resolved extraction timestamps. + // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest. + const extractionStart = this.event.payload.event_context.extract_from; + const extractionEnd = this.event.payload.event_context.extract_to; + + if ( + extractionStart && + (!sdkState.workersOldest || extractionStart < sdkState.workersOldest) + ) { + console.log( + `Updating workersOldest from '${sdkState.workersOldest}' to '${extractionStart}'.` + ); + sdkState.workersOldest = extractionStart; + } + + if ( + extractionEnd && + (!sdkState.workersNewest || extractionEnd > sdkState.workersNewest) + ) { + console.log( + `Updating workersNewest from '${sdkState.workersNewest}' to '${extractionEnd}'.` + ); + sdkState.workersNewest = extractionEnd; + } + } + } + + protected buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData { + const isExtractionEvent = Object.values(ExtractorEventType).includes( + newEventType as ExtractorEventType + ); + return isExtractionEvent ? { artifacts: this.artifacts } : {}; + } + + protected afterEmit(): void { + this.artifacts = []; + } + + /** + * Emits an extraction event. For ExternalSyncUnitExtractionDone, inline + * external sync units are uploaded via a Repo first (then stripped from the + * payload to avoid SQS size limits); `beforeEmit` uploads that repo with the + * rest before the event is sent. + * TODO: Remove the external sync unit handling in v2.0.0 + * + * @param newEventType - The event type to be emitted + * @param data - The data to be sent with the event + */ + override async emit( + newEventType: ExtractorEventType | LoaderEventType, + data?: EventData + ): Promise { + if ( + newEventType === ExtractorEventType.ExternalSyncUnitExtractionDone && + data?.external_sync_units && + data.external_sync_units.length > 0 + ) { + console.log( + `Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.` + ); + + this.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + overridenOptions: { + batchSize: 25000, + skipConfirmation: true, + }, + }, + ]); + + await this.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push( + data.external_sync_units + ); + + // Remove inline external_sync_units from data to avoid SQS size issues + delete data.external_sync_units; + } + + return super.emit(newEventType, data); + } + + async uploadAllRepos(): Promise { + for (const repo of this.repos) { + const error = await repo.upload(); + this.artifacts.push(...repo.uploadedArtifacts); + if (error) { + throw error; + } + } + } + + async processAttachment( + attachment: NormalizedAttachment, + stream: ExternalSystemAttachmentStreamingFunction + ): Promise { + return runWithSdkLogContext(async () => { + const { httpStream, delay, error } = await runWithUserLogContext( + async () => + stream({ + item: attachment, + event: this.event, + }) + ); + + if (error) { + return { error }; + } else if (delay) { + return { delay }; + } + + if (httpStream) { + const fileType = + attachment.content_type || + httpStream.headers['content-type']?.toString() || + 'application/octet-stream'; + const contentLength = httpStream.headers['content-length']?.toString(); + const fileSize = contentLength ? parseInt(contentLength) : undefined; + + // Get upload URL + const { error: artifactUrlError, response: artifactUrlResponse } = + await this.uploader.getArtifactUploadUrl( + attachment.file_name, + fileType, + fileSize + ); + + if (artifactUrlError) { + this.destroyHttpStream(httpStream); + return { + error: { + message: `Error while preparing artifact for attachment ID ${ + attachment.id + }. Skipping attachment. ${serializeError(artifactUrlError)}`, + fileSize: fileSize, + }, + }; + } + + // Stream attachment + const { error: uploadedArtifactError } = + await this.uploader.streamArtifact(artifactUrlResponse!, httpStream); + + if (uploadedArtifactError) { + this.destroyHttpStream(httpStream); + return { + error: { + message: + `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` + + serializeError(uploadedArtifactError), + fileSize: fileSize, + }, + }; + } + + // Confirm attachment upload + const { error: confirmArtifactUploadError } = + await this.uploader.confirmArtifactUpload( + artifactUrlResponse!.artifact_id + ); + if (confirmArtifactUploadError) { + return { + error: { + message: + `Error while confirming upload for attachment ID ${attachment.id}. ` + + serializeError(confirmArtifactUploadError), + fileSize: fileSize, + }, + }; + } + + const ssorAttachment: SsorAttachment = { + id: { + devrev: artifactUrlResponse!.artifact_id, + external: attachment.id, + }, + parent_id: { + external: attachment.parent_id, + }, + }; + + if (attachment.author_id) { + ssorAttachment.actor_id = { + external: attachment.author_id, + }; + } + + // This will set inline flag in ssor_attachment only if it is explicity + // set in the attachment object. + if (attachment.inline === true) { + ssorAttachment.inline = true; + } else if (attachment.inline === false) { + ssorAttachment.inline = false; + } + + await this.getRepo('ssor_attachment')?.push([ssorAttachment]); + return; + } + return { + error: { + message: `Error while opening attachment stream. Skipping attachment.`, + }, + }; + }); + } + + /** + * Destroys a stream to prevent memory leaks. + * @param httpStream - The axios response stream to destroy + */ + private destroyHttpStream(httpStream: AxiosResponse): void { + try { + if (httpStream && httpStream.data) { + if (typeof httpStream.data.destroy === 'function') { + httpStream.data.destroy(); + } else if (typeof httpStream.data.close === 'function') { + httpStream.data.close(); + } + } + } catch (error) { + console.warn('Error while destroying HTTP stream:', error); + } + } + + /** + * Streams the attachments to the DevRev platform. + * The attachments are streamed to the platform and the artifact information is returned. + * @param params - The parameters to stream the attachments + * @returns The response object containing the ssorAttachment artifact information + * or error information if there was an error + */ + async streamAttachments({ + stream, + processors, + batchSize = 1, // By default, we want to stream one attachment at a time + }: { + stream: ExternalSystemAttachmentStreamingFunction; + processors?: ExternalSystemAttachmentProcessors< + ConnectorState, + NormalizedAttachment[], + NewBatch + >; + batchSize?: number; + }): Promise { + return runWithSdkLogContext(async () => { + if (batchSize <= 0) { + console.warn( + `The specified batch size (${batchSize}) is invalid. Using 1 instead.` + ); + batchSize = 1; + } + + if (batchSize > 50) { + console.warn( + `The specified batch size (${batchSize}) is too large. Using 50 instead.` + ); + batchSize = 50; + } + + const repos = [ + { + itemType: 'ssor_attachment', + }, + ]; + this.initializeRepos(repos); + + const attachmentsMetadata = this.sdkState.toDevRev?.attachmentsMetadata; + + // If there are no attachments metadata artifact IDs in state, finish here + if (!attachmentsMetadata?.artifactIds?.length) { + console.log(`No attachments metadata artifact IDs found in state.`); + return; + } else { + console.log( + `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` + ); + } + + // Loop through the attachments metadata artifact IDs + while (attachmentsMetadata.artifactIds.length > 0) { + const attachmentsMetadataArtifactId = + attachmentsMetadata.artifactIds[0]; + + console.log( + `Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.` + ); + + const { attachments, error } = + await this.uploader.getAttachmentsFromArtifactId({ + artifact: attachmentsMetadataArtifactId, + }); + + if (error) { + console.error( + `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + return { error }; + } + + if (!attachments || attachments.length === 0) { + console.warn( + `No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + // Remove empty artifact and reset lastProcessed + attachmentsMetadata.artifactIds.shift(); + attachmentsMetadata.lastProcessed = 0; + continue; + } + + console.log( + `Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + + let response; + + if (processors) { + console.log(`Using custom processors for attachments.`); + + const reducer = processors.reducer; + const iterator = processors.iterator; + + const reducedAttachments = runWithUserLogContext(() => + reducer({ + attachments, + adapter: this, + batchSize, + }) + ); + + response = await runWithUserLogContext(async () => { + return await iterator({ + reducedAttachments, + adapter: this, + stream, + }); + }); + } else { + console.log( + `Using attachments streaming pool for attachments streaming.` + ); + + const attachmentsPool = new AttachmentsStreamingPool({ + adapter: this, + attachments, + batchSize, + stream, + }); + + response = await attachmentsPool.streamAll(); + } + + if (response?.delay || response?.error) { + return response; + } + + // On timeout, emit progress and exit to allow continuation. + if (this.isTimeout) { + console.log( + `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.` + ); + await this.emit(ExtractorEventType.AttachmentExtractionProgress); + process.exit(0); + return; + } + + console.log( + `Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + attachmentsMetadata.artifactIds.shift(); + attachmentsMetadata.lastProcessed = 0; + if (attachmentsMetadata.lastProcessedAttachmentsIdsList) { + attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0; + } + } + + return; + }); + } +} diff --git a/src/multithreading/worker-adapter/worker-adapter.helpers.ts b/src/multithreading/adapters/loading-adapter.helpers.ts similarity index 100% rename from src/multithreading/worker-adapter/worker-adapter.helpers.ts rename to src/multithreading/adapters/loading-adapter.helpers.ts diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts new file mode 100644 index 00000000..66a3fd20 --- /dev/null +++ b/src/multithreading/adapters/loading-adapter.ts @@ -0,0 +1,645 @@ +import axios from 'axios'; + +import { serializeError } from '../../logger/logger'; +import { + runWithSdkLogContext, + runWithUserLogContext, +} from '../../logger/logger.context'; +import { Mappers } from '../../mappers/mappers'; +import { SyncMapperRecordStatus } from '../../mappers/mappers.interface'; +import { BaseState } from '../../state/state'; +import { + AirSyncEvent, + EventData, + EventType, + ExtractorEventType, +} from '../../types/extraction'; +import { + ActionType, + ExternalSystemAttachment, + ExternalSystemItem, + ExternalSystemLoadingFunction, + FileToLoad, + ItemTypesToLoadParams, + ItemTypeToLoad, + LoaderEventType, + LoaderReport, + LoadItemResponse, + LoadItemTypesResponse, + StatsFileObject, +} from '../../types/loading'; +import { WorkerAdapterOptions } from '../../types/workers'; + +import { BaseAdapter } from './base-adapter'; +import { + addReportToLoaderReport, + getFilesToLoad, +} from './loading-adapter.helpers'; + +/** + * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading + * surface (item/attachment loading, mappers, loader reports). + * + * @public + */ +export class LoadingAdapter< + ConnectorState +> extends BaseAdapter { + private loaderReports: LoaderReport[]; + private _processedFiles: string[]; + private _mappers: Mappers; + + constructor(params: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + super(params); + this.loaderReports = []; + this._processedFiles = []; + this._mappers = new Mappers({ + event: params.event, + options: params.options, + }); + } + + get reports(): LoaderReport[] { + return this.loaderReports; + } + + get processedFiles(): string[] { + return this._processedFiles; + } + + get mappers(): Mappers { + return this._mappers; + } + + protected async beforeEmit(): Promise { + // Loading has no pre-emit work (no repos, no extraction boundaries). + } + + protected buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData { + const isLoaderEvent = Object.values(LoaderEventType).includes( + newEventType as LoaderEventType + ); + return isLoaderEvent + ? { + reports: this.reports, + processed_files: this.processedFiles, + } + : {}; + } + + protected afterEmit(): void { + // Loading keeps its accumulated reports/processed files across emits. + } + + async loadItemTypes({ + itemTypesToLoad, + }: ItemTypesToLoadParams): Promise { + return runWithSdkLogContext(async () => { + if (this.event.payload.event_type === EventType.StartLoadingData) { + const itemTypes = itemTypesToLoad.map( + (itemTypeToLoad) => itemTypeToLoad.itemType + ); + + if (!itemTypes.length) { + console.warn('No item types to load, returning.'); + return { + reports: this.reports, + processed_files: this.processedFiles, + }; + } + + const filesToLoad = await this.getLoaderBatches({ + supportedItemTypes: itemTypes, + }); + this.sdkState.fromDevRev = { + filesToLoad, + }; + } + + if ( + !this.sdkState.fromDevRev || + !this.sdkState.fromDevRev.filesToLoad.length + ) { + console.warn('No files to load, returning.'); + return { + reports: this.reports, + processed_files: this.processedFiles, + }; + } + + console.log( + 'Files to load in state', + this.sdkState.fromDevRev?.filesToLoad + ); + + try { + outerloop: for (const fileToLoad of this.sdkState.fromDevRev + .filesToLoad) { + const itemTypeToLoad = itemTypesToLoad.find( + (itemTypeToLoad: ItemTypeToLoad) => + itemTypeToLoad.itemType === fileToLoad.itemType + ); + + if (!itemTypeToLoad) { + console.error( + `Item type to load not found for item type: ${fileToLoad.itemType}.` + ); + + await this.emit(LoaderEventType.DataLoadingError, { + error: { + message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, + }, + }); + + break; + } + + if (!fileToLoad.completed) { + const { response, error: transformerFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: fileToLoad.id, + isGzipped: true, + }); + + if (transformerFileError) { + console.error( + `Transformer file not found for artifact ID: ${fileToLoad.id}.` + ); + await this.emit(LoaderEventType.DataLoadingError, { + error: { + message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, + }, + }); + break outerloop; + } + + const transformerFile = response as ExternalSystemItem[]; + + for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { + if (this.isTimeout) { + console.log( + 'Timeout detected during data loading. Emitting progress to allow continuation.' + ); + await this.emit(LoaderEventType.DataLoadingProgress); + process.exit(0); + } + + const { report, rateLimit } = await this.loadItem({ + item: transformerFile[i], + itemTypeToLoad, + }); + + if (rateLimit?.delay) { + await this.emit(LoaderEventType.DataLoadingDelayed, { + delay: rateLimit.delay, + reports: this.reports, + processed_files: this.processedFiles, + }); + + break outerloop; + } + + if (report) { + addReportToLoaderReport({ + loaderReports: this.loaderReports, + report, + }); + fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; + } + } + + fileToLoad.completed = true; + this._processedFiles.push(fileToLoad.id); + } + } + } catch (error) { + console.error('Error during data loading.', serializeError(error)); + await this.emit(LoaderEventType.DataLoadingError, { + error: { + message: `Error during data loading. ${serializeError(error)}`, + }, + }); + process.exit(1); + } + + return { + reports: this.reports, + processed_files: this.processedFiles, + }; + }); + } + + async getLoaderBatches({ + supportedItemTypes, + }: { + supportedItemTypes: string[]; + }) { + return runWithSdkLogContext(async () => { + const statsFileArtifactId = this.event.payload.event_data?.stats_file; + + if (statsFileArtifactId) { + const { response, error: statsFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: statsFileArtifactId, + }); + + const statsFile = response as StatsFileObject[]; + + if (statsFileError || statsFile.length === 0) { + return [] as FileToLoad[]; + } + + const filesToLoad = getFilesToLoad({ + supportedItemTypes, + statsFile, + }); + + return filesToLoad; + } + + return [] as FileToLoad[]; + }); + } + + async loadAttachments({ + create, + }: { + create: ExternalSystemLoadingFunction; + }): Promise { + return runWithSdkLogContext(async () => { + if (this.event.payload.event_type === EventType.StartLoadingAttachments) { + this.sdkState.fromDevRev = { + filesToLoad: await this.getLoaderBatches({ + supportedItemTypes: ['attachment'], + }), + }; + } + + if ( + !this.sdkState.fromDevRev || + this.sdkState.fromDevRev?.filesToLoad.length === 0 + ) { + console.log('No files to load, returning.'); + return { + reports: this.reports, + processed_files: this.processedFiles, + }; + } + + const filesToLoad = this.sdkState.fromDevRev?.filesToLoad; + + try { + outerloop: for (const fileToLoad of filesToLoad) { + if (!fileToLoad.completed) { + const { response, error: transformerFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: fileToLoad.id, + isGzipped: true, + }); + + const transformerFile = response as ExternalSystemAttachment[]; + + if (transformerFileError) { + console.error( + `Transformer file not found for artifact ID: ${fileToLoad.id}.` + ); + await this.emit(LoaderEventType.AttachmentLoadingError, { + error: { + message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, + }, + }); + break outerloop; + } + + for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { + if (this.isTimeout) { + console.log( + 'Timeout detected during attachment loading. Emitting progress to allow continuation.' + ); + await this.emit(LoaderEventType.AttachmentLoadingProgress); + process.exit(0); + } + + const { report, rateLimit } = await this.loadAttachment({ + item: transformerFile[i], + create, + }); + + if (rateLimit?.delay) { + await this.emit(LoaderEventType.AttachmentLoadingDelayed, { + delay: rateLimit.delay, + reports: this.reports, + processed_files: this.processedFiles, + }); + + break outerloop; + } + + if (report) { + addReportToLoaderReport({ + loaderReports: this.loaderReports, + report, + }); + fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; + } + } + + fileToLoad.completed = true; + this._processedFiles.push(fileToLoad.id); + } + } + } catch (error) { + console.error( + 'Error during attachment loading.', + serializeError(error) + ); + await this.emit(LoaderEventType.AttachmentLoadingError, { + error: { + message: `Error during attachment loading. ${serializeError( + error + )}`, + }, + }); + process.exit(1); + } + + return { + reports: this.reports, + processed_files: this.processedFiles, + }; + }); + } + + async loadItem({ + item, + itemTypeToLoad, + }: { + item: ExternalSystemItem; + itemTypeToLoad: ItemTypeToLoad; + }): Promise { + return runWithSdkLogContext(async () => { + const devrevId = item.id.devrev; + + try { + const syncMapperRecordResponse = await this._mappers.getByTargetId({ + sync_unit: this.event.payload.event_context.sync_unit, + target: devrevId, + }); + + const syncMapperRecord = syncMapperRecordResponse.data; + if (!syncMapperRecord) { + console.warn('Failed to get sync mapper record from response.'); + return { + error: { + message: 'Failed to get sync mapper record from response.', + }, + }; + } + + // Update item in external system + const { id, modifiedDate, delay, error } = await runWithUserLogContext( + async () => { + return await itemTypeToLoad.update({ + item, + mappers: this._mappers, + event: this.event, + }); + } + ); + + if (id) { + try { + const syncMapperRecordUpdateResponse = await this._mappers.update({ + id: syncMapperRecord.sync_mapper_record.id, + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + ...(modifiedDate && { + external_versions: { + add: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }, + }), + external_ids: { + add: [id], + }, + targets: { + add: [devrevId], + }, + }); + + console.log( + 'Successfully updated sync mapper record.', + syncMapperRecordUpdateResponse.data + ); + } catch (error) { + console.warn( + 'Failed to update sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to update sync mapper record' + serializeError(error), + }, + }; + } + + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.UPDATED]: 1, + }, + }; + } else if (delay) { + console.log( + `Rate limited while updating item in external system, delaying for ${delay} seconds.` + ); + + return { + rateLimit: { + delay, + }, + }; + } else { + console.warn('Failed to update item in external system', error); + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.FAILED]: 1, + }, + }; + } + + // TODO: Update mapper (optional) + } catch (error) { + if (axios.isAxiosError(error)) { + if (error.response?.status === 404) { + // Create item in external system if mapper record not found + const { id, modifiedDate, delay, error } = + await runWithUserLogContext(async () => { + return await itemTypeToLoad.create({ + item, + mappers: this._mappers, + event: this.event, + }); + }); + + if (id) { + // Create mapper + try { + const syncMapperRecordCreateResponse = + await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + external_ids: [id], + targets: [devrevId], + ...(modifiedDate && { + external_versions: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }), + }); + + console.log( + 'Successfully created sync mapper record.', + syncMapperRecordCreateResponse.data + ); + + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.CREATED]: 1, + }, + }; + } catch (error) { + console.warn( + 'Failed to create sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to create sync mapper record. ' + + serializeError(error), + }, + }; + } + } else if (delay) { + return { + rateLimit: { + delay, + }, + }; + } else { + console.warn( + 'Failed to create item in external system.', + serializeError(error) + ); + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.FAILED]: 1, + }, + }; + } + } else { + console.warn( + 'Failed to get sync mapper record.', + serializeError(error) + ); + return { + error: { + message: error.message, + }, + }; + } + } + + console.warn( + 'Failed to get sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to get sync mapper record. ' + serializeError(error), + }, + }; + } + }); + } + + async loadAttachment({ + item, + create, + }: { + item: ExternalSystemAttachment; + create: ExternalSystemLoadingFunction; + }): Promise { + return runWithSdkLogContext(async () => { + // Create item + const { id, delay, error } = await runWithUserLogContext(async () => + create({ + item, + mappers: this._mappers, + event: this.event, + }) + ); + + if (delay) { + return { + rateLimit: { + delay, + }, + }; + } else if (id) { + try { + const syncMapperRecordCreateResponse = await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + external_ids: [id], + targets: [item.reference_id], + status: SyncMapperRecordStatus.OPERATIONAL, + }); + + console.log( + 'Successfully created sync mapper record.', + syncMapperRecordCreateResponse.data + ); + } catch (error) { + console.warn( + 'Failed to create sync mapper record.', + serializeError(error) + ); + } + + return { + report: { + item_type: 'attachment', + [ActionType.CREATED]: 1, + }, + }; + } else { + console.warn('Failed to create attachment in external system', error); + return { + report: { + item_type: 'attachment', + [ActionType.FAILED]: 1, + }, + }; + } + }); + } +} diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 0818f55f..619bf854 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -5,12 +5,15 @@ import { runWithUserLogContext, } from '../logger/logger.context'; import { createAdapterState } from '../state/state'; +import { SyncMode } from '../types/common'; import { ProcessTaskInterface, + WorkerAdapter, WorkerEvent, WorkerMessageSubject, } from '../types/workers'; -import { WorkerAdapter } from './worker-adapter/worker-adapter'; +import { ExtractionAdapter } from './adapters/extraction-adapter'; +import { LoadingAdapter } from './adapters/loading-adapter'; export function processTask({ task, @@ -38,11 +41,18 @@ export function processTask({ options, }); - const adapter = new WorkerAdapter({ - event, - adapterState, - options, - }); + const adapter: WorkerAdapter = + event.payload.event_context.mode === SyncMode.LOADING + ? new LoadingAdapter({ + event, + adapterState, + options, + }) + : new ExtractionAdapter({ + event, + adapterState, + options, + }); parentPort?.on(WorkerEvent.WorkerMessage, (message) => { if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts deleted file mode 100644 index aae497a5..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ /dev/null @@ -1,1244 +0,0 @@ -import axios, { AxiosResponse } from 'axios'; -import { parentPort } from 'node:worker_threads'; -import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; -import { - AirSyncDefaultItemTypes, - EVENT_SIZE_THRESHOLD_BYTES, - SSOR_ATTACHMENT, - STATELESS_EVENT_TYPES, -} from '../../common/constants'; -import { emit } from '../../common/control-protocol'; -import { - addReportToLoaderReport, - getFilesToLoad, -} from './worker-adapter.helpers'; -import { serializeError } from '../../logger/logger'; -import { - runWithSdkLogContext, - runWithUserLogContext, -} from '../../logger/logger.context'; -import { Mappers } from '../../mappers/mappers'; -import { SyncMapperRecordStatus } from '../../mappers/mappers.interface'; -import { Repo } from '../../repo/repo'; -import { - NormalizedAttachment, - RepoInterface, -} from '../../repo/repo.interfaces'; -import { BaseState } from '../../state/state'; -import { SdkState } from '../../state/state.interfaces'; -import { - AirSyncEvent, - EventData, - EventType, - ExternalSystemAttachmentProcessors, - ExternalSystemAttachmentStreamingFunction, - ExtractorEventType, - ProcessAttachmentReturnType, - StreamAttachmentsReturnType, -} from '../../types/extraction'; -import { - ActionType, - ExternalSystemAttachment, - ExternalSystemItem, - ExternalSystemLoadingFunction, - FileToLoad, - ItemTypesToLoadParams, - ItemTypeToLoad, - LoaderEventType, - LoaderReport, - LoadItemResponse, - LoadItemTypesResponse, - StatsFileObject, -} from '../../types/loading'; -import { - WorkerAdapterInterface, - WorkerAdapterOptions, - WorkerMessageEmitted, - WorkerMessageSubject, -} from '../../types/workers'; -import { Uploader } from '../../uploader/uploader'; -import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; -import { truncateMessage } from '../../common/helpers'; - -export function createWorkerAdapter({ - event, - adapterState, - options, -}: WorkerAdapterInterface): WorkerAdapter { - return new WorkerAdapter({ - event, - adapterState, - options, - }); -} - -/** - * WorkerAdapter class is used to interact with AirSync platform. It is passed to the snap-in - * as parameter in processTask and onTimeout functions. The class provides - * utilities to emit control events to the platform, update the state of the connector, - * and upload artifacts to the platform. - * @class WorkerAdapter - * @constructor - * @param options - The options to create a new instance of WorkerAdapter class - * @param event - The event object received from the platform - * @param initialState - The initial state of the adapter - * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @param workerPath - The path to the worker file - * - * @public - */ -export class WorkerAdapter { - readonly event: AirSyncEvent; - readonly options?: WorkerAdapterOptions; - isTimeout: boolean; - hasWorkerEmitted: boolean; - - private adapterState: BaseState; - private _artifacts: Artifact[]; - private repos: Repo[] = []; - private currentEventDataLength: number = 0; - - // Loader - private loaderReports: LoaderReport[]; - private _processedFiles: string[]; - private _mappers: Mappers; - private uploader: Uploader; - - constructor({ - event, - adapterState, - options, - }: WorkerAdapterInterface) { - this.event = event; - this.options = options; - this.adapterState = adapterState; - this._artifacts = []; - this.hasWorkerEmitted = false; - this.isTimeout = false; - - // Loader - this.loaderReports = []; - this._processedFiles = []; - this._mappers = new Mappers({ - event, - options, - }); - this.uploader = new Uploader({ - event, - options, - }); - } - - /** Connector-owned state exposed to snap-in code. */ - get state(): ConnectorState { - return this.adapterState.state; - } - - set state(value: ConnectorState) { - this.adapterState.state = value; - } - - /** SDK-internal bookkeeping state. Used by SDK internals; not for connector use. */ - get sdkState(): SdkState { - return this.adapterState.sdkState; - } - - get reports(): LoaderReport[] { - return this.loaderReports; - } - - get processedFiles(): string[] { - return this._processedFiles; - } - - get mappers(): Mappers { - return this._mappers; - } - - get extractionScope() { - return this.adapterState.extractionScope; - } - - /** - * Returns whether the given item type should be extracted. - * Defaults to true if the scope is empty or the item type is not listed. - */ - shouldExtract(itemType: string): boolean { - const scope = this.extractionScope; - if (Object.keys(scope).length === 0) return true; - if (!(itemType in scope)) return true; - return scope[itemType].extract; - } - - initializeRepos(repos: RepoInterface[]) { - this.repos = repos.map((repo) => { - const shouldNormalize = - repo.itemType !== AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && - repo.itemType !== SSOR_ATTACHMENT; - - return new Repo({ - event: this.event, - itemType: repo.itemType, - ...(shouldNormalize && { normalize: repo.normalize }), - onUpload: (artifact: Artifact) => { - // We need to store artifacts ids in state for later use when streaming attachments - if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { - this.sdkState.toDevRev?.attachmentsMetadata.artifactIds.push( - artifact.id - ); - } - - // Calculate size of the entire artifact object that goes in the SQS message - this.currentEventDataLength += Buffer.byteLength( - JSON.stringify(artifact), - 'utf8' - ); - - if ( - this.currentEventDataLength > EVENT_SIZE_THRESHOLD_BYTES && - !this.isTimeout - ) { - this.isTimeout = true; - } - }, - options: { - ...this.options, - ...repo.overridenOptions, - }, - }); - }); - } - - getRepo(itemType: string): Repo | undefined { - return runWithSdkLogContext(() => { - const repo = this.repos.find((repo) => repo.itemType === itemType); - - if (!repo) { - console.error(`Repo for item type ${itemType} not found.`); - return; - } - - return repo; - }); - } - - async postState() { - return runWithSdkLogContext(async () => { - await this.adapterState.postState(); - }); - } - - get artifacts(): Artifact[] { - return this._artifacts; - } - - set artifacts(artifacts: Artifact[]) { - this._artifacts = this._artifacts - .concat(artifacts) - .filter((value, index, self) => self.indexOf(value) === index); - } - - /** - * Emits an event to the platform. - * - * @param newEventType - The event type to be emitted - * @param data - The data to be sent with the event - */ - async emit( - newEventType: ExtractorEventType | LoaderEventType, - data?: EventData - ): Promise { - return runWithSdkLogContext(async () => { - if (this.hasWorkerEmitted) { - console.warn( - `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` - ); - return; - } - - // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting - // TODO: Remove in v2.0.0 - if ( - newEventType === ExtractorEventType.ExternalSyncUnitExtractionDone && - data?.external_sync_units && - data.external_sync_units.length > 0 - ) { - console.log( - `Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.` - ); - - this.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: 25000, - skipConfirmation: true, - }, - }, - ]); - - await this.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push( - data.external_sync_units - ); - - // Remove inline external_sync_units from data to avoid SQS size issues - delete data.external_sync_units; - } - - // Upload all repos before emitting the event - console.log( - `Uploading all repos before emitting event with event type: ${newEventType}.` - ); - - try { - await this.uploadAllRepos(); - } catch (error) { - console.error('Error while uploading repos', error); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - return; - } - - // If the extraction is done, we want to save the timestamp of the last successful sync - if (newEventType === ExtractorEventType.AttachmentExtractionDone) { - console.log( - `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.sdkState.lastSyncStarted}).` - ); - - this.sdkState.lastSuccessfulSyncStarted = this.sdkState.lastSyncStarted; - this.sdkState.lastSyncStarted = ''; - - // Clear pending extraction boundaries now that the cycle is complete - this.sdkState.pendingWorkersOldest = ''; - this.sdkState.pendingWorkersNewest = ''; - - // Update workersOldest and workersNewest boundaries from resolved extraction timestamps. - // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest. - const extractionStart = this.event.payload.event_context.extract_from; - const extractionEnd = this.event.payload.event_context.extract_to; - - if ( - extractionStart && - (!this.sdkState.workersOldest || - extractionStart < this.sdkState.workersOldest) - ) { - console.log( - `Updating workersOldest from '${this.sdkState.workersOldest}' to '${extractionStart}'.` - ); - this.sdkState.workersOldest = extractionStart; - } - - if ( - extractionEnd && - (!this.sdkState.workersNewest || - extractionEnd > this.sdkState.workersNewest) - ) { - console.log( - `Updating workersNewest from '${this.sdkState.workersNewest}' to '${extractionEnd}'.` - ); - this.sdkState.workersNewest = extractionEnd; - } - } - - // We want to save the state every time we emit an event, except for the start and delete events - if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { - console.log( - `Saving state before emitting event with event type: ${newEventType}.` - ); - - try { - await this.adapterState.postState(this.state); - } catch (error) { - console.error('Error while posting state', error); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - return; - } - } - - try { - // Always prune error messages to make them shorter before emit - if (data?.error?.message) { - data.error.message = truncateMessage(data.error.message); - } - - const isExtractionEvent = Object.values(ExtractorEventType).includes( - newEventType as ExtractorEventType - ); - const isLoaderEvent = Object.values(LoaderEventType).includes( - newEventType as LoaderEventType - ); - - await emit({ - eventType: newEventType, - event: this.event, - data: { - ...data, - ...(isExtractionEvent ? { artifacts: this.artifacts } : {}), - ...(isLoaderEvent - ? { reports: this.reports, processed_files: this.processedFiles } - : {}), - }, - }); - - const message: WorkerMessageEmitted = { - subject: WorkerMessageSubject.WorkerMessageEmitted, - payload: { eventType: newEventType }, - }; - this.artifacts = []; - parentPort?.postMessage(message); - this.hasWorkerEmitted = true; - } catch (error) { - console.error( - `Error while emitting event with event type: ${newEventType}.`, - serializeError(error) - ); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - } - }); - } - - async uploadAllRepos(): Promise { - for (const repo of this.repos) { - const error = await repo.upload(); - this.artifacts.push(...repo.uploadedArtifacts); - if (error) { - throw error; - } - } - } - - async loadItemTypes({ - itemTypesToLoad, - }: ItemTypesToLoadParams): Promise { - return runWithSdkLogContext(async () => { - if (this.event.payload.event_type === EventType.StartLoadingData) { - const itemTypes = itemTypesToLoad.map( - (itemTypeToLoad) => itemTypeToLoad.itemType - ); - - if (!itemTypes.length) { - console.warn('No item types to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - const filesToLoad = await this.getLoaderBatches({ - supportedItemTypes: itemTypes, - }); - this.adapterState.sdkState.fromDevRev = { - filesToLoad, - }; - } - - if ( - !this.adapterState.sdkState.fromDevRev || - !this.adapterState.sdkState.fromDevRev.filesToLoad.length - ) { - console.warn('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - console.log( - 'Files to load in state', - this.adapterState.sdkState.fromDevRev?.filesToLoad - ); - - try { - outerloop: for (const fileToLoad of this.adapterState.sdkState - .fromDevRev.filesToLoad) { - const itemTypeToLoad = itemTypesToLoad.find( - (itemTypeToLoad: ItemTypeToLoad) => - itemTypeToLoad.itemType === fileToLoad.itemType - ); - - if (!itemTypeToLoad) { - console.error( - `Item type to load not found for item type: ${fileToLoad.itemType}.` - ); - - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, - }, - }); - - break; - } - - if (!fileToLoad.completed) { - const { response, error: transformerFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: fileToLoad.id, - isGzipped: true, - }); - - if (transformerFileError) { - console.error( - `Transformer file not found for artifact ID: ${fileToLoad.id}.` - ); - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, - }, - }); - break outerloop; - } - - const transformerFile = response as ExternalSystemItem[]; - - for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { - if (this.isTimeout) { - console.log( - 'Timeout detected during data loading. Emitting progress to allow continuation.' - ); - await this.emit(LoaderEventType.DataLoadingProgress); - process.exit(0); - } - - const { report, rateLimit } = await this.loadItem({ - item: transformerFile[i], - itemTypeToLoad, - }); - - if (rateLimit?.delay) { - await this.emit(LoaderEventType.DataLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; - } - - if (report) { - addReportToLoaderReport({ - loaderReports: this.loaderReports, - report, - }); - fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; - } - } - - fileToLoad.completed = true; - this._processedFiles.push(fileToLoad.id); - } - } - } catch (error) { - console.error('Error during data loading.', serializeError(error)); - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Error during data loading. ${serializeError(error)}`, - }, - }); - process.exit(1); - } - - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - }); - } - - async getLoaderBatches({ - supportedItemTypes, - }: { - supportedItemTypes: string[]; - }) { - return runWithSdkLogContext(async () => { - const statsFileArtifactId = this.event.payload.event_data?.stats_file; - - if (statsFileArtifactId) { - const { response, error: statsFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: statsFileArtifactId, - }); - - const statsFile = response as StatsFileObject[]; - - if (statsFileError || statsFile.length === 0) { - return [] as FileToLoad[]; - } - - const filesToLoad = getFilesToLoad({ - supportedItemTypes, - statsFile, - }); - - return filesToLoad; - } - - return [] as FileToLoad[]; - }); - } - - async loadAttachments({ - create, - }: { - create: ExternalSystemLoadingFunction; - }): Promise { - return runWithSdkLogContext(async () => { - if (this.event.payload.event_type === EventType.StartLoadingAttachments) { - this.adapterState.sdkState.fromDevRev = { - filesToLoad: await this.getLoaderBatches({ - supportedItemTypes: ['attachment'], - }), - }; - } - - if ( - !this.adapterState.sdkState.fromDevRev || - this.adapterState.sdkState.fromDevRev?.filesToLoad.length === 0 - ) { - console.log('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - const filesToLoad = this.adapterState.sdkState.fromDevRev?.filesToLoad; - - try { - outerloop: for (const fileToLoad of filesToLoad) { - if (!fileToLoad.completed) { - const { response, error: transformerFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: fileToLoad.id, - isGzipped: true, - }); - - const transformerFile = response as ExternalSystemAttachment[]; - - if (transformerFileError) { - console.error( - `Transformer file not found for artifact ID: ${fileToLoad.id}.` - ); - await this.emit(LoaderEventType.AttachmentLoadingError, { - error: { - message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, - }, - }); - break outerloop; - } - - for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { - if (this.isTimeout) { - console.log( - 'Timeout detected during attachment loading. Emitting progress to allow continuation.' - ); - await this.emit(LoaderEventType.AttachmentLoadingProgress); - process.exit(0); - } - - const { report, rateLimit } = await this.loadAttachment({ - item: transformerFile[i], - create, - }); - - if (rateLimit?.delay) { - await this.emit(LoaderEventType.AttachmentLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; - } - - if (report) { - addReportToLoaderReport({ - loaderReports: this.loaderReports, - report, - }); - fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; - } - } - - fileToLoad.completed = true; - this._processedFiles.push(fileToLoad.id); - } - } - } catch (error) { - console.error( - 'Error during attachment loading.', - serializeError(error) - ); - await this.emit(LoaderEventType.AttachmentLoadingError, { - error: { - message: `Error during attachment loading. ${serializeError( - error - )}`, - }, - }); - process.exit(1); - } - - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - }); - } - - async loadItem({ - item, - itemTypeToLoad, - }: { - item: ExternalSystemItem; - itemTypeToLoad: ItemTypeToLoad; - }): Promise { - return runWithSdkLogContext(async () => { - const devrevId = item.id.devrev; - - try { - const syncMapperRecordResponse = await this._mappers.getByTargetId({ - sync_unit: this.event.payload.event_context.sync_unit, - target: devrevId, - }); - - const syncMapperRecord = syncMapperRecordResponse.data; - if (!syncMapperRecord) { - console.warn('Failed to get sync mapper record from response.'); - return { - error: { - message: 'Failed to get sync mapper record from response.', - }, - }; - } - - // Update item in external system - const { id, modifiedDate, delay, error } = await runWithUserLogContext( - async () => { - return await itemTypeToLoad.update({ - item, - mappers: this._mappers, - event: this.event, - }); - } - ); - - if (id) { - try { - const syncMapperRecordUpdateResponse = await this._mappers.update({ - id: syncMapperRecord.sync_mapper_record.id, - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - ...(modifiedDate && { - external_versions: { - add: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }, - }), - external_ids: { - add: [id], - }, - targets: { - add: [devrevId], - }, - }); - - console.log( - 'Successfully updated sync mapper record.', - syncMapperRecordUpdateResponse.data - ); - } catch (error) { - console.warn( - 'Failed to update sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to update sync mapper record' + serializeError(error), - }, - }; - } - - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.UPDATED]: 1, - }, - }; - } else if (delay) { - console.log( - `Rate limited while updating item in external system, delaying for ${delay} seconds.` - ); - - return { - rateLimit: { - delay, - }, - }; - } else { - console.warn('Failed to update item in external system', error); - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.FAILED]: 1, - }, - }; - } - - // TODO: Update mapper (optional) - } catch (error) { - if (axios.isAxiosError(error)) { - if (error.response?.status === 404) { - // Create item in external system if mapper record not found - const { id, modifiedDate, delay, error } = - await runWithUserLogContext(async () => { - return await itemTypeToLoad.create({ - item, - mappers: this._mappers, - event: this.event, - }); - }); - - if (id) { - // Create mapper - try { - const syncMapperRecordCreateResponse = - await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - external_ids: [id], - targets: [devrevId], - ...(modifiedDate && { - external_versions: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }), - }); - - console.log( - 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data - ); - - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.CREATED]: 1, - }, - }; - } catch (error) { - console.warn( - 'Failed to create sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to create sync mapper record. ' + - serializeError(error), - }, - }; - } - } else if (delay) { - return { - rateLimit: { - delay, - }, - }; - } else { - console.warn( - 'Failed to create item in external system.', - serializeError(error) - ); - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.FAILED]: 1, - }, - }; - } - } else { - console.warn( - 'Failed to get sync mapper record.', - serializeError(error) - ); - return { - error: { - message: error.message, - }, - }; - } - } - - console.warn( - 'Failed to get sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to get sync mapper record. ' + serializeError(error), - }, - }; - } - }); - } - - async processAttachment( - attachment: NormalizedAttachment, - stream: ExternalSystemAttachmentStreamingFunction - ): Promise { - return runWithSdkLogContext(async () => { - const { httpStream, delay, error } = await runWithUserLogContext( - async () => - stream({ - item: attachment, - event: this.event, - }) - ); - - if (error) { - return { error }; - } else if (delay) { - return { delay }; - } - - if (httpStream) { - const fileType = - attachment.content_type || - httpStream.headers['content-type']?.toString() || - 'application/octet-stream'; - const contentLength = httpStream.headers['content-length']?.toString(); - const fileSize = contentLength ? parseInt(contentLength) : undefined; - - // Get upload URL - const { error: artifactUrlError, response: artifactUrlResponse } = - await this.uploader.getArtifactUploadUrl( - attachment.file_name, - fileType, - fileSize - ); - - if (artifactUrlError) { - this.destroyHttpStream(httpStream); - return { - error: { - message: `Error while preparing artifact for attachment ID ${ - attachment.id - }. Skipping attachment. ${serializeError(artifactUrlError)}`, - fileSize: fileSize, - }, - }; - } - - // Stream attachment - const { error: uploadedArtifactError } = - await this.uploader.streamArtifact(artifactUrlResponse!, httpStream); - - if (uploadedArtifactError) { - this.destroyHttpStream(httpStream); - return { - error: { - message: - `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` + - serializeError(uploadedArtifactError), - fileSize: fileSize, - }, - }; - } - - // Confirm attachment upload - const { error: confirmArtifactUploadError } = - await this.uploader.confirmArtifactUpload( - artifactUrlResponse!.artifact_id - ); - if (confirmArtifactUploadError) { - return { - error: { - message: - `Error while confirming upload for attachment ID ${attachment.id}. ` + - serializeError(confirmArtifactUploadError), - fileSize: fileSize, - }, - }; - } - - const ssorAttachment: SsorAttachment = { - id: { - devrev: artifactUrlResponse!.artifact_id, - external: attachment.id, - }, - parent_id: { - external: attachment.parent_id, - }, - }; - - if (attachment.author_id) { - ssorAttachment.actor_id = { - external: attachment.author_id, - }; - } - - // This will set inline flag in ssor_attachment only if it is explicity - // set in the attachment object. - if (attachment.inline === true) { - ssorAttachment.inline = true; - } else if (attachment.inline === false) { - ssorAttachment.inline = false; - } - - await this.getRepo('ssor_attachment')?.push([ssorAttachment]); - return; - } - return { - error: { - message: `Error while opening attachment stream. Skipping attachment.`, - }, - }; - }); - } - - /** - * Destroys a stream to prevent memory leaks. - * @param httpStream - The axios response stream to destroy - */ - private destroyHttpStream(httpStream: AxiosResponse): void { - try { - if (httpStream && httpStream.data) { - if (typeof httpStream.data.destroy === 'function') { - httpStream.data.destroy(); - } else if (typeof httpStream.data.close === 'function') { - httpStream.data.close(); - } - } - } catch (error) { - console.warn('Error while destroying HTTP stream:', error); - } - } - - async loadAttachment({ - item, - create, - }: { - item: ExternalSystemAttachment; - create: ExternalSystemLoadingFunction; - }): Promise { - return runWithSdkLogContext(async () => { - // Create item - const { id, delay, error } = await runWithUserLogContext(async () => - create({ - item, - mappers: this._mappers, - event: this.event, - }) - ); - - if (delay) { - return { - rateLimit: { - delay, - }, - }; - } else if (id) { - try { - const syncMapperRecordCreateResponse = await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - external_ids: [id], - targets: [item.reference_id], - status: SyncMapperRecordStatus.OPERATIONAL, - }); - - console.log( - 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data - ); - } catch (error) { - console.warn( - 'Failed to create sync mapper record.', - serializeError(error) - ); - } - - return { - report: { - item_type: 'attachment', - [ActionType.CREATED]: 1, - }, - }; - } else { - console.warn('Failed to create attachment in external system', error); - return { - report: { - item_type: 'attachment', - [ActionType.FAILED]: 1, - }, - }; - } - }); - } - - /** - * Streams the attachments to the DevRev platform. - * The attachments are streamed to the platform and the artifact information is returned. - * @param params - The parameters to stream the attachments - * @returns The response object containing the ssorAttachment artifact information - * or error information if there was an error - */ - async streamAttachments({ - stream, - processors, - batchSize = 1, // By default, we want to stream one attachment at a time - }: { - stream: ExternalSystemAttachmentStreamingFunction; - processors?: ExternalSystemAttachmentProcessors< - ConnectorState, - NormalizedAttachment[], - NewBatch - >; - batchSize?: number; - }): Promise { - return runWithSdkLogContext(async () => { - if (batchSize <= 0) { - console.warn( - `The specified batch size (${batchSize}) is invalid. Using 1 instead.` - ); - batchSize = 1; - } - - if (batchSize > 50) { - console.warn( - `The specified batch size (${batchSize}) is too large. Using 50 instead.` - ); - batchSize = 50; - } - - const repos = [ - { - itemType: 'ssor_attachment', - }, - ]; - this.initializeRepos(repos); - - const attachmentsMetadata = this.sdkState.toDevRev?.attachmentsMetadata; - - // If there are no attachments metadata artifact IDs in state, finish here - if (!attachmentsMetadata?.artifactIds?.length) { - console.log(`No attachments metadata artifact IDs found in state.`); - return; - } else { - console.log( - `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` - ); - } - - // Loop through the attachments metadata artifact IDs - while (attachmentsMetadata.artifactIds.length > 0) { - const attachmentsMetadataArtifactId = - attachmentsMetadata.artifactIds[0]; - - console.log( - `Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.` - ); - - const { attachments, error } = - await this.uploader.getAttachmentsFromArtifactId({ - artifact: attachmentsMetadataArtifactId, - }); - - if (error) { - console.error( - `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - return { error }; - } - - if (!attachments || attachments.length === 0) { - console.warn( - `No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - // Remove empty artifact and reset lastProcessed - attachmentsMetadata.artifactIds.shift(); - attachmentsMetadata.lastProcessed = 0; - continue; - } - - console.log( - `Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - - let response; - - if (processors) { - console.log(`Using custom processors for attachments.`); - - const reducer = processors.reducer; - const iterator = processors.iterator; - - const reducedAttachments = runWithUserLogContext(() => - reducer({ - attachments, - adapter: this, - batchSize, - }) - ); - - response = await runWithUserLogContext(async () => { - return await iterator({ - reducedAttachments, - adapter: this, - stream, - }); - }); - } else { - console.log( - `Using attachments streaming pool for attachments streaming.` - ); - - const attachmentsPool = new AttachmentsStreamingPool({ - adapter: this, - attachments, - batchSize, - stream, - }); - - response = await attachmentsPool.streamAll(); - } - - if (response?.delay || response?.error) { - return response; - } - - // On timeout, emit progress and exit to allow continuation. - if (this.isTimeout) { - console.log( - `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.` - ); - await this.emit(ExtractorEventType.AttachmentExtractionProgress); - process.exit(0); - return; - } - - console.log( - `Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - attachmentsMetadata.artifactIds.shift(); - attachmentsMetadata.lastProcessed = 0; - if (attachmentsMetadata.lastProcessedAttachmentsIdsList) { - attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0; - } - } - - return; - }); - } -} diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 34189d04..9b3a7c70 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -6,7 +6,7 @@ import { ErrorRecord } from './common'; import { AxiosResponse } from 'axios'; import { NormalizedAttachment } from '../repo/repo.interfaces'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; import { DonV2, LoaderReport, RateLimited } from './loading'; /** @@ -434,7 +434,7 @@ export type ExternalSystemAttachmentReducerFunction< batchSize, }: { attachments: Batch; - adapter: WorkerAdapter; + adapter: ExtractionAdapter; batchSize?: number; }) => NewBatch; @@ -453,7 +453,7 @@ export type ExternalSystemAttachmentIteratorFunction = stream, }: { reducedAttachments: NewBatch; - adapter: WorkerAdapter; + adapter: ExtractionAdapter; stream: ExternalSystemAttachmentStreamingFunction; }) => Promise; diff --git a/src/types/workers.ts b/src/types/workers.ts index 8be46ef5..529d7835 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -2,7 +2,8 @@ import { Worker } from 'worker_threads'; import type { LogLevel } from '../logger/logger.interfaces'; import { BaseState } from '../state/state'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { LoadingAdapter } from '../multithreading/adapters/loading-adapter'; import { AirSyncEvent, EventType, ExtractorEventType } from './extraction'; @@ -10,6 +11,16 @@ import { LoaderEventType } from './loading'; import { InitialDomainMapping } from './common'; +/** + * WorkerAdapter is the adapter passed to a worker's task/onTimeout callbacks. + * It is the extraction adapter for extraction workers and the loading adapter + * for loading workers; the SDK constructs the concrete type based on the sync + * mode of the event. + */ +export type WorkerAdapter = + | ExtractionAdapter + | LoadingAdapter; + /** * WorkerAdapterInterface is an interface for WorkerAdapter class. * @interface WorkerAdapterInterface From b405c1b27e87aad6a4ca8891478ac93a4b6e289d Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 09:41:52 +0200 Subject: [PATCH 15/30] docs: mark C5 done Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 609a2efd..9dc272e8 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -107,8 +107,42 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi - time-value-resolver.ts: signature unchanged. BREAKING: connectors reading SDK fields via adapter.state break; on-disk state auto-migrates v1→v2 on read. - **C5 — Adapter split (structural only).** `BaseAdapter` + `ExtractionAdapter` + `LoadingAdapter`. - KEEP existing `emit`-based contract working (behavior identical). Author fresh intermediate form - (this exact form exists in NO branch — v2-old-backup (oracle tag)'s split already assumes emit-from-return). + KEEP existing `emit(eventType, data)` contract (behavior identical). + CORRECTION: the oracle's adapter split IS C5-compatible — its `emit()` still has the + `emit(eventType, data)` signature (emit-from-return is genuinely deferred to C6 in the oracle too). + So follow the oracle's adapter design closely (it's a faithful reference), re-authored cleanly. + DESIGN (decided after reading current worker-adapter + oracle adapters): + - `adapters/base-adapter.ts`: `abstract class BaseAdapter` holds event/options/isTimeout/ + hasWorkerEmitted, `protected adapterState: BaseState`, `protected uploader`, + `state` get/set, `sdkState` get, `extractionScope` get, `postState()`, and the TEMPLATE-METHOD `emit()` + skeleton (hasWorkerEmitted guard → beforeEmit hook → state save (non-stateless) → emit() w/ buildEmitPayload + → afterEmit hook → WorkerMessageEmitted). Three abstract hooks: `beforeEmit(type)`, `buildEmitPayload(type)`, + `afterEmit(type)`. + - `adapters/extraction-adapter.ts`: `ExtractionAdapter extends BaseAdapter`. Owns _artifacts, repos, + currentEventDataLength, mappers? NO (mappers is loader). Owns: shouldExtract, initializeRepos, getRepo, + artifacts get/set, uploadAllRepos, processAttachment, destroyHttpStream, streamAttachments. + beforeEmit = ESU-repo-upload (the TODO block) + uploadAllRepos + the AttachmentExtractionDone boundary + update (lastSuccessfulSyncStarted/workers* on sdkState). buildEmitPayload = { artifacts }. afterEmit = + clear artifacts. ctor seeds mappers? no. (mappers stays loader-only per oracle.) + - `adapters/loading-adapter.ts`: `LoadingAdapter extends BaseAdapter`. Owns loaderReports, _processedFiles, + _mappers, reports/processedFiles/mappers getters, loadItemTypes, getLoaderBatches, loadAttachments, + loadItem, loadAttachment. beforeEmit = noop. buildEmitPayload = { reports, processed_files }. afterEmit = noop. + - NOTE the current emit() computes isExtraction/isLoader to decide payload extras; the template split makes + that implicit (each subclass's buildEmitPayload returns its own extras). Behavior-equivalent. + - `worker-adapter.ts`: REPLACE with a thin re-export module OR keep file but it just re-exports. The + `WorkerAdapter` public type becomes a UNION alias in types/workers.ts (see below). `createWorkerAdapter` + factory: keep but split or drop (process-task builds directly). Check consumers. + - `types/workers.ts`: `export type WorkerAdapter = ExtractionAdapter | LoadingAdapter`. + WorkerAdapterInterface stays (adapterState: BaseState). + - `process-task.ts`: dispatch by `event_context.mode === SyncMode.LOADING` → new LoadingAdapter, else + new ExtractionAdapter; pass concrete adapter to task/onTimeout. (process-task SPLIT into two entry points + is C6, NOT here — keep single processTask dispatching.) + - `attachments-streaming-pool.ts` + `.interfaces.ts`: retype `adapter: WorkerAdapter` → `ExtractionAdapter` + (pool is extraction-only; uses sdkState.toDevRev, processAttachment, isTimeout, emit). + - `types/extraction.ts`: imports WorkerAdapter for processor fn types — keep importing the union alias. + - DECIDED: union alias is throwaway, removed in C6 when processTask splits into typed entry points. + - Helpers `worker-adapter.helpers.ts` (getFilesToLoad, addReportToLoaderReport): move to loading-adapter's + dir or keep; used only by loading now. Keep path stable to minimize churn (import from new loading-adapter). - **C6 — Emit-from-return contract.** `task`/`onTimeout` return a `TaskResult` (`{ status: 'success'|'progress'|'delay'|'error', ... }`); the SDK maps status→phase event and emits exactly once; `emit` removed from public surface. `processTask` → `processExtractionTask` + @@ -223,7 +257,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C3 enum cleanup | ☑ done | cc05f41. Deleted deprecated enum members (EventType, ExtractorEventType, LoaderEventType) + event-type-translation.ts/.test; rewired 4 callers (process-task, spawn, control-protocol, worker-adapter) + spawn.helpers cases. Behavior-equivalent. Reviewer-approved. | | C4a state split | ☑ done | b63f3ab. BaseState + ExtractionState + LoadingState, flat shape preserved; state.ts is dispatcher by mode. Reviewer-confirmed behavior-equivalent (loading only loses inert logs). | | C4b state envelope | ☑ done | 30ba1b3. { connectorState, sdkState } envelope + v1->v2 migration shim (normalizeFetchedState). adapter.state→connector-only, new adapter.sdkState; ~28 SDK-field access sites moved. SdkState kept combined (narrowing deferred to C5). Reviewer-approved (migration cases verified). | -| C5 adapter split | ☐ todo | NOTE for C5: narrow SdkState into Base/Extraction/Loading variants here (oracle state.interfaces shows the split). | +| C5 adapter split | ☑ done | a7a877f. BaseAdapter (template emit + hooks) + ExtractionAdapter + LoadingAdapter; WorkerAdapter→union alias; processTask dispatches by mode (still single entry). worker-adapter.ts deleted; helpers→loading-adapter.helpers. Reviewer-approved (emit equivalence verified). SdkState kept combined (narrowing dropped from scope). | | C6 emit-from-return | ☐ todo | | | C7 JSDoc | ☐ todo | Phase 2 | | C8 api report | ☐ todo | Phase 2 | From 0fb6116895569ee7270ea5c1d059fd0886b7b7b2 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:02:04 +0200 Subject: [PATCH 16/30] feat(v2)!: emit from returned TaskResult instead of adapter.emit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connectors now signal how a phase ended by RETURNING a TaskResult from their task/onTimeout callbacks; the SDK maps that status to the phase-appropriate platform event and emits it exactly once. Connectors never call emit directly. - types/workers.ts: add TaskResult ({status: 'success'|'progress'|'delay' (+delaySeconds)|'error' (+error)}) and TaskStatus. ProcessTaskInterface.task now returns Promise; onTimeout is optional and also returns Promise. TaskAdapterInterface is generic over the adapter. The WorkerAdapter union alias is removed. - spawn.helpers.ts: add getEventTypeForResult + EVENT_PHASE_MAP, mapping each incoming EventType + status to the outgoing event. Resumable phases (data/ attachment extraction, data/attachment loading) honour every status; non- resumable phases (ESU, metadata, state deletions) only have done/error, so a progress/delay there is illegal and emits an error with a descriptive msg. - base-adapter.ts: emit() is now protected (SDK-internal). New public emitFromResult(result) — the driver-invoked bridge that picks the event from the mapping and emits with the right payload. State is still saved before emit for non-stateless events. - extraction-adapter.ts: streamAttachments returns a TaskResult (timeout -> progress, pool error/delay -> error/delay, otherwise success) instead of emitting and exiting. The inline external-sync-unit emit override is removed (ESUs flow through the repo; connectors push then return success). - loading-adapter.ts: loadItemTypes and loadAttachments return a TaskResult; their former mid-flight emits (rate-limit -> delay, timeout -> progress, error -> error) become returns. They no longer call process.exit. - process-task.ts: processTask is split into processExtractionTask and processLoadingTask over a shared runWorkerTask driver that runs task (then onTimeout, or a default progress result, on timeout) and emits once from the returned TaskResult. - index.ts: export processExtractionTask + processLoadingTask; remove processTask. emit is no longer part of the public surface. BREAKING CHANGE: connectors must return a TaskResult from task/onTimeout instead of calling adapter.emit(...), and import processExtractionTask / processLoadingTask instead of processTask. adapter.emit is no longer public; the loader and attachment-streaming methods return a TaskResult to be returned from the task. onTimeout is now optional (defaults to a progress handoff). Ref: V2_PROGRESS.md C6 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/index.ts | 5 +- src/multithreading/adapters/base-adapter.ts | 35 ++- .../adapters/extraction-adapter.ts | 71 ++----- .../adapters/loading-adapter.ts | 94 +++------ src/multithreading/process-task.ts | 199 ++++++++++++------ src/multithreading/spawn/spawn.helpers.ts | 170 +++++++++++++++ src/types/workers.ts | 87 +++++--- 7 files changed, 448 insertions(+), 213 deletions(-) diff --git a/src/index.ts b/src/index.ts index ce506898..f079f05e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,10 @@ export type { RetryConfig, RouteConfig, } from './mock-server/mock-server.interfaces'; -export { processTask } from './multithreading/process-task'; +export { + processExtractionTask, + processLoadingTask, +} from './multithreading/process-task'; export { spawn } from './multithreading/spawn/spawn'; export { BaseAdapter } from './multithreading/adapters/base-adapter'; export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter'; diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts index e05c5411..dec92853 100644 --- a/src/multithreading/adapters/base-adapter.ts +++ b/src/multithreading/adapters/base-adapter.ts @@ -14,11 +14,13 @@ import { } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; import { + TaskResult, WorkerAdapterOptions, WorkerMessageEmitted, WorkerMessageSubject, } from '../../types/workers'; import { Uploader } from '../../uploader/uploader'; +import { getEventTypeForResult } from '../spawn/spawn.helpers'; /** * BaseAdapter holds the state and behavior shared by both sync modes and owns @@ -107,13 +109,44 @@ export abstract class BaseAdapter { newEventType: ExtractorEventType | LoaderEventType ): void; + /** + * Maps a {@link TaskResult} returned by a worker's task/onTimeout callback to + * the phase-appropriate platform event and emits it exactly once. + * + * This is the SDK-internal bridge between the return-based connector contract + * and the control protocol; it is invoked by the worker driver, not by + * connectors. Connectors signal outcomes by returning a `TaskResult`, never by + * calling `emit` directly. + * + * @param result - The status the worker reported for the current phase. + */ + async emitFromResult(result: TaskResult): Promise { + const { eventType, illegal } = getEventTypeForResult( + this.event.payload.event_type, + result.status + ); + + const data: EventData = {}; + if (result.status === 'delay') { + data.delay = result.delaySeconds; + } else if (result.status === 'error') { + data.error = result.error; + } else if (illegal) { + data.error = { + message: `Worker returned status '${result.status}' for a non-resumable phase (${this.event.payload.event_type}), which is not allowed. Emitting an error event instead.`, + }; + } + + await this.emit(eventType, data); + } + /** * Emits an event to the platform. * * @param newEventType - The event type to be emitted * @param data - The data to be sent with the event */ - async emit( + protected async emit( newEventType: ExtractorEventType | LoaderEventType, data?: EventData ): Promise { diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index 6aba8cc1..5bbbffa0 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -23,11 +23,10 @@ import { ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, - StreamAttachmentsReturnType, } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; import { BaseState } from '../../state/state'; -import { WorkerAdapterOptions } from '../../types/workers'; +import { TaskResult, WorkerAdapterOptions } from '../../types/workers'; import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; import { BaseAdapter } from './base-adapter'; @@ -192,50 +191,6 @@ export class ExtractionAdapter< this.artifacts = []; } - /** - * Emits an extraction event. For ExternalSyncUnitExtractionDone, inline - * external sync units are uploaded via a Repo first (then stripped from the - * payload to avoid SQS size limits); `beforeEmit` uploads that repo with the - * rest before the event is sent. - * TODO: Remove the external sync unit handling in v2.0.0 - * - * @param newEventType - The event type to be emitted - * @param data - The data to be sent with the event - */ - override async emit( - newEventType: ExtractorEventType | LoaderEventType, - data?: EventData - ): Promise { - if ( - newEventType === ExtractorEventType.ExternalSyncUnitExtractionDone && - data?.external_sync_units && - data.external_sync_units.length > 0 - ) { - console.log( - `Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.` - ); - - this.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: 25000, - skipConfirmation: true, - }, - }, - ]); - - await this.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push( - data.external_sync_units - ); - - // Remove inline external_sync_units from data to avoid SQS size issues - delete data.external_sync_units; - } - - return super.emit(newEventType, data); - } - async uploadAllRepos(): Promise { for (const repo of this.repos) { const error = await repo.upload(); @@ -397,7 +352,7 @@ export class ExtractionAdapter< NewBatch >; batchSize?: number; - }): Promise { + }): Promise { return runWithSdkLogContext(async () => { if (batchSize <= 0) { console.warn( @@ -425,7 +380,7 @@ export class ExtractionAdapter< // If there are no attachments metadata artifact IDs in state, finish here if (!attachmentsMetadata?.artifactIds?.length) { console.log(`No attachments metadata artifact IDs found in state.`); - return; + return { status: 'success' }; } else { console.log( `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` @@ -450,7 +405,7 @@ export class ExtractionAdapter< console.error( `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` ); - return { error }; + return { status: 'error', error }; } if (!attachments || attachments.length === 0) { @@ -505,18 +460,20 @@ export class ExtractionAdapter< response = await attachmentsPool.streamAll(); } - if (response?.delay || response?.error) { - return response; + if (response?.error) { + return { status: 'error', error: response.error }; + } + + if (response?.delay) { + return { status: 'delay', delaySeconds: response.delay }; } - // On timeout, emit progress and exit to allow continuation. + // On timeout, return progress to allow continuation in a fresh invocation. if (this.isTimeout) { console.log( - `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.` + `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.` ); - await this.emit(ExtractorEventType.AttachmentExtractionProgress); - process.exit(0); - return; + return { status: 'progress' }; } console.log( @@ -529,7 +486,7 @@ export class ExtractionAdapter< } } - return; + return { status: 'success' }; }); } } diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts index 66a3fd20..49fdd2bf 100644 --- a/src/multithreading/adapters/loading-adapter.ts +++ b/src/multithreading/adapters/loading-adapter.ts @@ -25,10 +25,9 @@ import { LoaderEventType, LoaderReport, LoadItemResponse, - LoadItemTypesResponse, StatsFileObject, } from '../../types/loading'; -import { WorkerAdapterOptions } from '../../types/workers'; +import { TaskResult, WorkerAdapterOptions } from '../../types/workers'; import { BaseAdapter } from './base-adapter'; import { @@ -99,7 +98,7 @@ export class LoadingAdapter< async loadItemTypes({ itemTypesToLoad, - }: ItemTypesToLoadParams): Promise { + }: ItemTypesToLoadParams): Promise { return runWithSdkLogContext(async () => { if (this.event.payload.event_type === EventType.StartLoadingData) { const itemTypes = itemTypesToLoad.map( @@ -108,10 +107,7 @@ export class LoadingAdapter< if (!itemTypes.length) { console.warn('No item types to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; + return { status: 'success' }; } const filesToLoad = await this.getLoaderBatches({ @@ -127,10 +123,7 @@ export class LoadingAdapter< !this.sdkState.fromDevRev.filesToLoad.length ) { console.warn('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; + return { status: 'success' }; } console.log( @@ -139,8 +132,7 @@ export class LoadingAdapter< ); try { - outerloop: for (const fileToLoad of this.sdkState.fromDevRev - .filesToLoad) { + for (const fileToLoad of this.sdkState.fromDevRev.filesToLoad) { const itemTypeToLoad = itemTypesToLoad.find( (itemTypeToLoad: ItemTypeToLoad) => itemTypeToLoad.itemType === fileToLoad.itemType @@ -151,13 +143,12 @@ export class LoadingAdapter< `Item type to load not found for item type: ${fileToLoad.itemType}.` ); - await this.emit(LoaderEventType.DataLoadingError, { + return { + status: 'error', error: { message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, }, - }); - - break; + }; } if (!fileToLoad.completed) { @@ -171,12 +162,12 @@ export class LoadingAdapter< console.error( `Transformer file not found for artifact ID: ${fileToLoad.id}.` ); - await this.emit(LoaderEventType.DataLoadingError, { + return { + status: 'error', error: { message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, }, - }); - break outerloop; + }; } const transformerFile = response as ExternalSystemItem[]; @@ -184,10 +175,9 @@ export class LoadingAdapter< for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { if (this.isTimeout) { console.log( - 'Timeout detected during data loading. Emitting progress to allow continuation.' + 'Timeout detected during data loading. Returning progress to allow continuation.' ); - await this.emit(LoaderEventType.DataLoadingProgress); - process.exit(0); + return { status: 'progress' }; } const { report, rateLimit } = await this.loadItem({ @@ -196,13 +186,7 @@ export class LoadingAdapter< }); if (rateLimit?.delay) { - await this.emit(LoaderEventType.DataLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; + return { status: 'delay', delaySeconds: rateLimit.delay }; } if (report) { @@ -220,18 +204,15 @@ export class LoadingAdapter< } } catch (error) { console.error('Error during data loading.', serializeError(error)); - await this.emit(LoaderEventType.DataLoadingError, { + return { + status: 'error', error: { message: `Error during data loading. ${serializeError(error)}`, }, - }); - process.exit(1); + }; } - return { - reports: this.reports, - processed_files: this.processedFiles, - }; + return { status: 'success' }; }); } @@ -271,7 +252,7 @@ export class LoadingAdapter< create, }: { create: ExternalSystemLoadingFunction; - }): Promise { + }): Promise { return runWithSdkLogContext(async () => { if (this.event.payload.event_type === EventType.StartLoadingAttachments) { this.sdkState.fromDevRev = { @@ -286,16 +267,13 @@ export class LoadingAdapter< this.sdkState.fromDevRev?.filesToLoad.length === 0 ) { console.log('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; + return { status: 'success' }; } const filesToLoad = this.sdkState.fromDevRev?.filesToLoad; try { - outerloop: for (const fileToLoad of filesToLoad) { + for (const fileToLoad of filesToLoad) { if (!fileToLoad.completed) { const { response, error: transformerFileError } = await this.uploader.getJsonObjectByArtifactId({ @@ -309,21 +287,20 @@ export class LoadingAdapter< console.error( `Transformer file not found for artifact ID: ${fileToLoad.id}.` ); - await this.emit(LoaderEventType.AttachmentLoadingError, { + return { + status: 'error', error: { message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, }, - }); - break outerloop; + }; } for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { if (this.isTimeout) { console.log( - 'Timeout detected during attachment loading. Emitting progress to allow continuation.' + 'Timeout detected during attachment loading. Returning progress to allow continuation.' ); - await this.emit(LoaderEventType.AttachmentLoadingProgress); - process.exit(0); + return { status: 'progress' }; } const { report, rateLimit } = await this.loadAttachment({ @@ -332,13 +309,7 @@ export class LoadingAdapter< }); if (rateLimit?.delay) { - await this.emit(LoaderEventType.AttachmentLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; + return { status: 'delay', delaySeconds: rateLimit.delay }; } if (report) { @@ -359,20 +330,17 @@ export class LoadingAdapter< 'Error during attachment loading.', serializeError(error) ); - await this.emit(LoaderEventType.AttachmentLoadingError, { + return { + status: 'error', error: { message: `Error during attachment loading. ${serializeError( error )}`, }, - }); - process.exit(1); + }; } - return { - reports: this.reports, - processed_files: this.processedFiles, - }; + return { status: 'success' }; }); } diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 619bf854..a43debbb 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -1,85 +1,160 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads'; + import { Logger, serializeError } from '../logger/logger'; import { runWithSdkLogContext, runWithUserLogContext, } from '../logger/logger.context'; -import { createAdapterState } from '../state/state'; -import { SyncMode } from '../types/common'; +import { createExtractionState } from '../state/extraction-state'; +import { createLoadingState } from '../state/loading-state'; import { ProcessTaskInterface, - WorkerAdapter, + TaskResult, WorkerEvent, WorkerMessageSubject, } from '../types/workers'; + +import { BaseAdapter } from './adapters/base-adapter'; import { ExtractionAdapter } from './adapters/extraction-adapter'; import { LoadingAdapter } from './adapters/loading-adapter'; -export function processTask({ +/** + * Shared worker-thread driver. Builds the logger context, runs the task and + * (on timeout) the onTimeout callback against the provided adapter, maps the + * returned {@link TaskResult} to a platform event and emits it exactly once, + * and wires the error/exit plumbing. + * + * The adapter is constructed by the caller so each entry point can build its + * own typed adapter. + * + * If `onTimeout` is omitted, the SDK emits a phase-appropriate default on + * timeout: `progress` (resumable phases) or `error` (non-resumable phases) is + * handled by the status->event mapping when we emit a `progress` result. + */ +async function runWorkerTask>( + buildAdapter: () => Promise, + { task, onTimeout }: ProcessTaskInterface +): Promise { + await runWithSdkLogContext(async () => { + try { + const adapter = await buildAdapter(); + + parentPort?.on(WorkerEvent.WorkerMessage, (message) => { + if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { + return; + } + console.log('Timeout received. Waiting for the task to finish.'); + adapter.isTimeout = true; + }); + + let result: TaskResult = await runWithUserLogContext(async () => + task({ adapter }) + ); + + // On timeout, hand off to onTimeout (or default to a progress result). + if (adapter.isTimeout && !adapter.hasWorkerEmitted) { + result = onTimeout + ? await runWithUserLogContext(async () => onTimeout({ adapter })) + : { status: 'progress' }; + } + + if (!adapter.hasWorkerEmitted) { + await adapter.emitFromResult(result); + } + + process.exit(0); + } catch (error) { + runWithUserLogContext(() => { + const errorMessage = `Error while processing task. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + }); + } + }); +} + +/** + * Entry point for an extraction worker. Builds an {@link ExtractionAdapter} and + * runs the provided task against it. + * + * @public + */ +export function processExtractionTask({ task, onTimeout, -}: ProcessTaskInterface) { +}: ProcessTaskInterface>) { if (isMainThread) { return; } - void (async () => { - await runWithSdkLogContext(async () => { - try { - const event = workerData.event; - - const initialState = workerData.initialState as ConnectorState; - const initialDomainMapping = workerData.initialDomainMapping; - const options = workerData.options; - // eslint-disable-next-line no-global-assign - console = new Logger({ event, options }); - - const adapterState = await createAdapterState({ - event, - initialState, - initialDomainMapping, - options, - }); + void runWorkerTask>( + async () => { + const event = workerData.event; + const initialState = workerData.initialState as ConnectorState; + const initialDomainMapping = workerData.initialDomainMapping; + const options = workerData.options; + // eslint-disable-next-line no-global-assign + console = new Logger({ event, options }); - const adapter: WorkerAdapter = - event.payload.event_context.mode === SyncMode.LOADING - ? new LoadingAdapter({ - event, - adapterState, - options, - }) - : new ExtractionAdapter({ - event, - adapterState, - options, - }); - - parentPort?.on(WorkerEvent.WorkerMessage, (message) => { - if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { - return; - } - console.log('Timeout received. Waiting for the task to finish.'); - adapter.isTimeout = true; - }); + const adapterState = await createExtractionState({ + event, + initialState, + initialDomainMapping, + options, + }); - await runWithUserLogContext(async () => task({ adapter })); - if (adapter.isTimeout && !adapter.hasWorkerEmitted) { - await runWithUserLogContext(async () => onTimeout({ adapter })); - } - process.exit(0); - } catch (error) { - runWithUserLogContext(() => { - const errorMessage = `Error while processing task. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - }); - } - }); - })(); + return new ExtractionAdapter({ + event, + adapterState, + options, + }); + }, + { task, onTimeout } + ); +} + +/** + * Entry point for a loading worker. Builds a {@link LoadingAdapter} and runs the + * provided task against it. + * + * @public + */ +export function processLoadingTask({ + task, + onTimeout, +}: ProcessTaskInterface>) { + if (isMainThread) { + return; + } + + void runWorkerTask>( + async () => { + const event = workerData.event; + const initialState = workerData.initialState as ConnectorState; + const initialDomainMapping = workerData.initialDomainMapping; + const options = workerData.options; + // eslint-disable-next-line no-global-assign + console = new Logger({ event, options }); + + const adapterState = await createLoadingState({ + event, + initialState, + initialDomainMapping, + options, + }); + + return new LoadingAdapter({ + event, + adapterState, + options, + }); + }, + { task, onTimeout } + ); } diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index 3db03dfb..1f52764c 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -1,5 +1,175 @@ import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; +import { TaskStatus } from '../../types/workers'; + +/** + * Resolves the outgoing event type the SDK should emit for a given incoming + * event type and the {@link TaskResult} status a worker returned. + * + * The mapping follows the per-phase contract documented on `TaskResult`: + * resumable phases (data/attachment extraction, data/attachment loading) honor + * every status (success -> *_DONE, progress -> *_PROGRESS, delay -> *_DELAYED, + * error -> *_ERROR); non-resumable phases (external sync units, metadata, state + * deletion) only have a done and an error event, so a `progress`/`delay` status + * there is illegal and is mapped to the phase's error event. + * + * @param eventType - The incoming event type that started this worker. + * @param status - The status the worker's task/onTimeout returned. + * @returns The outgoing extractor/loader event type to emit, plus whether the + * status was illegal for the phase (so the caller can attach a descriptive + * error message). + */ +export function getEventTypeForResult( + eventType: EventType, + status: TaskStatus +): { + eventType: ExtractorEventType | LoaderEventType; + illegal: boolean; +} { + const phase = EVENT_PHASE_MAP[eventType]; + + if (!phase) { + console.error( + 'Event type not recognized in getEventTypeForResult function: ' + + eventType + ); + return { eventType: LoaderEventType.UnknownEventType, illegal: true }; + } + + // Non-resumable phases only define done/error events. + if (!phase.resumable) { + if (status === 'success') { + return { eventType: phase.done, illegal: false }; + } + // progress/delay are illegal here; collapse them (and error) to the error event. + return { eventType: phase.error, illegal: status !== 'error' }; + } + + switch (status) { + case 'success': + return { eventType: phase.done, illegal: false }; + case 'progress': + return { eventType: phase.progress!, illegal: false }; + case 'delay': + return { eventType: phase.delayed!, illegal: false }; + case 'error': + return { eventType: phase.error, illegal: false }; + } +} + +/** + * Per-phase outgoing event types, keyed by the incoming {@link EventType}. + * `resumable` phases define progress/delayed events; non-resumable ones do not. + */ +const EVENT_PHASE_MAP: Partial< + Record< + EventType, + { + resumable: boolean; + done: ExtractorEventType | LoaderEventType; + error: ExtractorEventType | LoaderEventType; + progress?: ExtractorEventType | LoaderEventType; + delayed?: ExtractorEventType | LoaderEventType; + } + > +> = { + // External sync units (non-resumable) + [EventType.StartExtractingExternalSyncUnits]: { + resumable: false, + done: ExtractorEventType.ExternalSyncUnitExtractionDone, + error: ExtractorEventType.ExternalSyncUnitExtractionError, + }, + // Metadata (non-resumable) + [EventType.StartExtractingMetadata]: { + resumable: false, + done: ExtractorEventType.MetadataExtractionDone, + error: ExtractorEventType.MetadataExtractionError, + }, + // Data extraction (resumable) + [EventType.StartExtractingData]: { + resumable: true, + done: ExtractorEventType.DataExtractionDone, + error: ExtractorEventType.DataExtractionError, + progress: ExtractorEventType.DataExtractionProgress, + delayed: ExtractorEventType.DataExtractionDelayed, + }, + [EventType.ContinueExtractingData]: { + resumable: true, + done: ExtractorEventType.DataExtractionDone, + error: ExtractorEventType.DataExtractionError, + progress: ExtractorEventType.DataExtractionProgress, + delayed: ExtractorEventType.DataExtractionDelayed, + }, + // Extractor state deletion (non-resumable) + [EventType.StartDeletingExtractorState]: { + resumable: false, + done: ExtractorEventType.ExtractorStateDeletionDone, + error: ExtractorEventType.ExtractorStateDeletionError, + }, + // Attachment extraction (resumable) + [EventType.StartExtractingAttachments]: { + resumable: true, + done: ExtractorEventType.AttachmentExtractionDone, + error: ExtractorEventType.AttachmentExtractionError, + progress: ExtractorEventType.AttachmentExtractionProgress, + delayed: ExtractorEventType.AttachmentExtractionDelayed, + }, + [EventType.ContinueExtractingAttachments]: { + resumable: true, + done: ExtractorEventType.AttachmentExtractionDone, + error: ExtractorEventType.AttachmentExtractionError, + progress: ExtractorEventType.AttachmentExtractionProgress, + delayed: ExtractorEventType.AttachmentExtractionDelayed, + }, + // Extractor attachments state deletion (non-resumable) + [EventType.StartDeletingExtractorAttachmentsState]: { + resumable: false, + done: ExtractorEventType.ExtractorAttachmentsStateDeletionDone, + error: ExtractorEventType.ExtractorAttachmentsStateDeletionError, + }, + // Data loading (resumable) + [EventType.StartLoadingData]: { + resumable: true, + done: LoaderEventType.DataLoadingDone, + error: LoaderEventType.DataLoadingError, + progress: LoaderEventType.DataLoadingProgress, + delayed: LoaderEventType.DataLoadingDelayed, + }, + [EventType.ContinueLoadingData]: { + resumable: true, + done: LoaderEventType.DataLoadingDone, + error: LoaderEventType.DataLoadingError, + progress: LoaderEventType.DataLoadingProgress, + delayed: LoaderEventType.DataLoadingDelayed, + }, + // Attachment loading (resumable) + [EventType.StartLoadingAttachments]: { + resumable: true, + done: LoaderEventType.AttachmentLoadingDone, + error: LoaderEventType.AttachmentLoadingError, + progress: LoaderEventType.AttachmentLoadingProgress, + delayed: LoaderEventType.AttachmentLoadingDelayed, + }, + [EventType.ContinueLoadingAttachments]: { + resumable: true, + done: LoaderEventType.AttachmentLoadingDone, + error: LoaderEventType.AttachmentLoadingError, + progress: LoaderEventType.AttachmentLoadingProgress, + delayed: LoaderEventType.AttachmentLoadingDelayed, + }, + // Loader state deletion (non-resumable) + [EventType.StartDeletingLoaderState]: { + resumable: false, + done: LoaderEventType.LoaderStateDeletionDone, + error: LoaderEventType.LoaderStateDeletionError, + }, + // Loader attachment state deletion (non-resumable) + [EventType.StartDeletingLoaderAttachmentState]: { + resumable: false, + done: LoaderEventType.LoaderAttachmentStateDeletionDone, + error: LoaderEventType.LoaderAttachmentStateDeletionError, + }, +}; /** * Gets the event type for the timeout error. diff --git a/src/types/workers.ts b/src/types/workers.ts index 529d7835..dc617d11 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -2,24 +2,12 @@ import { Worker } from 'worker_threads'; import type { LogLevel } from '../logger/logger.interfaces'; import { BaseState } from '../state/state'; -import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; -import { LoadingAdapter } from '../multithreading/adapters/loading-adapter'; import { AirSyncEvent, EventType, ExtractorEventType } from './extraction'; import { LoaderEventType } from './loading'; -import { InitialDomainMapping } from './common'; - -/** - * WorkerAdapter is the adapter passed to a worker's task/onTimeout callbacks. - * It is the extraction adapter for extraction workers and the loading adapter - * for loading workers; the SDK constructs the concrete type based on the sync - * mode of the event. - */ -export type WorkerAdapter = - | ExtractionAdapter - | LoadingAdapter; +import { ErrorRecord, InitialDomainMapping } from './common'; /** * WorkerAdapterInterface is an interface for WorkerAdapter class. @@ -99,25 +87,66 @@ export interface SpawnFactoryInterface { } /** - * TaskAdapterInterface is an interface for TaskAdapter class. - * @interface TaskAdapterInterface - * @constructor - * @param {WorkerAdapter} adapter - The adapter object - */ -export interface TaskAdapterInterface { - adapter: WorkerAdapter; + * TaskResult is the value a worker's `task` (and optional `onTimeout`) callback + * returns to tell the SDK how the current phase ended. The SDK — not the + * connector — maps this status to the phase-appropriate platform event and + * emits it exactly once. Connectors never call `emit` directly. + * + * One lambda invocation = one worker process = exactly one emitted event = + * terminal. Any continuation (CONTINUE_*, next phase, retry after delay) + * happens in a fresh invocation driven by the platform. + * + * The discriminant is a bare string literal, so connectors write e.g. + * `return { status: 'delay', delaySeconds: 60 }` with no import. + * + * Status -> emitted event, per phase: + * + * | status | resumable phases | non-resumable (ESU / metadata) | + * |------------|------------------|--------------------------------| + * | 'success' | *_DONE | *_DONE | + * | 'progress' | *_PROGRESS | *_ERROR (illegal; descriptive) | + * | 'delay' | *_DELAYED | *_ERROR (illegal; descriptive) | + * | 'error' | *_ERROR | *_ERROR | + * + * Resumable phases: data/attachment extraction, data/attachment loading. + * Non-resumable phases: external sync units, metadata. + */ +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } + | { status: 'error'; error: ErrorRecord }; + +/** + * Discriminant string of a {@link TaskResult}. + */ +export type TaskStatus = TaskResult['status']; + +/** + * TaskAdapterInterface is the parameter shape passed to a worker's task and + * onTimeout callbacks. + * @param adapter - The mode-specific adapter for the worker. + */ +export interface TaskAdapterInterface { + adapter: Adapter; } /** - * ProcessTaskInterface is an interface for ProcessTask class. - * @interface ProcessTaskInterface - * @constructor - * @param {function} task - The task to be executed, returns exit code - * @param {function} onTimeout - The task to be executed on timeout, returns exit code - */ -export interface ProcessTaskInterface { - task: (params: TaskAdapterInterface) => Promise; - onTimeout: (params: TaskAdapterInterface) => Promise; + * ProcessTaskInterface is the parameter shape for the process-task entry points. + * + * Both callbacks return a {@link TaskResult}; the SDK — not the connector — + * maps that status to the phase-appropriate platform event and emits it exactly + * once. Connectors never call `emit` directly. + * + * `onTimeout` is optional: if omitted, the SDK emits a phase-appropriate default + * on timeout (progress for resumable phases, error for ESU/metadata). + * + * @param task - Runs the phase; returns how it ended. + * @param onTimeout - Runs only on timeout; returns how to hand off. + */ +export interface ProcessTaskInterface { + task: (params: TaskAdapterInterface) => Promise; + onTimeout?: (params: TaskAdapterInterface) => Promise; } /** From d9ec86b0affcac473cf62748d0f6ad524e0d0b52 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:02:15 +0200 Subject: [PATCH 17/30] =?UTF-8?q?docs:=20mark=20C6=20done=20=E2=80=94=20Ph?= =?UTF-8?q?ase=201=20complete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- V2_PROGRESS.md | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 9dc272e8..6ebed941 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -143,11 +143,42 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi - DECIDED: union alias is throwaway, removed in C6 when processTask splits into typed entry points. - Helpers `worker-adapter.helpers.ts` (getFilesToLoad, addReportToLoaderReport): move to loading-adapter's dir or keep; used only by loading now. Keep path stable to minimize churn (import from new loading-adapter). -- **C6 — Emit-from-return contract.** `task`/`onTimeout` return a `TaskResult` - (`{ status: 'success'|'progress'|'delay'|'error', ... }`); the SDK maps status→phase event and emits - exactly once; `emit` removed from public surface. `processTask` → `processExtractionTask` + - `processLoadingTask`. Reference: v2-old-backup (oracle tag) `process-task.ts`, `base-adapter.ts` (mapping keys off - event_type/phase, NOT off state shape — so C4b and C6 are independent). +- **C6 — Emit-from-return contract.** NET-NEW DESIGN: emit-from-return was NEVER implemented in any + branch (oracle process-task.ts is still emit-based). Only spec is the TaskResult surface recovered from + `src/tests/backwards-compatibility/temp/airsync-sdk.api.md` (untracked leftover). DESIGN: + - types: add to types/workers.ts: + `export type TaskResult = {status:'success'} | {status:'progress'} | {status:'delay';delaySeconds:number} + | {status:'error';error:ErrorRecord};` and `export type TaskStatus = TaskResult['status'];` + Change ProcessTaskInterface: `task: (p)=>Promise`, `onTimeout?: (p)=>Promise` (optional). + - Status→event mapping (SDK-owned, per phase). Resumable phases (data/attachment extraction, data/attachment + loading): success→*_DONE, progress→*_PROGRESS, delay→*_DELAYED, error→*_ERROR. Non-resumable (ESU, metadata): + success→*_DONE, error→*_ERROR, progress/delay→*_ERROR (illegal, descriptive msg). Map keyed off + event.payload.event_type (which incoming EventType → which Extractor/Loader event per status). + - base-adapter: emit() becomes SDK-INTERNAL (drop from public surface / index). Add a driver-facing method + e.g. `async emitFromResult(result: TaskResult)` that computes the event type from event_type+status via the + mapping and calls the internal emit with the right payload (delaySeconds for delay, error for error). + - LOADER/STREAMING methods RETURN TaskResult (Rado decided): loadItemTypes, loadAttachments, streamAttachments + stop calling emit mid-flight; instead they RETURN a TaskResult (success/progress/delay/error). The reports/ + processed_files still flow via buildEmitPayload from adapter state (already wired in C5), so the methods just + return status. Their old in-method emit() calls (rate-limit→delay, timeout→progress, error→error) become + `return {status:...}`. They no longer call process.exit; the driver handles emit+exit. + - process-task.ts: split into `processExtractionTask` + `processLoadingTask` (drop single processTask + the + WorkerAdapter union alias). Shared `runWorkerTask(buildAdapter, {task,onTimeout})` driver (oracle has the + skeleton): run task→get TaskResult; if isTimeout && !hasWorkerEmitted run onTimeout (or SDK default if + onTimeout omitted: progress for resumable, error for ESU/metadata)→get TaskResult; then + adapter.emitFromResult(result); exit. Build the typed adapter via createExtractionState/createLoadingState. + - index.ts: export TaskResult, TaskStatus, processExtractionTask, processLoadingTask. REMOVE processTask. + - DECIDED (Rado): emit is SDK-INTERNAL — NOT on the public ExtractionAdapter/LoadingAdapter surface. + The spec confirms this (its adapter surfaces show no emit()). Implementation: the template-method skeleton + + beforeEmit/buildEmitPayload/afterEmit hooks become `protected` (already are for hooks); the actual emit + entry the DRIVER calls is `emitFromResult(result: TaskResult)` — make it a method the driver can call but + not documented as connector surface. Connectors calling adapter.emit() get a compile error (good). + Keep the old `emit(eventType,data)` as a protected/internal helper that emitFromResult delegates to (the + loader/streaming methods no longer call it — they return TaskResult). + - test connectors (src/tests/**): these CALL adapter.emit today → they'd break. They are test files (deferred + to C10), and build excludes them, so C6 build stays green. Migration of test workers to return-style = C10. + - Reference: oracle process-task.ts (the runWorkerTask skeleton + typed entry points) — but oracle is still + emit-based, so the TaskResult return + emitFromResult mapping is authored fresh per the spec above. ### Phase 2 — closing / interactive (batched, done at the end) - **C7 — JSDoc pass.** Bar = `src/mappers/mappers.ts` style (class block: what+when; method block: @@ -258,7 +289,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C4a state split | ☑ done | b63f3ab. BaseState + ExtractionState + LoadingState, flat shape preserved; state.ts is dispatcher by mode. Reviewer-confirmed behavior-equivalent (loading only loses inert logs). | | C4b state envelope | ☑ done | 30ba1b3. { connectorState, sdkState } envelope + v1->v2 migration shim (normalizeFetchedState). adapter.state→connector-only, new adapter.sdkState; ~28 SDK-field access sites moved. SdkState kept combined (narrowing deferred to C5). Reviewer-approved (migration cases verified). | | C5 adapter split | ☑ done | a7a877f. BaseAdapter (template emit + hooks) + ExtractionAdapter + LoadingAdapter; WorkerAdapter→union alias; processTask dispatches by mode (still single entry). worker-adapter.ts deleted; helpers→loading-adapter.helpers. Reviewer-approved (emit equivalence verified). SdkState kept combined (narrowing dropped from scope). | -| C6 emit-from-return | ☐ todo | | +| C6 emit-from-return | ☑ done | 0fb6116. task/onTimeout return TaskResult; SDK maps status→event via getEventTypeForResult and emits once (emitFromResult); emit now protected/internal; processTask→processExtractionTask+processLoadingTask; loader/stream methods return TaskResult. Reviewer-approved (mapping+state-save+no-double-emit verified). NET-NEW design (no oracle). | | C7 JSDoc | ☐ todo | Phase 2 | | C8 api report | ☐ todo | Phase 2 | | C9 exposure audit | ☐ todo | Phase 2, interactive | From d05434be0baa2cb373ba114ad3da769561aa6854 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:18:11 +0200 Subject: [PATCH 18/30] docs(v2): JSDoc pass over public surface and v2 internals Documentation-only pass (C7). Brings the v2-new structural code and the under-documented older modules up to the `src/mappers/mappers.ts` style bar (one-line what + "Used to/for ..." usage line + typed @param/@returns). Covered: - adapters: BaseAdapter template-method emit + emitFromResult, ExtractionAdapter (repos/artifacts/attachment streaming), LoadingAdapter (mappers/reports/loading). - state: BaseState lifecycle, the v1->v2 normalizeFetchedState migration shim, ExtractionState window resolution, the mode-based createAdapterState dispatcher, and state.interfaces (SdkState, envelope, V1_SDK_STATE_KEYS). - multithreading: processExtractionTask/processLoadingTask + runWorkerTask driver, spawn/Spawn supervision, and the getEventTypeForResult / EVENT_PHASE_MAP status->event mapping. - repo, uploader (+helpers/interfaces), attachments-streaming-pool. - common: control-protocol emit, install-initial-domain-mapping, errors; types/loading (previously zero JSDoc) and the types barrel. No executable code, type signatures, imports, or string literals changed (verified: every changed line is a comment). Build green, lint clean. --- .../attachments-streaming-pool.interfaces.ts | 10 ++ .../attachments-streaming-pool.ts | 41 ++++- src/common/control-protocol.ts | 20 +++ src/common/errors.ts | 9 + src/common/install-initial-domain-mapping.ts | 13 ++ src/multithreading/adapters/base-adapter.ts | 41 +++-- .../adapters/extraction-adapter.ts | 117 +++++++++++-- .../adapters/loading-adapter.helpers.ts | 26 ++- .../adapters/loading-adapter.ts | 97 ++++++++++- src/multithreading/create-worker.ts | 10 ++ src/multithreading/process-task.ts | 24 +++ src/multithreading/spawn/spawn.helpers.ts | 10 +- src/multithreading/spawn/spawn.ts | 35 +++- src/repo/repo.interfaces.ts | 40 ++++- src/repo/repo.ts | 26 +++ src/state/base-state.ts | 75 ++++++--- src/state/extraction-state.ts | 40 +++-- src/state/loading-state.ts | 19 ++- src/state/state.interfaces.ts | 45 ++++- src/state/state.ts | 10 +- src/types/index.ts | 8 + src/types/loading.ts | 154 ++++++++++++++++++ src/uploader/uploader.helpers.ts | 47 ++++-- src/uploader/uploader.interfaces.ts | 21 +++ src/uploader/uploader.ts | 110 ++++++++----- 25 files changed, 902 insertions(+), 146 deletions(-) diff --git a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts index 9a22ea08..c4127582 100644 --- a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts +++ b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts @@ -4,9 +4,19 @@ import { } from '../types'; import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +/** + * Construction parameters used to create an AttachmentsStreamingPool. + * + * Used to supply the driving extraction adapter, the attachments to stream, the concurrency limit, and + * the connector-provided streaming function. + */ export interface AttachmentsStreamingPoolParams { + /** The ExtractionAdapter that owns sync state, timeout detection, and the processAttachment call. */ adapter: ExtractionAdapter; + /** The normalized attachments to stream to DevRev. */ attachments: NormalizedAttachment[]; + /** Optional maximum number of attachments to stream concurrently (defaults to 10 in the pool). */ batchSize?: number; + /** Connector-provided function that downloads a single attachment from the external system. */ stream: ExternalSystemAttachmentStreamingFunction; } diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index 58281e16..d7cf4a05 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -8,6 +8,12 @@ import { } from '../types'; import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces'; +/** + * Concurrency-bounded pool that streams a batch of attachments from the external system to DevRev. + * + * Used during attachment extraction to download up to batchSize attachments in parallel while honoring + * timeouts, rate-limit delays, and per-attachment errors, and to track processed attachments for resumption. + */ export class AttachmentsStreamingPool { private adapter: ExtractionAdapter; private attachments: NormalizedAttachment[]; @@ -31,6 +37,14 @@ export class AttachmentsStreamingPool { this.stream = stream; } + /** + * Increments the processed counter and periodically logs progress. + * + * Used after each attachment to report progress every PROGRESS_REPORT_INTERVAL items and briefly + * yield the event loop. + * + * @returns Promise that resolves once progress has been recorded (and any brief sleep elapsed). + */ private async updateProgress() { this.totalProcessedCount++; if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) { @@ -41,10 +55,13 @@ export class AttachmentsStreamingPool { } /** - * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format. + * Migrates processed-attachment state from the legacy string[] format to ProcessedAttachment[]. * - * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[]) - * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid + * Used when resuming streaming so older saved state (a list of ids) is upgraded to the structured + * { id, parent_id } form before it is consulted for de-duplication. + * + * @param attachments - The persisted list to migrate, either a string[] of ids or a ProcessedAttachment[]. + * @returns Migrated array of ProcessedAttachment objects, or an empty array if the input is invalid. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any private migrateProcessedAttachments(attachments: any): ProcessedAttachment[] { @@ -69,6 +86,15 @@ export class AttachmentsStreamingPool { return []; } + /** + * Streams every attachment in the pool, running up to batchSize streams concurrently. + * + * Used as the pool's entry point: it initializes/migrates the processed-attachments state, starts the + * initial set of worker loops, and waits for them to drain the queue or stop early on a delay. + * + * @returns Promise resolving to a ProcessAttachmentReturnType: a delay if rate-limited, an error if + * state is uninitialized, or an empty object once all attachments are processed. + */ async streamAll(): Promise { console.log( `Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.` @@ -115,6 +141,15 @@ export class AttachmentsStreamingPool { return {}; } + /** + * Runs a single worker loop that pulls and streams attachments until the queue is drained. + * + * Used as one of the concurrent workers started by streamAll: it skips already-processed attachments, + * stops on timeout or a rate-limit delay, records successes, and logs/skips per-attachment errors. + * + * @returns Promise that resolves when this worker stops, either because the queue is empty or a + * timeout/delay was detected. + */ async startPoolStreaming() { // Process attachments until the attachments array is empty while (this.attachments.length > 0) { diff --git a/src/common/control-protocol.ts b/src/common/control-protocol.ts index 0d13be13..8470401d 100644 --- a/src/common/control-protocol.ts +++ b/src/common/control-protocol.ts @@ -10,12 +10,32 @@ import { import { LoaderEventType } from '../types/loading'; import { LIBRARY_VERSION } from './constants'; +/** + * Parameters for emitting a worker control message back to the platform. + * + * Used by {@link emit} to construct and post the outgoing extractor/loader event. + */ export interface EmitInterface { + /** The incoming AirSync event currently being processed; supplies the callback URL, event context, and auth secrets. */ event: AirSyncEvent; + /** The outgoing event type to report. In v2 this value is used directly (no event-type translation). */ eventType: ExtractorEventType | LoaderEventType; + /** Optional payload describing progress, results, or error details to attach to the event. */ data?: EventData; } +/** + * Emits a worker control message to the parent/platform via the event callback URL. + * + * Used to report extraction/loading progress, completion, delays, or errors back to AirSync. + * Wraps the given event type and data into an ExtractorEvent/LoaderEvent envelope (stamped with + * the library version) and POSTs it to the callback URL with the service account authorization. + * + * @param event - The incoming AirSyncEvent providing callback URL, event context, and auth secrets. + * @param eventType - The outgoing ExtractorEventType or LoaderEventType, used directly as the event_type. + * @param data - Optional EventData payload to include in the emitted event. + * @returns Promise resolving to the AxiosResponse of the callback POST request. + */ export const emit = async ({ event, eventType, diff --git a/src/common/errors.ts b/src/common/errors.ts index cd420a95..cb32a8be 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -1,6 +1,15 @@ +/** Prefix used to namespace common error codes emitted by extractors. */ const ERROR_PREFIX = 'ERROR_CODE'; +/** Delimiter joining the error prefix and the specific error name in the encoded code. */ const ERROR_DELIMITER = '='; +/** + * Well-known error codes an extractor can report to signal common, externally-caused failure conditions. + * + * Used to communicate persistent source-system states (deletion, deactivation, missing access/permission) + * or sync-completion signals back to AirSync in a recognized, machine-readable form. Each member's value + * is the encoded string `ERROR_CODE=`. + */ export const enum ExtractionCommonError { // Indicates that the external system is permanently inactive or inaccessible. // This is used for persistent conditions (system deleted, deactivated, access permanently revoked) diff --git a/src/common/install-initial-domain-mapping.ts b/src/common/install-initial-domain-mapping.ts index 5a8ec436..b0adb6fd 100644 --- a/src/common/install-initial-domain-mapping.ts +++ b/src/common/install-initial-domain-mapping.ts @@ -4,6 +4,19 @@ import { AirSyncEvent } from '../types/extraction'; import { serializeError } from '../logger/logger'; import { InitialDomainMapping } from '../types/common'; +/** + * Installs the connector's initial domain mapping into the sync. + * + * Used the first time a sync needs the mapping that describes how external record types/fields map + * to DevRev. It resolves the snap-in's import and version slugs, optionally creates the starting + * recipe blueprint, then installs the initial domain mapping (plus any additional mappings) via the + * airdrop recipe API. If no mapping JSON is provided the call is a no-op; blueprint creation failures + * are logged and the install continues without a blueprint. + * + * @param event - The AirSyncEvent providing the DevRev endpoint, service account token, and snap-in ID. + * @param initialDomainMappingJson - The InitialDomainMapping JSON containing the starting recipe blueprint and additional mappings. + * @returns Promise that resolves once the mapping has been installed (or early-returns if none is provided). + */ export async function installInitialDomainMapping( event: AirSyncEvent, initialDomainMappingJson: InitialDomainMapping diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts index dec92853..787b2407 100644 --- a/src/multithreading/adapters/base-adapter.ts +++ b/src/multithreading/adapters/base-adapter.ts @@ -23,10 +23,13 @@ import { Uploader } from '../../uploader/uploader'; import { getEventTypeForResult } from '../spawn/spawn.helpers'; /** - * BaseAdapter holds the state and behavior shared by both sync modes and owns - * the `emit` control-protocol flow as a template method. Mode-specific adapters - * (`ExtractionAdapter`, `LoadingAdapter`) implement the abstract hooks to inject - * their own pre-emit work and event payload shaping. + * Abstract base for the worker adapters, holding state and behavior shared by + * both sync modes and owning the `emit` control-protocol flow as a template method. + * + * Used as the type passed to worker tasks; mode-specific adapters + * (`ExtractionAdapter`, `LoadingAdapter`) extend it and implement the abstract + * hooks (`beforeEmit`, `buildEmitPayload`, `afterEmit`) to inject their own + * pre-emit work and event payload shaping. * * @typeParam ConnectorState - the connector-owned state shape */ @@ -73,10 +76,18 @@ export abstract class BaseAdapter { return this.adapterState.sdkState; } + /** Per-item-type extraction scope (which item types to extract). */ get extractionScope() { return this.adapterState.extractionScope; } + /** + * Persists the current adapter state to the platform. + * + * Used to checkpoint connector and SDK state outside of an emit. + * + * @returns Promise that resolves once the state has been posted. + */ async postState() { return runWithSdkLogContext(async () => { await this.adapterState.postState(); @@ -113,12 +124,15 @@ export abstract class BaseAdapter { * Maps a {@link TaskResult} returned by a worker's task/onTimeout callback to * the phase-appropriate platform event and emits it exactly once. * - * This is the SDK-internal bridge between the return-based connector contract + * Used as the SDK-internal bridge between the return-based connector contract * and the control protocol; it is invoked by the worker driver, not by * connectors. Connectors signal outcomes by returning a `TaskResult`, never by - * calling `emit` directly. + * calling `emit` directly. A `delay`/`error` status carries its delay seconds + * or error into the event data; a status that is illegal for a non-resumable + * phase is downgraded to an error event. * - * @param result - The status the worker reported for the current phase. + * @param result - The TaskResult status the worker reported for the current phase. + * @returns Promise that resolves once the mapped event has been emitted. */ async emitFromResult(result: TaskResult): Promise { const { eventType, illegal } = getEventTypeForResult( @@ -141,10 +155,17 @@ export abstract class BaseAdapter { } /** - * Emits an event to the platform. + * Emits a single event to the platform via the template-method flow. + * + * Used as the one place that sends a control-protocol event: it runs the + * `beforeEmit` hook, persists state (except for stateless start/delete events), + * merges in the mode-specific `buildEmitPayload`, sends the event, then runs + * `afterEmit`. Guarded by `hasWorkerEmitted` so it emits at most once; any + * failure in preparation, state posting, or sending signals the worker to exit. * - * @param newEventType - The event type to be emitted - * @param data - The data to be sent with the event + * @param newEventType - The ExtractorEventType or LoaderEventType to emit. + * @param data - Optional EventData (e.g. delay or error) merged into the payload. + * @returns Promise that resolves once the emit attempt has completed. */ protected async emit( newEventType: ExtractorEventType | LoaderEventType, diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index 5bbbffa0..39cd036f 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -32,10 +32,14 @@ import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; import { BaseAdapter } from './base-adapter'; /** - * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the - * extraction surface (repos, artifacts, attachment streaming) and uploads - * pending repos and updates the extraction boundaries before emitting. + * Worker adapter passed to extraction tasks, exposing the extraction surface + * (repos, artifacts, attachment streaming). * + * Used during the extraction phases; before emitting it uploads all pending + * repos and, on `AttachmentExtractionDone`, advances the sync boundaries on + * `sdkState`. + * + * @typeParam ConnectorState - the connector-owned state shape * @public */ export class ExtractionAdapter< @@ -56,7 +60,12 @@ export class ExtractionAdapter< /** * Returns whether the given item type should be extracted. - * Defaults to true if the scope is empty or the item type is not listed. + * + * Used to honor the per-item-type extraction scope; defaults to true when the + * scope is empty or the item type is not listed. + * + * @param itemType - The item type name to check. + * @returns True if the item type should be extracted. */ shouldExtract(itemType: string): boolean { const scope = this.extractionScope; @@ -65,6 +74,17 @@ export class ExtractionAdapter< return scope[itemType].extract; } + /** + * Initializes the adapter's repos from the given repo definitions. + * + * Used to set up the in-memory item buffers an extraction task pushes to; + * each repo normalizes items (except external domain metadata and SSOR + * attachments) and, on upload, records attachment artifact IDs in state and + * tracks event payload size to trigger a timeout when the SQS size threshold + * is exceeded. + * + * @param repos - The RepoInterface definitions to build repos from. + */ initializeRepos(repos: RepoInterface[]) { this.repos = repos.map((repo) => { const shouldNormalize = @@ -104,6 +124,15 @@ export class ExtractionAdapter< }); } + /** + * Looks up an initialized repo by item type. + * + * Used by extraction tasks to get the buffer they push normalized items into; + * logs an error and returns undefined when no repo matches. + * + * @param itemType - The item type name of the repo to find. + * @returns The matching Repo, or undefined if none was initialized. + */ getRepo(itemType: string): Repo | undefined { return runWithSdkLogContext(() => { const repo = this.repos.find((repo) => repo.itemType === itemType); @@ -117,16 +146,32 @@ export class ExtractionAdapter< }); } + /** Artifacts accumulated since the last emit, sent with the next event. */ get artifacts(): Artifact[] { return this._artifacts; } + /** Appends artifacts to the accumulator, de-duplicating by reference. */ set artifacts(artifacts: Artifact[]) { this._artifacts = this._artifacts .concat(artifacts) .filter((value, index, self) => self.indexOf(value) === index); } + /** + * Pre-emit hook: uploads all repos and, when extraction completes, advances + * the sync boundaries. + * + * Used as the extraction implementation of the `BaseAdapter` template hook. + * Always uploads pending repos; on `AttachmentExtractionDone` it promotes + * `lastSyncStarted` to `lastSuccessfulSyncStarted`, clears the pending worker + * boundaries, and expands `workersOldest`/`workersNewest` from the event's + * extract-from/extract-to window. + * + * @param newEventType - The event type about to be emitted. + * @returns Promise that resolves once pre-emit work is done; rejects (aborting + * the emit) if a repo upload fails. + */ protected async beforeEmit( newEventType: ExtractorEventType | LoaderEventType ): Promise { @@ -178,6 +223,15 @@ export class ExtractionAdapter< } } + /** + * Builds the extraction-specific extras merged into the emitted event payload. + * + * Used as the extraction implementation of the `BaseAdapter` template hook; + * returns the accumulated artifacts for extraction events and nothing otherwise. + * + * @param newEventType - The event type about to be emitted. + * @returns EventData carrying the accumulated artifacts for extraction events. + */ protected buildEmitPayload( newEventType: ExtractorEventType | LoaderEventType ): EventData { @@ -187,10 +241,24 @@ export class ExtractionAdapter< return isExtractionEvent ? { artifacts: this.artifacts } : {}; } + /** + * Post-emit hook: clears the accumulated artifacts after a successful emit. + * + * Used as the extraction implementation of the `BaseAdapter` template hook so + * artifacts are not re-sent on a subsequent emit. + */ protected afterEmit(): void { this.artifacts = []; } + /** + * Uploads all initialized repos and collects their resulting artifacts. + * + * Used by `beforeEmit` to flush buffered items to artifacts before an event is + * sent; throws on the first repo that fails to upload. + * + * @returns Promise that resolves once every repo has been uploaded. + */ async uploadAllRepos(): Promise { for (const repo of this.repos) { const error = await repo.upload(); @@ -201,6 +269,20 @@ export class ExtractionAdapter< } } + /** + * Streams a single attachment from the external system into a DevRev artifact + * and records the SSOR attachment mapping. + * + * Used while streaming attachments: it opens the external stream, requests an + * artifact upload URL, streams the bytes, confirms the upload, then pushes an + * `ssor_attachment` linking the external and DevRev IDs. A delay or error from + * the stream is surfaced back to the caller and the HTTP stream is destroyed. + * + * @param attachment - The NormalizedAttachment to fetch and upload. + * @param stream - The ExternalSystemAttachmentStreamingFunction that opens the source stream. + * @returns Promise with undefined on success, or a ProcessAttachmentReturnType + * carrying a delay or error. + */ async processAttachment( attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction @@ -316,8 +398,12 @@ export class ExtractionAdapter< } /** - * Destroys a stream to prevent memory leaks. - * @param httpStream - The axios response stream to destroy + * Destroys an HTTP response stream to prevent memory leaks. + * + * Used to release an open attachment source stream when uploading fails; + * swallows any error raised while destroying. + * + * @param httpStream - The AxiosResponse stream to destroy. */ private destroyHttpStream(httpStream: AxiosResponse): void { try { @@ -334,11 +420,20 @@ export class ExtractionAdapter< } /** - * Streams the attachments to the DevRev platform. - * The attachments are streamed to the platform and the artifact information is returned. - * @param params - The parameters to stream the attachments - * @returns The response object containing the ssorAttachment artifact information - * or error information if there was an error + * Streams all pending attachments to the DevRev platform and returns the phase + * outcome. + * + * Used as the entry point for the attachment-streaming phase: it iterates the + * attachments-metadata artifact IDs recorded in state, fetches each batch, and + * streams them either through caller-provided processors or the default + * streaming pool (with batch size clamped to 1..50). Returns `delay` on a + * rate limit, `error` on failure, `progress` on timeout (to resume in a fresh + * invocation), and `success` once every artifact ID is processed. + * + * @param stream - The ExternalSystemAttachmentStreamingFunction used to open each source stream. + * @param processors - Optional ExternalSystemAttachmentProcessors (reducer + iterator) overriding the default pool. + * @param batchSize - Number of attachments to stream concurrently; defaults to 1, clamped to 1..50. + * @returns Promise with the TaskResult describing the phase outcome. */ async streamAttachments({ stream, diff --git a/src/multithreading/adapters/loading-adapter.helpers.ts b/src/multithreading/adapters/loading-adapter.helpers.ts index ecab2de1..e7f31876 100644 --- a/src/multithreading/adapters/loading-adapter.helpers.ts +++ b/src/multithreading/adapters/loading-adapter.helpers.ts @@ -6,10 +6,15 @@ import { } from '../../types/loading'; /** - * Gets the files to load for the loader. - * @param {string[]} supportedItemTypes - The supported item types - * @param {StatsFileObject[]} statsFile - The stats file - * @returns {FileToLoad[]} The files to load + * Builds the ordered list of files to load from a stats file. + * + * Used by the loading adapter to filter the stats file down to the supported + * item types, order the entries to match that item-type order, and shape each + * into a FileToLoad with progress fields reset. + * + * @param supportedItemTypes - The supported item type names, in desired load order. + * @param statsFile - The StatsFileObject entries describing available files. + * @returns The FileToLoad entries to process, ordered by supported item type. */ export function getFilesToLoad({ supportedItemTypes, @@ -50,10 +55,15 @@ export function getFilesToLoad({ } /** - * Adds a report to the loader report. - * @param {LoaderReport[]} loaderReports - The loader reports - * @param {LoaderReport} report - The report to add - * @returns {LoaderReport[]} The updated loader reports + * Merges a report into the accumulated loader reports. + * + * Used to keep one running report per item type: when a report for the same + * item type already exists its created/updated/failed counts are summed, + * otherwise the report is appended. + * + * @param loaderReports - The existing LoaderReport accumulator (mutated in place). + * @param report - The LoaderReport to merge in. + * @returns The updated LoaderReport accumulator. */ export function addReportToLoaderReport({ loaderReports, diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts index 49fdd2bf..70c4fe94 100644 --- a/src/multithreading/adapters/loading-adapter.ts +++ b/src/multithreading/adapters/loading-adapter.ts @@ -36,9 +36,14 @@ import { } from './loading-adapter.helpers'; /** - * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading - * surface (item/attachment loading, mappers, loader reports). + * Worker adapter passed to loading tasks, exposing the loading surface + * (item/attachment loading, mappers, loader reports). * + * Used during the loading phases to push transformed DevRev items into the + * external system, maintaining sync mapper records and accumulating per-item-type + * loader reports across emits. + * + * @typeParam ConnectorState - the connector-owned state shape * @public */ export class LoadingAdapter< @@ -62,22 +67,43 @@ export class LoadingAdapter< }); } + /** Per-item-type loader reports accumulated across loads, sent with each emit. */ get reports(): LoaderReport[] { return this.loaderReports; } + /** Artifact IDs of transformer files that have been fully loaded. */ get processedFiles(): string[] { return this._processedFiles; } + /** Mappers client for reading/writing sync mapper records. */ get mappers(): Mappers { return this._mappers; } + /** + * Pre-emit hook for loading; intentionally a no-op. + * + * Used as the loading implementation of the `BaseAdapter` template hook; + * loading has no repos to upload and no extraction boundaries to advance. + * + * @returns Promise that resolves immediately. + */ protected async beforeEmit(): Promise { // Loading has no pre-emit work (no repos, no extraction boundaries). } + /** + * Builds the loading-specific extras merged into the emitted event payload. + * + * Used as the loading implementation of the `BaseAdapter` template hook; + * returns the accumulated reports and processed files for loader events and + * nothing otherwise. + * + * @param newEventType - The event type about to be emitted. + * @returns EventData carrying loader reports and processed files for loader events. + */ protected buildEmitPayload( newEventType: ExtractorEventType | LoaderEventType ): EventData { @@ -92,10 +118,29 @@ export class LoadingAdapter< : {}; } + /** + * Post-emit hook for loading; intentionally a no-op. + * + * Used as the loading implementation of the `BaseAdapter` template hook; + * loading keeps its accumulated reports and processed files across emits. + */ protected afterEmit(): void { // Loading keeps its accumulated reports/processed files across emits. } + /** + * Loads all supported item types into the external system and returns the + * phase outcome. + * + * Used as the entry point for the data-loading phase: on `StartLoadingData` it + * resolves the batches of transformer files to load from the stats file, then + * for each file loads every item via `loadItem`, tracking progress per line so + * a fresh invocation can resume. Returns `delay` on a rate limit, `progress` + * on timeout, `error` on failure, and `success` once all files are processed. + * + * @param itemTypesToLoad - The ItemTypeToLoad definitions (create/update handlers per item type). + * @returns Promise with the TaskResult describing the phase outcome. + */ async loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise { @@ -216,6 +261,17 @@ export class LoadingAdapter< }); } + /** + * Resolves the ordered list of transformer files to load for the given item + * types. + * + * Used by `loadItemTypes`/`loadAttachments` to turn the event's stats file + * into `FileToLoad` batches, filtered to the supported item types; returns an + * empty list when there is no stats file, it cannot be fetched, or it is empty. + * + * @param supportedItemTypes - The item type names to include, in load order. + * @returns Promise with the FileToLoad batches to process. + */ async getLoaderBatches({ supportedItemTypes, }: { @@ -248,6 +304,18 @@ export class LoadingAdapter< }); } + /** + * Loads attachments into the external system and returns the phase outcome. + * + * Used as the entry point for the attachment-loading phase: on + * `StartLoadingAttachments` it resolves the attachment transformer files to + * load, then creates each attachment via `loadAttachment`, tracking progress + * per line for resumption. Returns `delay` on a rate limit, `progress` on + * timeout, `error` on failure, and `success` once all files are processed. + * + * @param create - The ExternalSystemLoadingFunction that creates an attachment in the external system. + * @returns Promise with the TaskResult describing the phase outcome. + */ async loadAttachments({ create, }: { @@ -344,6 +412,19 @@ export class LoadingAdapter< }); } + /** + * Loads a single item into the external system, creating or updating as needed. + * + * Used per item by `loadItemTypes`: it looks up the sync mapper record by + * DevRev target ID and calls the item type's `update` handler; if no mapper + * record exists (404) it falls back to the `create` handler. On success it + * creates or updates the sync mapper record and reports the action; a rate + * limit surfaces a delay and other failures are reported as failed. + * + * @param item - The ExternalSystemItem to load. + * @param itemTypeToLoad - The ItemTypeToLoad providing the create/update handlers. + * @returns Promise with a LoadItemResponse carrying a report, a rateLimit delay, or an error. + */ async loadItem({ item, itemTypeToLoad, @@ -550,6 +631,18 @@ export class LoadingAdapter< }); } + /** + * Creates a single attachment in the external system. + * + * Used per attachment by `loadAttachments`: it calls the `create` handler and, + * on success, creates a sync mapper record linking the new external ID to the + * attachment's reference ID and reports it as created. A rate limit surfaces a + * delay; a missing ID is reported as failed (attachments are create-only). + * + * @param item - The ExternalSystemAttachment to create. + * @param create - The ExternalSystemLoadingFunction that creates the attachment. + * @returns Promise with a LoadItemResponse carrying a report or a rateLimit delay. + */ async loadAttachment({ item, create, diff --git a/src/multithreading/create-worker.ts b/src/multithreading/create-worker.ts index abd35034..72b9655b 100644 --- a/src/multithreading/create-worker.ts +++ b/src/multithreading/create-worker.ts @@ -2,6 +2,16 @@ import { isMainThread, Worker } from 'node:worker_threads'; import { WorkerData, WorkerEvent } from '../types/workers'; +/** + * Creates a Node worker thread that runs the snap-in's task worker script. + * + * Used by `spawn` to launch the off-main-thread worker that processes an + * extraction/loading event; the promise settles once the worker comes online + * so the caller can wire up timeouts and lifecycle handling. + * + * @param workerData - The data of type WorkerData passed to the worker thread (event, initial state, options, etc.). + * @returns A Promise that resolves with the online Worker instance, or rejects with the Error if the worker fails to start or is itself a worker thread. + */ async function createWorker( workerData: WorkerData ): Promise { diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index a43debbb..6db0d7c0 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -30,6 +30,12 @@ import { LoadingAdapter } from './adapters/loading-adapter'; * If `onTimeout` is omitted, the SDK emits a phase-appropriate default on * timeout: `progress` (resumable phases) or `error` (non-resumable phases) is * handled by the status->event mapping when we emit a `progress` result. + * + * @param buildAdapter - Factory that constructs the typed adapter for this worker. + * @param params - The task hooks of type ProcessTaskInterface. + * @param params.task - The worker's main task; receives the adapter and resolves to a TaskResult. + * @param params.onTimeout - Optional callback run on soft timeout when nothing has emitted yet; resolves to a TaskResult. + * @returns A Promise that resolves once the result has been emitted and the worker process exits. */ async function runWorkerTask>( buildAdapter: () => Promise, @@ -83,6 +89,15 @@ async function runWorkerTask>( * Entry point for an extraction worker. Builds an {@link ExtractionAdapter} and * runs the provided task against it. * + * Used as the worker-script entry the snap-in calls inside an extraction worker + * thread; returns immediately on the main thread so the same module can be + * imported there safely. + * + * @param params - The task hooks of type ProcessTaskInterface for an ExtractionAdapter. + * @param params.task - The extraction task; receives the adapter and resolves to a TaskResult. + * @param params.onTimeout - Optional callback run on soft timeout; resolves to a TaskResult. + * @returns Nothing; emission and process exit are handled by the shared driver. + * * @public */ export function processExtractionTask({ @@ -123,6 +138,15 @@ export function processExtractionTask({ * Entry point for a loading worker. Builds a {@link LoadingAdapter} and runs the * provided task against it. * + * Used as the worker-script entry the snap-in calls inside a loading worker + * thread; returns immediately on the main thread so the same module can be + * imported there safely. + * + * @param params - The task hooks of type ProcessTaskInterface for a LoadingAdapter. + * @param params.task - The loading task; receives the adapter and resolves to a TaskResult. + * @param params.onTimeout - Optional callback run on soft timeout; resolves to a TaskResult. + * @returns Nothing; emission and process exit are handled by the shared driver. + * * @public */ export function processLoadingTask({ diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index 1f52764c..30a00e15 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -59,7 +59,15 @@ export function getEventTypeForResult( /** * Per-phase outgoing event types, keyed by the incoming {@link EventType}. - * `resumable` phases define progress/delayed events; non-resumable ones do not. + * + * Each entry maps a phase to its outgoing events: every phase has a `done` + * (success) and `error` event. `resumable` phases (data/attachment extraction, + * data/attachment loading) additionally define `progress` and `delayed` events + * and accept all four statuses. Non-resumable phases (external sync units, + * metadata, state deletions) omit `progress`/`delayed`; a `progress`/`delay` + * status there is illegal and {@link getEventTypeForResult} collapses it to the + * `error` event while flagging it. Event types absent from this partial map are + * treated as unrecognized. */ const EVENT_PHASE_MAP: Partial< Record< diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 02176ff0..f31ecba7 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -25,6 +25,17 @@ import { getNoScriptEventType, } from './spawn.helpers'; +/** + * Resolves the default worker script path for an incoming event type. + * + * Used by `spawn` to pick which built-in worker (external sync units, metadata, + * data/attachment extraction, data/attachment loading) to run when the caller + * has not supplied an explicit `workerPath` or override. + * + * @param event - The AirSync event whose `payload.event_type` selects the worker. + * @param workerBasePath - The base directory string the resolved relative worker path is appended to. + * @returns The full worker script path string, or null if the event type has no matching built-in worker. + */ function getWorkerPath({ event, workerBasePath, @@ -63,12 +74,14 @@ function getWorkerPath({ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker. * The class provides utilities to emit control events to the platform and exit the worker gracefully. * In case of lambda timeout, the class emits a lambda timeout event to the platform. - * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class - * @param {AirSyncEvent} options.event - The event object received from the platform - * @param {object} options.initialState - The initial state of the adapter - * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath - * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname` - * @returns {Promise} - A new instance of Spawn class + * @param options - The options of type SpawnFactoryInterface used to launch the worker. + * @param options.event - The AirSync event object received from the platform. + * @param options.initialState - The initial connector state handed to the worker. + * @param options.initialDomainMapping - The initial domain mapping handed to the worker. + * @param options.options - Optional SDK behavior overrides (timeout, local development, worker path overrides, etc.). + * @param options.workerPath - Optional explicit path to the worker script; takes precedence over overrides and the default resolver. + * @param options.baseWorkerPath - The base path for the worker files, usually `__dirname`. + * @returns A Promise that resolves once the worker finishes (or a no-script default event is emitted), or rejects if the worker fails to start. */ export async function spawn({ event, @@ -154,6 +167,16 @@ export async function spawn({ } } +/** + * Manages the lifecycle of a spawned worker thread for a single event. + * + * Used by `spawn` to supervise the worker: it arms a soft timeout (asks the + * worker to exit gracefully) and a hard timeout (terminates a stuck worker), + * relays the worker's log messages to the main thread, tracks whether the + * worker has already emitted an event, periodically logs memory usage, and on + * worker exit clears the timers and resolves the spawn promise -- emitting a + * timeout error event if the worker exited without emitting one itself. + */ export class Spawn { private event: AirSyncEvent; private alreadyEmitted: boolean; diff --git a/src/repo/repo.interfaces.ts b/src/repo/repo.interfaces.ts index f790be2a..583f71e9 100644 --- a/src/repo/repo.interfaces.ts +++ b/src/repo/repo.interfaces.ts @@ -4,54 +4,84 @@ import { AirSyncEvent } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; /** - * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data. + * Describes a repo configuration that stores and uploads extracted data of one item type. + * + * Used to declare which item type a repo holds and how its raw records should be normalized. */ export interface RepoInterface { + /** The item type the repo buffers and uploads. */ itemType: string; + /** Optional normalizer turning a raw record into a NormalizedItem or NormalizedAttachment. */ normalize?: (record: object) => NormalizedItem | NormalizedAttachment; + /** Optional worker adapter options that override defaults (e.g. batch size). */ overridenOptions?: WorkerAdapterOptions; } /** - * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo. + * Construction parameters used to create a Repo instance. + * + * Used to wire a repo to its triggering event, item type, normalizer, upload callback, and options. */ export interface RepoFactoryInterface { + /** The AirSync event that drives the extraction and supplies platform credentials. */ event: AirSyncEvent; + /** The item type the repo buffers and uploads. */ itemType: string; + /** Optional normalizer turning a raw record into a NormalizedItem or NormalizedAttachment. */ normalize?: (record: object) => NormalizedItem | NormalizedAttachment; + /** Callback invoked with each Artifact once it has been uploaded. */ onUpload: (artifact: Artifact) => void; + /** Optional worker adapter options that override defaults (e.g. batch size). */ options?: WorkerAdapterOptions; } /** - * NormalizedItem is an interface of item after normalization. + * An external system item after normalization into the shape AirSync expects. + * + * Used as the uploaded representation of a non-attachment record. */ export interface NormalizedItem { + /** External system identifier of the item. */ id: string; + /** ISO timestamp of when the item was created in the external system. */ created_date: string; + /** ISO timestamp of when the item was last modified in the external system. */ modified_date: string; + /** Normalized field values of the item. */ data: object; } /** - * NormalizedAttachment is an interface of attachment after normalization. + * An external system attachment after normalization into the shape AirSync expects. + * + * Used as the uploaded metadata for an attachment whose binary is streamed separately. */ export interface NormalizedAttachment { + /** Source URL the attachment binary can be downloaded from. */ url: string; + /** External system identifier of the attachment. */ id: string; + /** Name of the attached file. */ file_name: string; + /** External system identifier of the item the attachment belongs to. */ parent_id: string; + /** Optional external system identifier of the attachment's author. */ author_id?: string; + /** Whether the attachment is embedded inline (e.g. in rich text) rather than a standalone file. */ inline?: boolean; + /** Optional MIME type of the attachment. */ content_type?: string; // This should be a string, but it was a number in the past. Due to backwards // compatibility we are keeping it also as a number. + /** Optional external identifier of the parent's parent; kept as number for backwards compatibility. */ grand_parent_id?: number | string; } /** - * Item is an interface that defines the structure of an item. + * A raw, un-normalized record extracted from the external system. + * + * Used as the input to a repo's normalize function before items are uploaded. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export type Item = Record; diff --git a/src/repo/repo.ts b/src/repo/repo.ts index d119eea8..e694b584 100644 --- a/src/repo/repo.ts +++ b/src/repo/repo.ts @@ -16,6 +16,12 @@ import { RepoFactoryInterface, } from './repo.interfaces'; +/** + * In-memory buffer that accumulates normalized items of a single item type during extraction. + * + * Used to batch pushed items (ARTIFACT_BATCH_SIZE per batch), normalize them, and upload them as + * artifacts to the DevRev platform, firing the onUpload callback for each uploaded artifact. + */ export class Repo { readonly itemType: string; private items: (NormalizedItem | NormalizedAttachment | Item)[]; @@ -41,10 +47,20 @@ export class Repo { this.uploadedArtifacts = []; } + /** Returns the items currently buffered in the repo (not yet uploaded). */ getItems(): (NormalizedItem | NormalizedAttachment | Item)[] { return this.items; } + /** + * Uploads a batch of items (or all buffered items) as a single artifact. + * + * Used to flush buffered items to the DevRev platform; on success the artifact is passed to + * onUpload and recorded in uploadedArtifacts. When no explicit batch is given the buffer is cleared. + * + * @param batch - Optional explicit array of NormalizedItem, NormalizedAttachment, or Item to upload; defaults to all buffered items. + * @returns Promise that resolves to void on success, or an ErrorRecord describing the upload failure. + */ async upload( batch?: (NormalizedItem | NormalizedAttachment | Item)[] ): Promise { @@ -84,6 +100,16 @@ export class Repo { } } + /** + * Normalizes and buffers items, uploading full batches as they accumulate. + * + * Used by connectors to feed extracted items into the repo; items are normalized (unless the item + * type is external domain metadata or SSOR attachments) and any complete batches of batchSize are + * uploaded immediately, leaving the remainder buffered for a later flush. + * + * @param items - Array of raw Item records to normalize and buffer. + * @returns Promise that resolves to true when items were buffered/uploaded successfully, or false if a batch upload threw. + */ async push(items: Item[]): Promise { let recordsToPush: (NormalizedItem | NormalizedAttachment | Item)[]; diff --git a/src/state/base-state.ts b/src/state/base-state.ts index 06db8959..1b860e86 100644 --- a/src/state/base-state.ts +++ b/src/state/base-state.ts @@ -17,10 +17,11 @@ import { } from './state.interfaces'; /** - * BaseState owns the state lifecycle shared by every sync mode: connector vs. - * SDK state separation, fetch/init/post against the platform, the v1->v2 - * migration shim, and the snap-in-version-gated initial domain mapping install. + * Abstract base owning the adapter state lifecycle shared by every sync mode. * + * Used to keep connector-owned state separate from SDK bookkeeping, fetch/init/ + * post the persisted state against the platform, run the v1->v2 migration shim, + * and install the initial domain mapping gated on the snap-in version. * Mode-specific subclasses (`ExtractionState`, `LoadingState`) seed the * SDK-owned portion of the state and add mode-specific setup in their factories. * @@ -69,15 +70,21 @@ export abstract class BaseState { this._sdkState = value; } + /** The per-sync-unit extraction scope (object types to extract), loaded alongside state. */ get extractionScope(): ExtractionScope { return this._extractionScope; } /** - * Installs the initial domain mapping when the snap-in version in state does - * not match the version in the event context. Shared by all modes so that a - * loading run still installs the mapping if extraction has not done so. - * @param initialDomainMapping The initial domain mapping passed to spawn + * Installs the initial domain mapping when the version in state is stale. + * + * Used by all modes (so a loading run still installs the mapping if extraction + * has not) to (re)install whenever `sdkState.snapInVersionId` is absent or + * differs from the event context's snap-in version; on success the new version + * is recorded in state. A missing mapping or install error fails the worker. + * + * @param initialDomainMapping - The initial domain mapping of type InitialDomainMapping passed to the spawn function; required when an install is needed + * @returns Promise that resolves once the mapping is installed or the install is skipped */ async installInitialDomainMappingIfNeeded( initialDomainMapping?: InitialDomainMapping @@ -122,13 +129,17 @@ export abstract class BaseState { } /** - * Initializes the state for this adapter instance by fetching from API - * or creating an initial state if none exists (404). + * Initializes this adapter's state from persisted state, or seeds it on first run. * - * Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat - * v1 blob (connector keys merged with SDK keys), migrating the latter on read. - * Always persists the v2 envelope going forward. - * @param initialState The initial connector state provided by the spawn function + * Used at worker start to load and normalize state: it fetches the persisted + * blob, parses it, and runs `normalizeFetchedState` so both the v2 + * `{ connectorState, sdkState }` envelope and a legacy flat v1 blob are + * accepted (the latter migrated on read). It also restores the extraction + * scope. On a 404 it seeds the initial state and persists the v2 envelope; + * any other failure fails the worker. + * + * @param initialState - The initial connector state of type ConnectorState provided by the spawn function, used when no state exists yet + * @returns Promise that resolves once state has been loaded or seeded */ async init(initialState: ConnectorState): Promise { try { @@ -181,13 +192,20 @@ export abstract class BaseState { } /** - * Normalizes a parsed on-disk state into the `{ connectorState, sdkState }` - * envelope, migrating a legacy flat v1 blob if needed. + * Normalizes parsed on-disk state into the `{ connectorState, sdkState }` envelope, migrating legacy v1 state. + * + * Used as the v1->v2 migration shim so older snap-ins keep working after the + * state split. Behavior by shape of the parsed input: + * - v2 envelope (`{ connectorState, sdkState }`): used as-is, with `sdkState` + * merged over the mode's initial SDK state to backfill newly added fields. + * - Legacy v1 flat blob: top-level keys present in `V1_SDK_STATE_KEYS` are + * split into `sdkState`, everything else becomes connector state. + * - Malformed envelope (one of `connectorState`/`sdkState` present, the other + * missing) or non-object input: throws. * - * - v2 envelope (`{ connectorState, sdkState }`): used as-is. - * - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`, - * everything else becomes connector state. - * - Malformed envelope (one side present, the other missing) fails loud. + * @param parsed - The JSON-parsed persisted state of unknown shape (v2 envelope or legacy v1 flat blob) + * @returns The split state as `{ connectorState, sdkState }`, with `sdkState` merged over the initial SDK state + * @throws Error when the input is not an object or is a malformed envelope */ private normalizeFetchedState(parsed: unknown): { connectorState: ConnectorState; @@ -231,9 +249,14 @@ export abstract class BaseState { } /** - * Updates the state of the adapter by posting to API. - * Persists the v2 `{ connectorState, sdkState }` envelope. - * @param {object} state - The connector state to be updated + * Persists the adapter state to the platform. + * + * Used to checkpoint progress: wraps the current connector and SDK state into + * the v2 `{ connectorState, sdkState }` envelope, serializes it, and posts it. + * A serialization or request failure fails the worker. + * + * @param state - Optional connector state of type ConnectorState to set and persist; when omitted the current `this.state` is used + * @returns Promise that resolves once the state has been persisted */ async postState(state?: ConnectorState) { const url = this.workerUrl + '.update'; @@ -296,8 +319,12 @@ export abstract class BaseState { } /** - * Fetches the state of the adapter from API. - * @return The raw state data from API + * Fetches the raw persisted adapter state from the platform. + * + * Used by `init` to read the stored state before normalization; returns the + * raw, still-stringified payload without parsing or migrating it. + * + * @returns Promise resolving to `{ state, objects }`, where `state` is the stringified state blob and `objects` is the optional stringified extraction scope */ async fetchState(): Promise<{ state: string; objects?: string }> { console.log( diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts index b78dbbc0..49bd185d 100644 --- a/src/state/extraction-state.ts +++ b/src/state/extraction-state.ts @@ -10,10 +10,13 @@ import { BaseState } from './base-state'; import { extractionSdkState, StateInterface } from './state.interfaces'; /** - * ExtractionState is the per-mode state for extraction workers. It seeds the - * extraction SDK state (extraction boundaries + attachments bookkeeping) on top - * of the shared lifecycle provided by `BaseState` and adds extraction-window - * resolution. + * Per-mode adapter state for extraction workers. + * + * Used to seed the extraction SDK state (extraction-window boundaries + + * attachments bookkeeping) on top of the shared lifecycle provided by + * `BaseState`, and to resolve the incremental extraction window for each event. + * + * @typeParam ConnectorState - the connector-owned state shape */ export class ExtractionState extends BaseState { constructor(params: StateInterface) { @@ -21,13 +24,19 @@ export class ExtractionState extends BaseState { } /** - * Resolves the extraction window onto the event context. + * Computes the incremental extraction window and writes `extract_from`/`extract_to` onto the event context. * - * On StartExtractingData: stamp `lastSyncStarted` if not already set. - * On StartExtractingMetadata: resolve fresh from the TimeValue objects in the - * event context and cache them as pending boundaries (always overwrite). - * On all other events: reuse the pending boundaries cached during - * StartExtractingMetadata. Finally, validate that extract_from < extract_to. + * Used so every extraction phase shares one consistent time window, read from + * the SDK-owned boundary fields in `this.sdkState`. Behavior by event type: + * - StartExtractingData: stamp `lastSyncStarted` if not already set. + * - StartExtractingMetadata: resolve fresh from the TimeValue objects in the + * event context and cache them as pending boundaries (always overwrite). + * - All other events: reuse the pending boundaries cached during + * StartExtractingMetadata. + * Finally validates that `extract_from` is older than `extract_to`, failing + * the worker if the platform supplied an inverted window. + * + * @returns void; mutates the event context and `this.sdkState` in place */ resolveExtractionWindow(): void { const sdkState = this.sdkState; @@ -122,9 +131,14 @@ export class ExtractionState extends BaseState { /** * Creates and initializes an `ExtractionState` for an extraction worker. * - * For non-stateless events this fetches persisted state, installs the initial - * domain mapping if the snap-in version changed, then resolves the extraction - * window (time-value resolution + pending boundary reuse) and validates it. + * Used by the state dispatcher to build extraction-mode state. The initial state + * is deep-cloned to avoid mutating the caller's object; for non-stateless events + * this fetches persisted state, installs the initial domain mapping if the + * snap-in version changed, then resolves the extraction window (time-value + * resolution + pending boundary reuse) and validates it. + * + * @param params - The state factory parameters of type StateInterface (event, initial connector state, optional domain mapping and worker options) + * @returns Promise resolving to the initialized ExtractionState */ export async function createExtractionState({ event, diff --git a/src/state/loading-state.ts b/src/state/loading-state.ts index 151f5bd9..5b03ecb0 100644 --- a/src/state/loading-state.ts +++ b/src/state/loading-state.ts @@ -4,9 +4,13 @@ import { BaseState } from './base-state'; import { loadingSdkState, StateInterface } from './state.interfaces'; /** - * LoadingState is the per-mode state for loading workers. It seeds the loading - * SDK state (files-to-load bookkeeping) on top of the shared lifecycle provided - * by `BaseState`. Loading has no extraction-window resolution. + * Per-mode adapter state for loading workers. + * + * Used to seed the loading SDK state (files-to-load bookkeeping) on top of the + * shared lifecycle provided by `BaseState`. Loading has no extraction-window + * resolution. + * + * @typeParam ConnectorState - the connector-owned state shape */ export class LoadingState extends BaseState { constructor(params: StateInterface) { @@ -17,8 +21,13 @@ export class LoadingState extends BaseState { /** * Creates and initializes a `LoadingState` for a loading worker. * - * For non-stateless events this fetches persisted state and installs the - * initial domain mapping if the snap-in version changed. + * Used by the state dispatcher to build loading-mode state. The initial state is + * deep-cloned to avoid mutating the caller's object; for non-stateless events + * this fetches persisted state and installs the initial domain mapping if the + * snap-in version changed. + * + * @param params - The state factory parameters of type StateInterface (event, initial connector state, optional domain mapping and worker options) + * @returns Promise resolving to the initialized LoadingState */ export async function createLoadingState({ event, diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index 9e105772..e3c45cae 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -3,6 +3,13 @@ import { AirSyncEvent } from '../types/extraction'; import { FileToLoad } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; +/** + * The SDK-owned portion of the persisted adapter state. + * + * Used to hold bookkeeping the SDK manages itself (extraction-window boundaries, + * attachments/files progress, installed snap-in version) separately from + * connector-owned state, so SDK internals never collide with connector keys. + */ export interface SdkState { /** * @deprecated Use extract_from and extract_to from the event context instead, @@ -20,12 +27,15 @@ export interface SdkState { /** The pending (not yet committed) newest extraction boundary (ISO 8601 timestamp). * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */ pendingWorkersNewest?: string; - /** The oldest point of extraction (ISO 8601 timestamp). */ + /** The committed oldest point of extraction (ISO 8601 timestamp). */ workersOldest?: string; - /** The newest point of extraction (ISO 8601 timestamp). */ + /** The committed newest point of extraction (ISO 8601 timestamp). */ workersNewest?: string; + /** Attachments-extraction bookkeeping (artifact ids, progress cursor). Extraction mode only. */ toDevRev?: ToDevRev; + /** Loading bookkeeping (files still to load into DevRev). Loading mode only. */ fromDevRev?: FromDevRev; + /** The snap-in version id whose initial domain mapping is installed; drives reinstall on change. */ snapInVersionId?: string; } @@ -49,6 +59,12 @@ export interface AdapterStateEnvelope { sdkState: SdkState; } +/** + * SDK-owned attachments-extraction state (external system -> DevRev direction). + * + * Used to track which attachment artifacts have been streamed and how far the + * attachments phase has progressed so it can resume after a timeout. + */ export interface ToDevRev { attachmentsMetadata: { artifactIds: string[]; @@ -65,10 +81,23 @@ export interface ProcessedAttachment { parent_id: string; } +/** + * SDK-owned loading state (DevRev -> external system direction). + * + * Used to track which files still need to be loaded into the external system so + * the loading phase can resume after a timeout. + */ export interface FromDevRev { filesToLoad: FileToLoad[]; } +/** + * Constructor/factory parameters for building an adapter state instance. + * + * Used by `createAdapterState` and the per-mode factories to carry the AirSync + * event, the connector's seed state, and the optional initial domain mapping and + * worker options. + */ export interface StateInterface { event: AirSyncEvent; initialState: ConnectorState; @@ -76,6 +105,12 @@ export interface StateInterface { options?: WorkerAdapterOptions; } +/** + * The initial SDK state seeded for extraction-mode workers. + * + * Used by `ExtractionState` as the baseline `sdkState` (extraction-window + * boundaries plus attachments bookkeeping) before any persisted state is merged in. + */ export const extractionSdkState = { lastSyncStarted: '', lastSuccessfulSyncStarted: '', @@ -93,6 +128,12 @@ export const extractionSdkState = { }, }; +/** + * The initial SDK state seeded for loading-mode workers. + * + * Used by `LoadingState` as the baseline `sdkState` (files-to-load bookkeeping) + * before any persisted state is merged in. + */ export const loadingSdkState = { snapInVersionId: '', fromDevRev: { diff --git a/src/state/state.ts b/src/state/state.ts index ab7dbb5c..bcb48dcb 100644 --- a/src/state/state.ts +++ b/src/state/state.ts @@ -10,11 +10,13 @@ export { ExtractionState, createExtractionState } from './extraction-state'; export { LoadingState, createLoadingState } from './loading-state'; /** - * Creates and initializes the adapter state for the current worker, dispatching - * to the extraction or loading state based on the event's sync mode. + * Creates and initializes the adapter state for the current worker. * - * @param params The state factory parameters (event, initial state, options) - * @returns The initialized mode-specific state + * Used as the single entry point that dispatches to either `createLoadingState` + * or `createExtractionState` based on `event.payload.event_context.mode`. + * + * @param params - The state factory parameters of type StateInterface (event, initial state, optional domain mapping and worker options) + * @returns Promise resolving to the initialized mode-specific state (LoadingState when mode is LOADING, otherwise ExtractionState) */ export async function createAdapterState( params: StateInterface diff --git a/src/types/index.ts b/src/types/index.ts index f98c7251..9be77e32 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,3 +1,11 @@ +/** + * Public types barrel for the SDK. + * + * Aggregates and re-exports the commonly used types across the SDK domains — common, extraction, + * loading, repo, state, uploader, mappers, and external domain metadata — so consumers can import + * them from a single entry point. + */ + // Common export { AdapterUpdateParams, diff --git a/src/types/loading.ts b/src/types/loading.ts index 37d2d305..7c58c46b 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -2,22 +2,50 @@ import { Mappers } from '../mappers/mappers'; import { ErrorRecord } from './common'; import { AirSyncEvent } from './extraction'; +/** + * Describes a single prepared data file as listed in the loading stats manifest. + * + * Used during loading to enumerate the artifact files produced by extraction, along with their + * item type and record count, so the loader knows what is available to process. + */ export interface StatsFileObject { + /** Identifier of the artifact/file. */ id: string; + /** External item type contained in the file (e.g. the record type being loaded). */ item_type: string; + /** Name of the file. */ file_name: string; + /** Number of records in the file, as a string. */ count: string; } +/** + * Loader-side view of a file to be loaded, tracking its processing progress. + * + * Used to drive and resume loading of a single data file: it records how many lines exist, the next + * line to process, and whether the file has been fully consumed. + */ export interface FileToLoad { + /** Identifier of the artifact/file. */ id: string; + /** Name of the file. */ file_name: string; + /** External item type contained in the file. */ itemType: string; + /** Total number of records in the file. */ count: number; + /** Index of the next line/record to process; used to resume loading across batches. */ lineToProcess: number; + /** Whether all records in the file have been loaded. */ completed: boolean; } +/** + * An attachment to be loaded into the external system, with its source metadata and parent links. + * + * Used by attachment loading to describe a single file (location, type, size, validity window, + * audit fields) and the DevRev/external parent it belongs to. + */ export interface ExternalSystemAttachment { reference_id: DonV2; parent_type: string; @@ -35,13 +63,28 @@ export interface ExternalSystemAttachment { grand_parent_id?: string; } +/** + * A single item to be loaded into the external system. + * + * Used during loading to carry the DevRev (and optional external) identifiers, audit timestamps, + * and the system-specific payload for one record. + * + * Note: this interface is declared twice in this file with identical members (TypeScript merges + * the declarations); the duplicate is redundant — see report. + */ export interface ExternalSystemItem { + /** Identifiers linking this item to DevRev and, when known, the external system. */ id: { + /** DevRev object identifier (DON). */ devrev: DonV2; + /** External system identifier, present once the item exists in the external system. */ external?: string; }; + /** Creation timestamp of the item. */ created_date: string; + /** Last-modified timestamp of the item. */ modified_date: string; + /** System-specific record payload. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any data: any; } @@ -57,84 +100,195 @@ export interface ExternalSystemItem { data: any; } +/** + * Arguments passed to an external-system loading function for a single item. + * + * Used to give create/update handlers the item to load, the ID mappers for resolving DevRev <-> external + * references, and the current AirSync event for auth/context. + * + * @typeParam Type - The shape of the item being loaded. + */ export interface ExternalSystemItemLoadingParams { item: Type; mappers: Mappers; event: AirSyncEvent; } +/** + * Result returned by an external-system loading function for a single item. + * + * Used to report the outcome of a create/update: the resulting external id, an error message, + * the item's modified date, or a delay (in seconds) when the external system is rate limiting. + */ export interface ExternalSystemItemLoadingResponse { + /** External system id of the loaded item, when the operation succeeded. */ id?: string; + /** Error message when the operation failed. */ error?: string; + /** Modified timestamp reported by the external system after the operation. */ modifiedDate?: string; + /** Suggested delay in seconds before retrying, set when rate limited. */ delay?: number; } +/** + * Record of an item that was loaded into the external system. + * + * Used to persist the outcome of a load (external id, error, modified date) for reporting and + * subsequent runs. + */ export interface ExternalSystemItemLoadedItem { + /** External system id of the loaded item. */ id?: string; + /** Error message if loading the item failed. */ error?: string; + /** Modified timestamp reported by the external system. */ modifiedDate?: string; } +/** + * A handler that loads a single item into the external system. + * + * Used to implement the create or update behavior for an item type; receives the item, ID mappers, + * and event, and resolves with the loading outcome. + * + * @typeParam Item - The shape of the item to load. + * @returns Promise resolving to the ExternalSystemItemLoadingResponse for the item. + */ export type ExternalSystemLoadingFunction = ({ item, mappers, event, }: ExternalSystemItemLoadingParams) => Promise; +/** + * Registration of an item type and the functions that load it. + * + * Used to tell the loader, for a given external item type, how to create and update records in the + * external system. + */ export interface ItemTypeToLoad { + /** External item type these handlers apply to. */ itemType: string; + /** Handler that creates a new record in the external system. */ create: ExternalSystemLoadingFunction; + /** Handler that updates an existing record in the external system. */ update: ExternalSystemLoadingFunction; // requiresSecondPass: boolean; } +/** + * Parameters bundling the full set of item-type loaders for a loading run. + * + * Used to pass the configured list of loadable item types into the loading entry point. + */ export interface ItemTypesToLoadParams { + /** The item types to load, each with its create/update handlers. */ itemTypesToLoad: ItemTypeToLoad[]; } +/** + * Per-item-type counters summarizing the outcome of a loading run. + * + * Used to report, for one item type, how many records were created/updated/skipped/deleted/failed. + */ export interface LoaderReport { + /** External item type this report covers. */ item_type: string; + /** Number of records created. */ [ActionType.CREATED]?: number; + /** Number of records updated. */ [ActionType.UPDATED]?: number; + /** Number of records skipped (no-op). */ [ActionType.SKIPPED]?: number; + /** Number of records deleted. */ [ActionType.DELETED]?: number; + /** Number of records that failed to load. */ [ActionType.FAILED]?: number; } +/** + * Signals that the external system is rate limiting and loading should pause. + * + * Used to propagate a back-off duration from a loading function up to the loader. + */ export interface RateLimited { + /** Number of seconds to wait before resuming. */ delay: number; } +/** + * Result of loading a single item, capturing success report, error, or rate-limit signal. + * + * Used internally by the loader to aggregate per-item outcomes. + */ export interface LoadItemResponse { + /** Error record when the item could not be loaded. */ error?: ErrorRecord; + /** Per-type counters contributed by this item. */ report?: LoaderReport; + /** Rate-limit signal when the external system is throttling. */ rateLimit?: RateLimited; } +/** + * Aggregate result of loading one or more item types. + * + * Used to return the per-type reports and the list of processed files at the end of a loading phase. + */ export interface LoadItemTypesResponse { + /** Per-item-type loading reports. */ reports: LoaderReport[]; + /** Names of the data files that were processed. */ processed_files: string[]; } +/** + * The kinds of actions a loader can perform on a record, used as report counter keys. + * + * Used to key {@link LoaderReport} counters and to classify the outcome of each loaded item. + */ export enum ActionType { + /** A new record was created in the external system. */ CREATED = 'created', + /** An existing record was updated. */ UPDATED = 'updated', + /** The record required no change. */ SKIPPED = 'skipped', + /** The record was deleted. */ DELETED = 'deleted', + /** Loading the record failed. */ FAILED = 'failed', } +/** A DevRev object identifier (DON), represented as a string. */ export type DonV2 = string; +/** + * A sync mapper record linking external and DevRev identifiers for one mapping. + * + * Used to track the correspondence between external ids, secondary ids, and DevRev ids, along with + * status, for sync operations. + */ export type SyncMapperRecord = { + /** External system identifiers for the mapped item. */ external_ids: string[]; + /** Secondary external identifiers (e.g. alternate keys). */ secondary_ids: string[]; + /** DevRev object identifiers for the mapped item. */ devrev_ids: string[]; + /** Status values associated with the mapping. */ status: string[]; + /** Input file the record was sourced from, when applicable. */ input_file?: string; }; +/** + * Outgoing event types reported by the loading phases. + * + * Used as the event_type when a loader emits control messages for data loading, attachment loading, + * and loader-state deletion (progress / delayed / done / error), plus a fallback for unrecognized events. + */ export enum LoaderEventType { DataLoadingProgress = 'DATA_LOADING_PROGRESS', DataLoadingDelayed = 'DATA_LOADING_DELAYED', diff --git a/src/uploader/uploader.helpers.ts b/src/uploader/uploader.helpers.ts index ad389a5b..85087db4 100644 --- a/src/uploader/uploader.helpers.ts +++ b/src/uploader/uploader.helpers.ts @@ -9,9 +9,12 @@ import { import { UploaderResult } from './uploader.interfaces'; /** - * Compresses a JSONL string using gzip compression. - * @param {string} jsonlObject - The JSONL string to compress - * @returns {Buffer | void} The compressed buffer or undefined on error + * Compresses a JSONL string using gzip. + * + * Used to shrink a serialized JSONL batch before uploading it as an artifact. + * + * @param jsonlObject - The JSONL string to compress. + * @returns An UploaderResult wrapping the gzipped Buffer, or an error on failure. */ export function compressGzip(jsonlObject: string): UploaderResult { try { @@ -22,9 +25,12 @@ export function compressGzip(jsonlObject: string): UploaderResult { } /** - * Decompresses a gzipped buffer to a JSONL string. - * @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress - * @returns {string | void} The decompressed JSONL string or undefined on error + * Decompresses a gzipped buffer back into a JSONL string. + * + * Used to restore a downloaded gzipped artifact before parsing it. + * + * @param gzippedJsonlObject - The gzipped Buffer to decompress. + * @returns An UploaderResult wrapping the decompressed JSONL string, or an error on failure. */ export function decompressGzip( gzippedJsonlObject: Buffer @@ -39,8 +45,11 @@ export function decompressGzip( /** * Parses a JSONL string into an array of objects. - * @param {string} jsonlObject - The JSONL string to parse - * @returns {object[] | null} The parsed array of objects or null on error + * + * Used to turn a decompressed artifact into usable records. + * + * @param jsonlObject - The JSONL string to parse. + * @returns An UploaderResult wrapping the parsed object array, or an error on failure. */ export function parseJsonl(jsonlObject: string): UploaderResult { try { @@ -51,10 +60,14 @@ export function parseJsonl(jsonlObject: string): UploaderResult { } /** - * Downloads fetched objects to the local file system (for local development). - * @param {string} itemType - The type of items being downloaded - * @param {object | object[]} fetchedObjects - The objects to write to file - * @returns {Promise} Resolves when the file is written or rejects on error + * Writes fetched objects to the local file system for local development. + * + * Used to inspect extracted data on disk instead of uploading it when running locally; writes a + * timestamped JSON/JSONL file under the `extracted_files` directory. + * + * @param itemType - The string item type, used to name the output file and pick its extension. + * @param fetchedObjects - The object or array of objects to write, one JSON record per line. + * @returns Promise that resolves once the file is written, or rejects on a write error. */ export async function downloadToLocal( itemType: string, @@ -90,9 +103,13 @@ export async function downloadToLocal( } /** - * Truncates a filename if it exceeds the maximum allowed length. - * @param {string} filename - The filename to truncate - * @returns {string} The truncated filename + * Truncates a filename that exceeds the platform's maximum length. + * + * Used before requesting an upload URL so the registered file name stays within DevRev limits, + * preserving the extension and inserting an ellipsis in the middle. + * + * @param filename - The string filename to truncate. + * @returns The original filename if within the limit, otherwise a truncated `name...ext` string. */ export function truncateFilename(filename: string): string { // If the filename is already within the limit, return it as is. diff --git a/src/uploader/uploader.interfaces.ts b/src/uploader/uploader.interfaces.ts index 2224bed7..ee8027c0 100644 --- a/src/uploader/uploader.interfaces.ts +++ b/src/uploader/uploader.interfaces.ts @@ -3,8 +3,15 @@ import { AirSyncEvent } from '../types/extraction'; import { ExternalSystemItem, StatsFileObject } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; +/** + * Construction parameters used to create an Uploader instance. + * + * Used to supply the triggering event (platform endpoint, token, request id) and optional adapter options. + */ export interface UploaderFactoryInterface { + /** The AirSync event supplying the DevRev endpoint, service account token, and request id. */ event: AirSyncEvent; + /** Optional worker adapter options (e.g. local development and skip-confirmation flags). */ options?: WorkerAdapterOptions; } @@ -90,12 +97,26 @@ export interface SsorAttachment { inline?: boolean; } +/** + * Result of fetching and parsing a loading stats file artifact. + * + * Used to return the per-item-type stats produced by the loading phase, or an error if it could not be read. + */ export interface StatsFileResponse { + /** Error describing why the stats file could not be retrieved or parsed. */ error?: ErrorRecord; + /** Parsed stats file entries, one per item type. */ statsFile?: StatsFileObject[]; } +/** + * Result of fetching and parsing a transformer file artifact. + * + * Used to return the transformed external system items to be loaded into DevRev, or an error if it could not be read. + */ export interface TransformerFileResponse { + /** Error describing why the transformer file could not be retrieved or parsed. */ error?: ErrorRecord; + /** Parsed external system items to load. */ transformerFile?: ExternalSystemItem[]; } diff --git a/src/uploader/uploader.ts b/src/uploader/uploader.ts index fa9833af..29ae52c9 100644 --- a/src/uploader/uploader.ts +++ b/src/uploader/uploader.ts @@ -22,6 +22,12 @@ import { UploaderResult, } from './uploader.interfaces'; +/** + * Uploads extraction artifacts to the DevRev platform and reads them back. + * + * Used to compress and upload JSON batches and streamed attachment binaries, obtain upload/download + * URLs, confirm uploads, and download and parse previously uploaded artifacts during sync. + */ export class Uploader { private isLocalDevelopment?: boolean; private devrevApiEndpoint: string; @@ -42,10 +48,14 @@ export class Uploader { } /** - * Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform. - * @param {string} itemType - The type of the item to be uploaded - * @param {object[] | object} fetchedObjects - The objects to be uploaded - * @returns {Promise} - The response object containing the artifact information or error information if there was an error + * Uploads fetched objects to the DevRev platform as a single artifact. + * + * Used to compress the objects into a gzipped JSONL file, request an upload URL, push the file, and + * (unless skipped) confirm the upload, returning the resulting artifact descriptor. + * + * @param itemType - The string item type of the objects being uploaded. + * @param fetchedObjects - The object or array of objects to upload. + * @returns Promise resolving to an UploadResponse with the artifact descriptor, or an error message on failure. */ async upload( itemType: string, @@ -127,11 +137,14 @@ export class Uploader { } /** - * Gets the upload URL for an artifact from the DevRev API. - * @param {string} filename - The name of the file to upload - * @param {string} fileType - The MIME type of the file - * @param {number} [fileSize] - Optional file size in bytes - * @returns {Promise} The artifact upload information or undefined on error + * Requests a pre-signed upload URL and form data for a new artifact. + * + * Used before uploading or streaming a file so the binary can be POSTed to the returned URL. + * + * @param filename - The string file name to register (truncated if it exceeds the platform limit). + * @param fileType - The string MIME type of the file. + * @param fileSize - Optional number of bytes; rejected if 0 or less. + * @returns Promise resolving to an UploaderResult wrapping the ArtifactToUpload, or an error on failure. */ async getArtifactUploadUrl( filename: string, @@ -165,10 +178,13 @@ export class Uploader { } /** - * Uploads an artifact file to the provided upload URL using multipart form data. - * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data - * @param {Buffer} file - The file buffer to upload - * @returns {Promise} The axios response or undefined on error + * Uploads an in-memory file buffer to a pre-signed artifact upload URL. + * + * Used to push a fully buffered artifact (e.g. a compressed JSON batch) as multipart form data. + * + * @param artifact - The ArtifactToUpload descriptor holding the upload URL and form fields. + * @param file - The Buffer containing the file contents to upload. + * @returns Promise resolving to an UploaderResult wrapping the AxiosResponse, or an error on failure. */ async uploadArtifact( artifact: ArtifactToUpload, @@ -193,10 +209,14 @@ export class Uploader { } /** - * Streams an artifact file from an axios response to the upload URL. - * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data - * @param {AxiosResponse} fileStream - The axios response stream containing the file data - * @returns {Promise} The axios response or undefined on error + * Streams a file directly from a source response into a pre-signed artifact upload URL. + * + * Used to upload attachment binaries without buffering them in memory; falls back to the max + * artifact size for Content-Length when the source omits it, and always destroys the source stream. + * + * @param artifact - The ArtifactToUpload descriptor holding the upload URL and form fields. + * @param fileStream - The AxiosResponse whose data stream supplies the file contents. + * @returns Promise resolving to an UploaderResult wrapping the AxiosResponse, or an error on failure. */ async streamArtifact( artifact: ArtifactToUpload, @@ -232,9 +252,12 @@ export class Uploader { } /** - * Confirms that an artifact upload has been completed successfully. - * @param {string} artifactId - The ID of the artifact to confirm - * @returns {Promise} The axios response or undefined on error + * Confirms with the platform that an artifact upload has finished. + * + * Used after pushing the binary so the platform finalizes and accepts the artifact. + * + * @param artifactId - The string ID of the uploaded artifact to confirm. + * @returns Promise resolving to an object with the AxiosResponse on a 2xx, or an error otherwise. */ async confirmArtifactUpload(artifactId: string): Promise<{ response?: AxiosResponse; @@ -273,8 +296,11 @@ export class Uploader { } /** - * Destroys a stream to prevent resource leaks. - * @param {any} fileStream - The axios response stream to destroy + * Destroys a source stream to prevent resource leaks after streaming an artifact. + * + * Used internally by streamArtifact to close the AxiosResponse data stream on both success and error. + * + * @param fileStream - The AxiosResponse whose underlying data stream should be destroyed/closed. */ private destroyStream(fileStream: AxiosResponse): void { try { @@ -292,10 +318,13 @@ export class Uploader { } /** - * Retrieves attachment metadata from an artifact by downloading and parsing it. - * @param {object} param0 - Configuration object - * @param {string} param0.artifact - The artifact ID to download attachments from - * @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object + * Downloads an attachments-metadata artifact and parses it into normalized attachments. + * + * Used during attachment extraction to read back the previously uploaded attachment metadata so its + * binaries can be streamed; resolves the download URL, downloads, gunzips, and parses the JSONL. + * + * @param param0 - Object with `artifact`, the string artifact ID of the attachments-metadata artifact. + * @returns Promise resolving to an object with the NormalizedAttachment array, or an error message on failure. */ async getAttachmentsFromArtifactId({ artifact, @@ -364,9 +393,12 @@ export class Uploader { } /** - * Gets the download URL for an artifact from the DevRev API. - * @param {string} artifactId - The ID of the artifact to download - * @returns {Promise} The download URL or undefined on error + * Requests a pre-signed download URL for an artifact from the platform. + * + * Used internally before downloading an artifact's contents back from object storage. + * + * @param artifactId - The string ID of the artifact to download. + * @returns Promise resolving to an UploaderResult wrapping the download URL string, or an error on failure. */ private async getArtifactDownloadUrl( artifactId: string @@ -391,9 +423,12 @@ export class Uploader { } /** - * Downloads an artifact file from the given URL. - * @param {string} artifactUrl - The URL to download the artifact from - * @returns {Promise} The artifact file buffer or undefined on error + * Downloads an artifact's raw bytes from a pre-signed URL. + * + * Used internally to fetch artifact contents as a Buffer for later decompression and parsing. + * + * @param artifactUrl - The string pre-signed URL to download the artifact from. + * @returns Promise resolving to an UploaderResult wrapping the file Buffer, or an error on failure. */ private async downloadArtifact( artifactUrl: string @@ -410,11 +445,12 @@ export class Uploader { } /** - * Retrieves and parses JSON objects from an artifact by artifact ID. - * @param {object} param0 - Configuration object - * @param {string} param0.artifactId - The artifact ID to download and parse - * @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped - * @returns {Promise} The parsed JSON objects or undefined on error + * Downloads an artifact by ID and parses its JSONL contents into objects. + * + * Used to read back a previously uploaded JSON batch; optionally gunzips the bytes first. + * + * @param param0 - Object with `artifactId` (string artifact ID) and optional `isGzipped` (boolean, default false) flag. + * @returns Promise resolving to an UploaderResult wrapping the parsed object or object array, or an error on failure. */ async getJsonObjectByArtifactId({ artifactId, From ec6242a70016290fbeddeb4404077cd5bedaeed1 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:18:21 +0200 Subject: [PATCH 19/30] docs: mark C7 done --- V2_PROGRESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index 6ebed941..d9e2f9e7 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -290,7 +290,7 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C4b state envelope | ☑ done | 30ba1b3. { connectorState, sdkState } envelope + v1->v2 migration shim (normalizeFetchedState). adapter.state→connector-only, new adapter.sdkState; ~28 SDK-field access sites moved. SdkState kept combined (narrowing deferred to C5). Reviewer-approved (migration cases verified). | | C5 adapter split | ☑ done | a7a877f. BaseAdapter (template emit + hooks) + ExtractionAdapter + LoadingAdapter; WorkerAdapter→union alias; processTask dispatches by mode (still single entry). worker-adapter.ts deleted; helpers→loading-adapter.helpers. Reviewer-approved (emit equivalence verified). SdkState kept combined (narrowing dropped from scope). | | C6 emit-from-return | ☑ done | 0fb6116. task/onTimeout return TaskResult; SDK maps status→event via getEventTypeForResult and emits once (emitFromResult); emit now protected/internal; processTask→processExtractionTask+processLoadingTask; loader/stream methods return TaskResult. Reviewer-approved (mapping+state-save+no-double-emit verified). NET-NEW design (no oracle). | -| C7 JSDoc | ☐ todo | Phase 2 | +| C7 JSDoc | ☑ done | d05434b. Comments-only pass over 25 files to the mappers.ts bar: v2-new code (adapters, state incl. migration shim, process-task/spawn/getEventTypeForResult mapping) + under-documented older modules (repo, uploader, attachments pool, control-protocol, install-IDM, errors, types/loading barrel). Verified every changed line is a comment; build green, lint clean. Fanned out 5 implementer subagents over disjoint file groups. | | C8 api report | ☐ todo | Phase 2 | | C9 exposure audit | ☐ todo | Phase 2, interactive | | C10 tests + baseline | ☐ todo | Phase 2 | From 4fea7552c41e6ab51d13c4d70d45ed7401e34cfe Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:45:25 +0200 Subject: [PATCH 20/30] Revert "docs(v2): JSDoc pass over public surface and v2 internals" This reverts commit d05434be0baa2cb373ba114ad3da769561aa6854. --- .../attachments-streaming-pool.interfaces.ts | 10 -- .../attachments-streaming-pool.ts | 41 +---- src/common/control-protocol.ts | 20 --- src/common/errors.ts | 9 - src/common/install-initial-domain-mapping.ts | 13 -- src/multithreading/adapters/base-adapter.ts | 41 ++--- .../adapters/extraction-adapter.ts | 117 ++----------- .../adapters/loading-adapter.helpers.ts | 26 +-- .../adapters/loading-adapter.ts | 97 +---------- src/multithreading/create-worker.ts | 10 -- src/multithreading/process-task.ts | 24 --- src/multithreading/spawn/spawn.helpers.ts | 10 +- src/multithreading/spawn/spawn.ts | 35 +--- src/repo/repo.interfaces.ts | 40 +---- src/repo/repo.ts | 26 --- src/state/base-state.ts | 75 +++------ src/state/extraction-state.ts | 40 ++--- src/state/loading-state.ts | 19 +-- src/state/state.interfaces.ts | 45 +---- src/state/state.ts | 10 +- src/types/index.ts | 8 - src/types/loading.ts | 154 ------------------ src/uploader/uploader.helpers.ts | 47 ++---- src/uploader/uploader.interfaces.ts | 21 --- src/uploader/uploader.ts | 110 +++++-------- 25 files changed, 146 insertions(+), 902 deletions(-) diff --git a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts index c4127582..9a22ea08 100644 --- a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts +++ b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts @@ -4,19 +4,9 @@ import { } from '../types'; import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; -/** - * Construction parameters used to create an AttachmentsStreamingPool. - * - * Used to supply the driving extraction adapter, the attachments to stream, the concurrency limit, and - * the connector-provided streaming function. - */ export interface AttachmentsStreamingPoolParams { - /** The ExtractionAdapter that owns sync state, timeout detection, and the processAttachment call. */ adapter: ExtractionAdapter; - /** The normalized attachments to stream to DevRev. */ attachments: NormalizedAttachment[]; - /** Optional maximum number of attachments to stream concurrently (defaults to 10 in the pool). */ batchSize?: number; - /** Connector-provided function that downloads a single attachment from the external system. */ stream: ExternalSystemAttachmentStreamingFunction; } diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index d7cf4a05..58281e16 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -8,12 +8,6 @@ import { } from '../types'; import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces'; -/** - * Concurrency-bounded pool that streams a batch of attachments from the external system to DevRev. - * - * Used during attachment extraction to download up to batchSize attachments in parallel while honoring - * timeouts, rate-limit delays, and per-attachment errors, and to track processed attachments for resumption. - */ export class AttachmentsStreamingPool { private adapter: ExtractionAdapter; private attachments: NormalizedAttachment[]; @@ -37,14 +31,6 @@ export class AttachmentsStreamingPool { this.stream = stream; } - /** - * Increments the processed counter and periodically logs progress. - * - * Used after each attachment to report progress every PROGRESS_REPORT_INTERVAL items and briefly - * yield the event loop. - * - * @returns Promise that resolves once progress has been recorded (and any brief sleep elapsed). - */ private async updateProgress() { this.totalProcessedCount++; if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) { @@ -55,13 +41,10 @@ export class AttachmentsStreamingPool { } /** - * Migrates processed-attachment state from the legacy string[] format to ProcessedAttachment[]. + * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format. * - * Used when resuming streaming so older saved state (a list of ids) is upgraded to the structured - * { id, parent_id } form before it is consulted for de-duplication. - * - * @param attachments - The persisted list to migrate, either a string[] of ids or a ProcessedAttachment[]. - * @returns Migrated array of ProcessedAttachment objects, or an empty array if the input is invalid. + * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[]) + * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid */ // eslint-disable-next-line @typescript-eslint/no-explicit-any private migrateProcessedAttachments(attachments: any): ProcessedAttachment[] { @@ -86,15 +69,6 @@ export class AttachmentsStreamingPool { return []; } - /** - * Streams every attachment in the pool, running up to batchSize streams concurrently. - * - * Used as the pool's entry point: it initializes/migrates the processed-attachments state, starts the - * initial set of worker loops, and waits for them to drain the queue or stop early on a delay. - * - * @returns Promise resolving to a ProcessAttachmentReturnType: a delay if rate-limited, an error if - * state is uninitialized, or an empty object once all attachments are processed. - */ async streamAll(): Promise { console.log( `Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.` @@ -141,15 +115,6 @@ export class AttachmentsStreamingPool { return {}; } - /** - * Runs a single worker loop that pulls and streams attachments until the queue is drained. - * - * Used as one of the concurrent workers started by streamAll: it skips already-processed attachments, - * stops on timeout or a rate-limit delay, records successes, and logs/skips per-attachment errors. - * - * @returns Promise that resolves when this worker stops, either because the queue is empty or a - * timeout/delay was detected. - */ async startPoolStreaming() { // Process attachments until the attachments array is empty while (this.attachments.length > 0) { diff --git a/src/common/control-protocol.ts b/src/common/control-protocol.ts index 8470401d..0d13be13 100644 --- a/src/common/control-protocol.ts +++ b/src/common/control-protocol.ts @@ -10,32 +10,12 @@ import { import { LoaderEventType } from '../types/loading'; import { LIBRARY_VERSION } from './constants'; -/** - * Parameters for emitting a worker control message back to the platform. - * - * Used by {@link emit} to construct and post the outgoing extractor/loader event. - */ export interface EmitInterface { - /** The incoming AirSync event currently being processed; supplies the callback URL, event context, and auth secrets. */ event: AirSyncEvent; - /** The outgoing event type to report. In v2 this value is used directly (no event-type translation). */ eventType: ExtractorEventType | LoaderEventType; - /** Optional payload describing progress, results, or error details to attach to the event. */ data?: EventData; } -/** - * Emits a worker control message to the parent/platform via the event callback URL. - * - * Used to report extraction/loading progress, completion, delays, or errors back to AirSync. - * Wraps the given event type and data into an ExtractorEvent/LoaderEvent envelope (stamped with - * the library version) and POSTs it to the callback URL with the service account authorization. - * - * @param event - The incoming AirSyncEvent providing callback URL, event context, and auth secrets. - * @param eventType - The outgoing ExtractorEventType or LoaderEventType, used directly as the event_type. - * @param data - Optional EventData payload to include in the emitted event. - * @returns Promise resolving to the AxiosResponse of the callback POST request. - */ export const emit = async ({ event, eventType, diff --git a/src/common/errors.ts b/src/common/errors.ts index cb32a8be..cd420a95 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -1,15 +1,6 @@ -/** Prefix used to namespace common error codes emitted by extractors. */ const ERROR_PREFIX = 'ERROR_CODE'; -/** Delimiter joining the error prefix and the specific error name in the encoded code. */ const ERROR_DELIMITER = '='; -/** - * Well-known error codes an extractor can report to signal common, externally-caused failure conditions. - * - * Used to communicate persistent source-system states (deletion, deactivation, missing access/permission) - * or sync-completion signals back to AirSync in a recognized, machine-readable form. Each member's value - * is the encoded string `ERROR_CODE=`. - */ export const enum ExtractionCommonError { // Indicates that the external system is permanently inactive or inaccessible. // This is used for persistent conditions (system deleted, deactivated, access permanently revoked) diff --git a/src/common/install-initial-domain-mapping.ts b/src/common/install-initial-domain-mapping.ts index b0adb6fd..5a8ec436 100644 --- a/src/common/install-initial-domain-mapping.ts +++ b/src/common/install-initial-domain-mapping.ts @@ -4,19 +4,6 @@ import { AirSyncEvent } from '../types/extraction'; import { serializeError } from '../logger/logger'; import { InitialDomainMapping } from '../types/common'; -/** - * Installs the connector's initial domain mapping into the sync. - * - * Used the first time a sync needs the mapping that describes how external record types/fields map - * to DevRev. It resolves the snap-in's import and version slugs, optionally creates the starting - * recipe blueprint, then installs the initial domain mapping (plus any additional mappings) via the - * airdrop recipe API. If no mapping JSON is provided the call is a no-op; blueprint creation failures - * are logged and the install continues without a blueprint. - * - * @param event - The AirSyncEvent providing the DevRev endpoint, service account token, and snap-in ID. - * @param initialDomainMappingJson - The InitialDomainMapping JSON containing the starting recipe blueprint and additional mappings. - * @returns Promise that resolves once the mapping has been installed (or early-returns if none is provided). - */ export async function installInitialDomainMapping( event: AirSyncEvent, initialDomainMappingJson: InitialDomainMapping diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts index 787b2407..dec92853 100644 --- a/src/multithreading/adapters/base-adapter.ts +++ b/src/multithreading/adapters/base-adapter.ts @@ -23,13 +23,10 @@ import { Uploader } from '../../uploader/uploader'; import { getEventTypeForResult } from '../spawn/spawn.helpers'; /** - * Abstract base for the worker adapters, holding state and behavior shared by - * both sync modes and owning the `emit` control-protocol flow as a template method. - * - * Used as the type passed to worker tasks; mode-specific adapters - * (`ExtractionAdapter`, `LoadingAdapter`) extend it and implement the abstract - * hooks (`beforeEmit`, `buildEmitPayload`, `afterEmit`) to inject their own - * pre-emit work and event payload shaping. + * BaseAdapter holds the state and behavior shared by both sync modes and owns + * the `emit` control-protocol flow as a template method. Mode-specific adapters + * (`ExtractionAdapter`, `LoadingAdapter`) implement the abstract hooks to inject + * their own pre-emit work and event payload shaping. * * @typeParam ConnectorState - the connector-owned state shape */ @@ -76,18 +73,10 @@ export abstract class BaseAdapter { return this.adapterState.sdkState; } - /** Per-item-type extraction scope (which item types to extract). */ get extractionScope() { return this.adapterState.extractionScope; } - /** - * Persists the current adapter state to the platform. - * - * Used to checkpoint connector and SDK state outside of an emit. - * - * @returns Promise that resolves once the state has been posted. - */ async postState() { return runWithSdkLogContext(async () => { await this.adapterState.postState(); @@ -124,15 +113,12 @@ export abstract class BaseAdapter { * Maps a {@link TaskResult} returned by a worker's task/onTimeout callback to * the phase-appropriate platform event and emits it exactly once. * - * Used as the SDK-internal bridge between the return-based connector contract + * This is the SDK-internal bridge between the return-based connector contract * and the control protocol; it is invoked by the worker driver, not by * connectors. Connectors signal outcomes by returning a `TaskResult`, never by - * calling `emit` directly. A `delay`/`error` status carries its delay seconds - * or error into the event data; a status that is illegal for a non-resumable - * phase is downgraded to an error event. + * calling `emit` directly. * - * @param result - The TaskResult status the worker reported for the current phase. - * @returns Promise that resolves once the mapped event has been emitted. + * @param result - The status the worker reported for the current phase. */ async emitFromResult(result: TaskResult): Promise { const { eventType, illegal } = getEventTypeForResult( @@ -155,17 +141,10 @@ export abstract class BaseAdapter { } /** - * Emits a single event to the platform via the template-method flow. - * - * Used as the one place that sends a control-protocol event: it runs the - * `beforeEmit` hook, persists state (except for stateless start/delete events), - * merges in the mode-specific `buildEmitPayload`, sends the event, then runs - * `afterEmit`. Guarded by `hasWorkerEmitted` so it emits at most once; any - * failure in preparation, state posting, or sending signals the worker to exit. + * Emits an event to the platform. * - * @param newEventType - The ExtractorEventType or LoaderEventType to emit. - * @param data - Optional EventData (e.g. delay or error) merged into the payload. - * @returns Promise that resolves once the emit attempt has completed. + * @param newEventType - The event type to be emitted + * @param data - The data to be sent with the event */ protected async emit( newEventType: ExtractorEventType | LoaderEventType, diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index 39cd036f..5bbbffa0 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -32,14 +32,10 @@ import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; import { BaseAdapter } from './base-adapter'; /** - * Worker adapter passed to extraction tasks, exposing the extraction surface - * (repos, artifacts, attachment streaming). + * ExtractionAdapter is the adapter passed to extraction tasks. It exposes the + * extraction surface (repos, artifacts, attachment streaming) and uploads + * pending repos and updates the extraction boundaries before emitting. * - * Used during the extraction phases; before emitting it uploads all pending - * repos and, on `AttachmentExtractionDone`, advances the sync boundaries on - * `sdkState`. - * - * @typeParam ConnectorState - the connector-owned state shape * @public */ export class ExtractionAdapter< @@ -60,12 +56,7 @@ export class ExtractionAdapter< /** * Returns whether the given item type should be extracted. - * - * Used to honor the per-item-type extraction scope; defaults to true when the - * scope is empty or the item type is not listed. - * - * @param itemType - The item type name to check. - * @returns True if the item type should be extracted. + * Defaults to true if the scope is empty or the item type is not listed. */ shouldExtract(itemType: string): boolean { const scope = this.extractionScope; @@ -74,17 +65,6 @@ export class ExtractionAdapter< return scope[itemType].extract; } - /** - * Initializes the adapter's repos from the given repo definitions. - * - * Used to set up the in-memory item buffers an extraction task pushes to; - * each repo normalizes items (except external domain metadata and SSOR - * attachments) and, on upload, records attachment artifact IDs in state and - * tracks event payload size to trigger a timeout when the SQS size threshold - * is exceeded. - * - * @param repos - The RepoInterface definitions to build repos from. - */ initializeRepos(repos: RepoInterface[]) { this.repos = repos.map((repo) => { const shouldNormalize = @@ -124,15 +104,6 @@ export class ExtractionAdapter< }); } - /** - * Looks up an initialized repo by item type. - * - * Used by extraction tasks to get the buffer they push normalized items into; - * logs an error and returns undefined when no repo matches. - * - * @param itemType - The item type name of the repo to find. - * @returns The matching Repo, or undefined if none was initialized. - */ getRepo(itemType: string): Repo | undefined { return runWithSdkLogContext(() => { const repo = this.repos.find((repo) => repo.itemType === itemType); @@ -146,32 +117,16 @@ export class ExtractionAdapter< }); } - /** Artifacts accumulated since the last emit, sent with the next event. */ get artifacts(): Artifact[] { return this._artifacts; } - /** Appends artifacts to the accumulator, de-duplicating by reference. */ set artifacts(artifacts: Artifact[]) { this._artifacts = this._artifacts .concat(artifacts) .filter((value, index, self) => self.indexOf(value) === index); } - /** - * Pre-emit hook: uploads all repos and, when extraction completes, advances - * the sync boundaries. - * - * Used as the extraction implementation of the `BaseAdapter` template hook. - * Always uploads pending repos; on `AttachmentExtractionDone` it promotes - * `lastSyncStarted` to `lastSuccessfulSyncStarted`, clears the pending worker - * boundaries, and expands `workersOldest`/`workersNewest` from the event's - * extract-from/extract-to window. - * - * @param newEventType - The event type about to be emitted. - * @returns Promise that resolves once pre-emit work is done; rejects (aborting - * the emit) if a repo upload fails. - */ protected async beforeEmit( newEventType: ExtractorEventType | LoaderEventType ): Promise { @@ -223,15 +178,6 @@ export class ExtractionAdapter< } } - /** - * Builds the extraction-specific extras merged into the emitted event payload. - * - * Used as the extraction implementation of the `BaseAdapter` template hook; - * returns the accumulated artifacts for extraction events and nothing otherwise. - * - * @param newEventType - The event type about to be emitted. - * @returns EventData carrying the accumulated artifacts for extraction events. - */ protected buildEmitPayload( newEventType: ExtractorEventType | LoaderEventType ): EventData { @@ -241,24 +187,10 @@ export class ExtractionAdapter< return isExtractionEvent ? { artifacts: this.artifacts } : {}; } - /** - * Post-emit hook: clears the accumulated artifacts after a successful emit. - * - * Used as the extraction implementation of the `BaseAdapter` template hook so - * artifacts are not re-sent on a subsequent emit. - */ protected afterEmit(): void { this.artifacts = []; } - /** - * Uploads all initialized repos and collects their resulting artifacts. - * - * Used by `beforeEmit` to flush buffered items to artifacts before an event is - * sent; throws on the first repo that fails to upload. - * - * @returns Promise that resolves once every repo has been uploaded. - */ async uploadAllRepos(): Promise { for (const repo of this.repos) { const error = await repo.upload(); @@ -269,20 +201,6 @@ export class ExtractionAdapter< } } - /** - * Streams a single attachment from the external system into a DevRev artifact - * and records the SSOR attachment mapping. - * - * Used while streaming attachments: it opens the external stream, requests an - * artifact upload URL, streams the bytes, confirms the upload, then pushes an - * `ssor_attachment` linking the external and DevRev IDs. A delay or error from - * the stream is surfaced back to the caller and the HTTP stream is destroyed. - * - * @param attachment - The NormalizedAttachment to fetch and upload. - * @param stream - The ExternalSystemAttachmentStreamingFunction that opens the source stream. - * @returns Promise with undefined on success, or a ProcessAttachmentReturnType - * carrying a delay or error. - */ async processAttachment( attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction @@ -398,12 +316,8 @@ export class ExtractionAdapter< } /** - * Destroys an HTTP response stream to prevent memory leaks. - * - * Used to release an open attachment source stream when uploading fails; - * swallows any error raised while destroying. - * - * @param httpStream - The AxiosResponse stream to destroy. + * Destroys a stream to prevent memory leaks. + * @param httpStream - The axios response stream to destroy */ private destroyHttpStream(httpStream: AxiosResponse): void { try { @@ -420,20 +334,11 @@ export class ExtractionAdapter< } /** - * Streams all pending attachments to the DevRev platform and returns the phase - * outcome. - * - * Used as the entry point for the attachment-streaming phase: it iterates the - * attachments-metadata artifact IDs recorded in state, fetches each batch, and - * streams them either through caller-provided processors or the default - * streaming pool (with batch size clamped to 1..50). Returns `delay` on a - * rate limit, `error` on failure, `progress` on timeout (to resume in a fresh - * invocation), and `success` once every artifact ID is processed. - * - * @param stream - The ExternalSystemAttachmentStreamingFunction used to open each source stream. - * @param processors - Optional ExternalSystemAttachmentProcessors (reducer + iterator) overriding the default pool. - * @param batchSize - Number of attachments to stream concurrently; defaults to 1, clamped to 1..50. - * @returns Promise with the TaskResult describing the phase outcome. + * Streams the attachments to the DevRev platform. + * The attachments are streamed to the platform and the artifact information is returned. + * @param params - The parameters to stream the attachments + * @returns The response object containing the ssorAttachment artifact information + * or error information if there was an error */ async streamAttachments({ stream, diff --git a/src/multithreading/adapters/loading-adapter.helpers.ts b/src/multithreading/adapters/loading-adapter.helpers.ts index e7f31876..ecab2de1 100644 --- a/src/multithreading/adapters/loading-adapter.helpers.ts +++ b/src/multithreading/adapters/loading-adapter.helpers.ts @@ -6,15 +6,10 @@ import { } from '../../types/loading'; /** - * Builds the ordered list of files to load from a stats file. - * - * Used by the loading adapter to filter the stats file down to the supported - * item types, order the entries to match that item-type order, and shape each - * into a FileToLoad with progress fields reset. - * - * @param supportedItemTypes - The supported item type names, in desired load order. - * @param statsFile - The StatsFileObject entries describing available files. - * @returns The FileToLoad entries to process, ordered by supported item type. + * Gets the files to load for the loader. + * @param {string[]} supportedItemTypes - The supported item types + * @param {StatsFileObject[]} statsFile - The stats file + * @returns {FileToLoad[]} The files to load */ export function getFilesToLoad({ supportedItemTypes, @@ -55,15 +50,10 @@ export function getFilesToLoad({ } /** - * Merges a report into the accumulated loader reports. - * - * Used to keep one running report per item type: when a report for the same - * item type already exists its created/updated/failed counts are summed, - * otherwise the report is appended. - * - * @param loaderReports - The existing LoaderReport accumulator (mutated in place). - * @param report - The LoaderReport to merge in. - * @returns The updated LoaderReport accumulator. + * Adds a report to the loader report. + * @param {LoaderReport[]} loaderReports - The loader reports + * @param {LoaderReport} report - The report to add + * @returns {LoaderReport[]} The updated loader reports */ export function addReportToLoaderReport({ loaderReports, diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts index 70c4fe94..49fdd2bf 100644 --- a/src/multithreading/adapters/loading-adapter.ts +++ b/src/multithreading/adapters/loading-adapter.ts @@ -36,14 +36,9 @@ import { } from './loading-adapter.helpers'; /** - * Worker adapter passed to loading tasks, exposing the loading surface - * (item/attachment loading, mappers, loader reports). + * LoadingAdapter is the adapter passed to loading tasks. It exposes the loading + * surface (item/attachment loading, mappers, loader reports). * - * Used during the loading phases to push transformed DevRev items into the - * external system, maintaining sync mapper records and accumulating per-item-type - * loader reports across emits. - * - * @typeParam ConnectorState - the connector-owned state shape * @public */ export class LoadingAdapter< @@ -67,43 +62,22 @@ export class LoadingAdapter< }); } - /** Per-item-type loader reports accumulated across loads, sent with each emit. */ get reports(): LoaderReport[] { return this.loaderReports; } - /** Artifact IDs of transformer files that have been fully loaded. */ get processedFiles(): string[] { return this._processedFiles; } - /** Mappers client for reading/writing sync mapper records. */ get mappers(): Mappers { return this._mappers; } - /** - * Pre-emit hook for loading; intentionally a no-op. - * - * Used as the loading implementation of the `BaseAdapter` template hook; - * loading has no repos to upload and no extraction boundaries to advance. - * - * @returns Promise that resolves immediately. - */ protected async beforeEmit(): Promise { // Loading has no pre-emit work (no repos, no extraction boundaries). } - /** - * Builds the loading-specific extras merged into the emitted event payload. - * - * Used as the loading implementation of the `BaseAdapter` template hook; - * returns the accumulated reports and processed files for loader events and - * nothing otherwise. - * - * @param newEventType - The event type about to be emitted. - * @returns EventData carrying loader reports and processed files for loader events. - */ protected buildEmitPayload( newEventType: ExtractorEventType | LoaderEventType ): EventData { @@ -118,29 +92,10 @@ export class LoadingAdapter< : {}; } - /** - * Post-emit hook for loading; intentionally a no-op. - * - * Used as the loading implementation of the `BaseAdapter` template hook; - * loading keeps its accumulated reports and processed files across emits. - */ protected afterEmit(): void { // Loading keeps its accumulated reports/processed files across emits. } - /** - * Loads all supported item types into the external system and returns the - * phase outcome. - * - * Used as the entry point for the data-loading phase: on `StartLoadingData` it - * resolves the batches of transformer files to load from the stats file, then - * for each file loads every item via `loadItem`, tracking progress per line so - * a fresh invocation can resume. Returns `delay` on a rate limit, `progress` - * on timeout, `error` on failure, and `success` once all files are processed. - * - * @param itemTypesToLoad - The ItemTypeToLoad definitions (create/update handlers per item type). - * @returns Promise with the TaskResult describing the phase outcome. - */ async loadItemTypes({ itemTypesToLoad, }: ItemTypesToLoadParams): Promise { @@ -261,17 +216,6 @@ export class LoadingAdapter< }); } - /** - * Resolves the ordered list of transformer files to load for the given item - * types. - * - * Used by `loadItemTypes`/`loadAttachments` to turn the event's stats file - * into `FileToLoad` batches, filtered to the supported item types; returns an - * empty list when there is no stats file, it cannot be fetched, or it is empty. - * - * @param supportedItemTypes - The item type names to include, in load order. - * @returns Promise with the FileToLoad batches to process. - */ async getLoaderBatches({ supportedItemTypes, }: { @@ -304,18 +248,6 @@ export class LoadingAdapter< }); } - /** - * Loads attachments into the external system and returns the phase outcome. - * - * Used as the entry point for the attachment-loading phase: on - * `StartLoadingAttachments` it resolves the attachment transformer files to - * load, then creates each attachment via `loadAttachment`, tracking progress - * per line for resumption. Returns `delay` on a rate limit, `progress` on - * timeout, `error` on failure, and `success` once all files are processed. - * - * @param create - The ExternalSystemLoadingFunction that creates an attachment in the external system. - * @returns Promise with the TaskResult describing the phase outcome. - */ async loadAttachments({ create, }: { @@ -412,19 +344,6 @@ export class LoadingAdapter< }); } - /** - * Loads a single item into the external system, creating or updating as needed. - * - * Used per item by `loadItemTypes`: it looks up the sync mapper record by - * DevRev target ID and calls the item type's `update` handler; if no mapper - * record exists (404) it falls back to the `create` handler. On success it - * creates or updates the sync mapper record and reports the action; a rate - * limit surfaces a delay and other failures are reported as failed. - * - * @param item - The ExternalSystemItem to load. - * @param itemTypeToLoad - The ItemTypeToLoad providing the create/update handlers. - * @returns Promise with a LoadItemResponse carrying a report, a rateLimit delay, or an error. - */ async loadItem({ item, itemTypeToLoad, @@ -631,18 +550,6 @@ export class LoadingAdapter< }); } - /** - * Creates a single attachment in the external system. - * - * Used per attachment by `loadAttachments`: it calls the `create` handler and, - * on success, creates a sync mapper record linking the new external ID to the - * attachment's reference ID and reports it as created. A rate limit surfaces a - * delay; a missing ID is reported as failed (attachments are create-only). - * - * @param item - The ExternalSystemAttachment to create. - * @param create - The ExternalSystemLoadingFunction that creates the attachment. - * @returns Promise with a LoadItemResponse carrying a report or a rateLimit delay. - */ async loadAttachment({ item, create, diff --git a/src/multithreading/create-worker.ts b/src/multithreading/create-worker.ts index 72b9655b..abd35034 100644 --- a/src/multithreading/create-worker.ts +++ b/src/multithreading/create-worker.ts @@ -2,16 +2,6 @@ import { isMainThread, Worker } from 'node:worker_threads'; import { WorkerData, WorkerEvent } from '../types/workers'; -/** - * Creates a Node worker thread that runs the snap-in's task worker script. - * - * Used by `spawn` to launch the off-main-thread worker that processes an - * extraction/loading event; the promise settles once the worker comes online - * so the caller can wire up timeouts and lifecycle handling. - * - * @param workerData - The data of type WorkerData passed to the worker thread (event, initial state, options, etc.). - * @returns A Promise that resolves with the online Worker instance, or rejects with the Error if the worker fails to start or is itself a worker thread. - */ async function createWorker( workerData: WorkerData ): Promise { diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 6db0d7c0..a43debbb 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -30,12 +30,6 @@ import { LoadingAdapter } from './adapters/loading-adapter'; * If `onTimeout` is omitted, the SDK emits a phase-appropriate default on * timeout: `progress` (resumable phases) or `error` (non-resumable phases) is * handled by the status->event mapping when we emit a `progress` result. - * - * @param buildAdapter - Factory that constructs the typed adapter for this worker. - * @param params - The task hooks of type ProcessTaskInterface. - * @param params.task - The worker's main task; receives the adapter and resolves to a TaskResult. - * @param params.onTimeout - Optional callback run on soft timeout when nothing has emitted yet; resolves to a TaskResult. - * @returns A Promise that resolves once the result has been emitted and the worker process exits. */ async function runWorkerTask>( buildAdapter: () => Promise, @@ -89,15 +83,6 @@ async function runWorkerTask>( * Entry point for an extraction worker. Builds an {@link ExtractionAdapter} and * runs the provided task against it. * - * Used as the worker-script entry the snap-in calls inside an extraction worker - * thread; returns immediately on the main thread so the same module can be - * imported there safely. - * - * @param params - The task hooks of type ProcessTaskInterface for an ExtractionAdapter. - * @param params.task - The extraction task; receives the adapter and resolves to a TaskResult. - * @param params.onTimeout - Optional callback run on soft timeout; resolves to a TaskResult. - * @returns Nothing; emission and process exit are handled by the shared driver. - * * @public */ export function processExtractionTask({ @@ -138,15 +123,6 @@ export function processExtractionTask({ * Entry point for a loading worker. Builds a {@link LoadingAdapter} and runs the * provided task against it. * - * Used as the worker-script entry the snap-in calls inside a loading worker - * thread; returns immediately on the main thread so the same module can be - * imported there safely. - * - * @param params - The task hooks of type ProcessTaskInterface for a LoadingAdapter. - * @param params.task - The loading task; receives the adapter and resolves to a TaskResult. - * @param params.onTimeout - Optional callback run on soft timeout; resolves to a TaskResult. - * @returns Nothing; emission and process exit are handled by the shared driver. - * * @public */ export function processLoadingTask({ diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index 30a00e15..1f52764c 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -59,15 +59,7 @@ export function getEventTypeForResult( /** * Per-phase outgoing event types, keyed by the incoming {@link EventType}. - * - * Each entry maps a phase to its outgoing events: every phase has a `done` - * (success) and `error` event. `resumable` phases (data/attachment extraction, - * data/attachment loading) additionally define `progress` and `delayed` events - * and accept all four statuses. Non-resumable phases (external sync units, - * metadata, state deletions) omit `progress`/`delayed`; a `progress`/`delay` - * status there is illegal and {@link getEventTypeForResult} collapses it to the - * `error` event while flagging it. Event types absent from this partial map are - * treated as unrecognized. + * `resumable` phases define progress/delayed events; non-resumable ones do not. */ const EVENT_PHASE_MAP: Partial< Record< diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index f31ecba7..02176ff0 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -25,17 +25,6 @@ import { getNoScriptEventType, } from './spawn.helpers'; -/** - * Resolves the default worker script path for an incoming event type. - * - * Used by `spawn` to pick which built-in worker (external sync units, metadata, - * data/attachment extraction, data/attachment loading) to run when the caller - * has not supplied an explicit `workerPath` or override. - * - * @param event - The AirSync event whose `payload.event_type` selects the worker. - * @param workerBasePath - The base directory string the resolved relative worker path is appended to. - * @returns The full worker script path string, or null if the event type has no matching built-in worker. - */ function getWorkerPath({ event, workerBasePath, @@ -74,14 +63,12 @@ function getWorkerPath({ * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker. * The class provides utilities to emit control events to the platform and exit the worker gracefully. * In case of lambda timeout, the class emits a lambda timeout event to the platform. - * @param options - The options of type SpawnFactoryInterface used to launch the worker. - * @param options.event - The AirSync event object received from the platform. - * @param options.initialState - The initial connector state handed to the worker. - * @param options.initialDomainMapping - The initial domain mapping handed to the worker. - * @param options.options - Optional SDK behavior overrides (timeout, local development, worker path overrides, etc.). - * @param options.workerPath - Optional explicit path to the worker script; takes precedence over overrides and the default resolver. - * @param options.baseWorkerPath - The base path for the worker files, usually `__dirname`. - * @returns A Promise that resolves once the worker finishes (or a no-script default event is emitted), or rejects if the worker fails to start. + * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class + * @param {AirSyncEvent} options.event - The event object received from the platform + * @param {object} options.initialState - The initial state of the adapter + * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath + * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname` + * @returns {Promise} - A new instance of Spawn class */ export async function spawn({ event, @@ -167,16 +154,6 @@ export async function spawn({ } } -/** - * Manages the lifecycle of a spawned worker thread for a single event. - * - * Used by `spawn` to supervise the worker: it arms a soft timeout (asks the - * worker to exit gracefully) and a hard timeout (terminates a stuck worker), - * relays the worker's log messages to the main thread, tracks whether the - * worker has already emitted an event, periodically logs memory usage, and on - * worker exit clears the timers and resolves the spawn promise -- emitting a - * timeout error event if the worker exited without emitting one itself. - */ export class Spawn { private event: AirSyncEvent; private alreadyEmitted: boolean; diff --git a/src/repo/repo.interfaces.ts b/src/repo/repo.interfaces.ts index 583f71e9..f790be2a 100644 --- a/src/repo/repo.interfaces.ts +++ b/src/repo/repo.interfaces.ts @@ -4,84 +4,54 @@ import { AirSyncEvent } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; /** - * Describes a repo configuration that stores and uploads extracted data of one item type. - * - * Used to declare which item type a repo holds and how its raw records should be normalized. + * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data. */ export interface RepoInterface { - /** The item type the repo buffers and uploads. */ itemType: string; - /** Optional normalizer turning a raw record into a NormalizedItem or NormalizedAttachment. */ normalize?: (record: object) => NormalizedItem | NormalizedAttachment; - /** Optional worker adapter options that override defaults (e.g. batch size). */ overridenOptions?: WorkerAdapterOptions; } /** - * Construction parameters used to create a Repo instance. - * - * Used to wire a repo to its triggering event, item type, normalizer, upload callback, and options. + * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo. */ export interface RepoFactoryInterface { - /** The AirSync event that drives the extraction and supplies platform credentials. */ event: AirSyncEvent; - /** The item type the repo buffers and uploads. */ itemType: string; - /** Optional normalizer turning a raw record into a NormalizedItem or NormalizedAttachment. */ normalize?: (record: object) => NormalizedItem | NormalizedAttachment; - /** Callback invoked with each Artifact once it has been uploaded. */ onUpload: (artifact: Artifact) => void; - /** Optional worker adapter options that override defaults (e.g. batch size). */ options?: WorkerAdapterOptions; } /** - * An external system item after normalization into the shape AirSync expects. - * - * Used as the uploaded representation of a non-attachment record. + * NormalizedItem is an interface of item after normalization. */ export interface NormalizedItem { - /** External system identifier of the item. */ id: string; - /** ISO timestamp of when the item was created in the external system. */ created_date: string; - /** ISO timestamp of when the item was last modified in the external system. */ modified_date: string; - /** Normalized field values of the item. */ data: object; } /** - * An external system attachment after normalization into the shape AirSync expects. - * - * Used as the uploaded metadata for an attachment whose binary is streamed separately. + * NormalizedAttachment is an interface of attachment after normalization. */ export interface NormalizedAttachment { - /** Source URL the attachment binary can be downloaded from. */ url: string; - /** External system identifier of the attachment. */ id: string; - /** Name of the attached file. */ file_name: string; - /** External system identifier of the item the attachment belongs to. */ parent_id: string; - /** Optional external system identifier of the attachment's author. */ author_id?: string; - /** Whether the attachment is embedded inline (e.g. in rich text) rather than a standalone file. */ inline?: boolean; - /** Optional MIME type of the attachment. */ content_type?: string; // This should be a string, but it was a number in the past. Due to backwards // compatibility we are keeping it also as a number. - /** Optional external identifier of the parent's parent; kept as number for backwards compatibility. */ grand_parent_id?: number | string; } /** - * A raw, un-normalized record extracted from the external system. - * - * Used as the input to a repo's normalize function before items are uploaded. + * Item is an interface that defines the structure of an item. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export type Item = Record; diff --git a/src/repo/repo.ts b/src/repo/repo.ts index e694b584..d119eea8 100644 --- a/src/repo/repo.ts +++ b/src/repo/repo.ts @@ -16,12 +16,6 @@ import { RepoFactoryInterface, } from './repo.interfaces'; -/** - * In-memory buffer that accumulates normalized items of a single item type during extraction. - * - * Used to batch pushed items (ARTIFACT_BATCH_SIZE per batch), normalize them, and upload them as - * artifacts to the DevRev platform, firing the onUpload callback for each uploaded artifact. - */ export class Repo { readonly itemType: string; private items: (NormalizedItem | NormalizedAttachment | Item)[]; @@ -47,20 +41,10 @@ export class Repo { this.uploadedArtifacts = []; } - /** Returns the items currently buffered in the repo (not yet uploaded). */ getItems(): (NormalizedItem | NormalizedAttachment | Item)[] { return this.items; } - /** - * Uploads a batch of items (or all buffered items) as a single artifact. - * - * Used to flush buffered items to the DevRev platform; on success the artifact is passed to - * onUpload and recorded in uploadedArtifacts. When no explicit batch is given the buffer is cleared. - * - * @param batch - Optional explicit array of NormalizedItem, NormalizedAttachment, or Item to upload; defaults to all buffered items. - * @returns Promise that resolves to void on success, or an ErrorRecord describing the upload failure. - */ async upload( batch?: (NormalizedItem | NormalizedAttachment | Item)[] ): Promise { @@ -100,16 +84,6 @@ export class Repo { } } - /** - * Normalizes and buffers items, uploading full batches as they accumulate. - * - * Used by connectors to feed extracted items into the repo; items are normalized (unless the item - * type is external domain metadata or SSOR attachments) and any complete batches of batchSize are - * uploaded immediately, leaving the remainder buffered for a later flush. - * - * @param items - Array of raw Item records to normalize and buffer. - * @returns Promise that resolves to true when items were buffered/uploaded successfully, or false if a batch upload threw. - */ async push(items: Item[]): Promise { let recordsToPush: (NormalizedItem | NormalizedAttachment | Item)[]; diff --git a/src/state/base-state.ts b/src/state/base-state.ts index 1b860e86..06db8959 100644 --- a/src/state/base-state.ts +++ b/src/state/base-state.ts @@ -17,11 +17,10 @@ import { } from './state.interfaces'; /** - * Abstract base owning the adapter state lifecycle shared by every sync mode. + * BaseState owns the state lifecycle shared by every sync mode: connector vs. + * SDK state separation, fetch/init/post against the platform, the v1->v2 + * migration shim, and the snap-in-version-gated initial domain mapping install. * - * Used to keep connector-owned state separate from SDK bookkeeping, fetch/init/ - * post the persisted state against the platform, run the v1->v2 migration shim, - * and install the initial domain mapping gated on the snap-in version. * Mode-specific subclasses (`ExtractionState`, `LoadingState`) seed the * SDK-owned portion of the state and add mode-specific setup in their factories. * @@ -70,21 +69,15 @@ export abstract class BaseState { this._sdkState = value; } - /** The per-sync-unit extraction scope (object types to extract), loaded alongside state. */ get extractionScope(): ExtractionScope { return this._extractionScope; } /** - * Installs the initial domain mapping when the version in state is stale. - * - * Used by all modes (so a loading run still installs the mapping if extraction - * has not) to (re)install whenever `sdkState.snapInVersionId` is absent or - * differs from the event context's snap-in version; on success the new version - * is recorded in state. A missing mapping or install error fails the worker. - * - * @param initialDomainMapping - The initial domain mapping of type InitialDomainMapping passed to the spawn function; required when an install is needed - * @returns Promise that resolves once the mapping is installed or the install is skipped + * Installs the initial domain mapping when the snap-in version in state does + * not match the version in the event context. Shared by all modes so that a + * loading run still installs the mapping if extraction has not done so. + * @param initialDomainMapping The initial domain mapping passed to spawn */ async installInitialDomainMappingIfNeeded( initialDomainMapping?: InitialDomainMapping @@ -129,17 +122,13 @@ export abstract class BaseState { } /** - * Initializes this adapter's state from persisted state, or seeds it on first run. + * Initializes the state for this adapter instance by fetching from API + * or creating an initial state if none exists (404). * - * Used at worker start to load and normalize state: it fetches the persisted - * blob, parses it, and runs `normalizeFetchedState` so both the v2 - * `{ connectorState, sdkState }` envelope and a legacy flat v1 blob are - * accepted (the latter migrated on read). It also restores the extraction - * scope. On a 404 it seeds the initial state and persists the v2 envelope; - * any other failure fails the worker. - * - * @param initialState - The initial connector state of type ConnectorState provided by the spawn function, used when no state exists yet - * @returns Promise that resolves once state has been loaded or seeded + * Reads both the v2 `{ connectorState, sdkState }` envelope and a legacy flat + * v1 blob (connector keys merged with SDK keys), migrating the latter on read. + * Always persists the v2 envelope going forward. + * @param initialState The initial connector state provided by the spawn function */ async init(initialState: ConnectorState): Promise { try { @@ -192,20 +181,13 @@ export abstract class BaseState { } /** - * Normalizes parsed on-disk state into the `{ connectorState, sdkState }` envelope, migrating legacy v1 state. - * - * Used as the v1->v2 migration shim so older snap-ins keep working after the - * state split. Behavior by shape of the parsed input: - * - v2 envelope (`{ connectorState, sdkState }`): used as-is, with `sdkState` - * merged over the mode's initial SDK state to backfill newly added fields. - * - Legacy v1 flat blob: top-level keys present in `V1_SDK_STATE_KEYS` are - * split into `sdkState`, everything else becomes connector state. - * - Malformed envelope (one of `connectorState`/`sdkState` present, the other - * missing) or non-object input: throws. + * Normalizes a parsed on-disk state into the `{ connectorState, sdkState }` + * envelope, migrating a legacy flat v1 blob if needed. * - * @param parsed - The JSON-parsed persisted state of unknown shape (v2 envelope or legacy v1 flat blob) - * @returns The split state as `{ connectorState, sdkState }`, with `sdkState` merged over the initial SDK state - * @throws Error when the input is not an object or is a malformed envelope + * - v2 envelope (`{ connectorState, sdkState }`): used as-is. + * - v1 flat blob: SDK-owned keys (`V1_SDK_STATE_KEYS`) split into `sdkState`, + * everything else becomes connector state. + * - Malformed envelope (one side present, the other missing) fails loud. */ private normalizeFetchedState(parsed: unknown): { connectorState: ConnectorState; @@ -249,14 +231,9 @@ export abstract class BaseState { } /** - * Persists the adapter state to the platform. - * - * Used to checkpoint progress: wraps the current connector and SDK state into - * the v2 `{ connectorState, sdkState }` envelope, serializes it, and posts it. - * A serialization or request failure fails the worker. - * - * @param state - Optional connector state of type ConnectorState to set and persist; when omitted the current `this.state` is used - * @returns Promise that resolves once the state has been persisted + * Updates the state of the adapter by posting to API. + * Persists the v2 `{ connectorState, sdkState }` envelope. + * @param {object} state - The connector state to be updated */ async postState(state?: ConnectorState) { const url = this.workerUrl + '.update'; @@ -319,12 +296,8 @@ export abstract class BaseState { } /** - * Fetches the raw persisted adapter state from the platform. - * - * Used by `init` to read the stored state before normalization; returns the - * raw, still-stringified payload without parsing or migrating it. - * - * @returns Promise resolving to `{ state, objects }`, where `state` is the stringified state blob and `objects` is the optional stringified extraction scope + * Fetches the state of the adapter from API. + * @return The raw state data from API */ async fetchState(): Promise<{ state: string; objects?: string }> { console.log( diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts index 49bd185d..b78dbbc0 100644 --- a/src/state/extraction-state.ts +++ b/src/state/extraction-state.ts @@ -10,13 +10,10 @@ import { BaseState } from './base-state'; import { extractionSdkState, StateInterface } from './state.interfaces'; /** - * Per-mode adapter state for extraction workers. - * - * Used to seed the extraction SDK state (extraction-window boundaries + - * attachments bookkeeping) on top of the shared lifecycle provided by - * `BaseState`, and to resolve the incremental extraction window for each event. - * - * @typeParam ConnectorState - the connector-owned state shape + * ExtractionState is the per-mode state for extraction workers. It seeds the + * extraction SDK state (extraction boundaries + attachments bookkeeping) on top + * of the shared lifecycle provided by `BaseState` and adds extraction-window + * resolution. */ export class ExtractionState extends BaseState { constructor(params: StateInterface) { @@ -24,19 +21,13 @@ export class ExtractionState extends BaseState { } /** - * Computes the incremental extraction window and writes `extract_from`/`extract_to` onto the event context. + * Resolves the extraction window onto the event context. * - * Used so every extraction phase shares one consistent time window, read from - * the SDK-owned boundary fields in `this.sdkState`. Behavior by event type: - * - StartExtractingData: stamp `lastSyncStarted` if not already set. - * - StartExtractingMetadata: resolve fresh from the TimeValue objects in the - * event context and cache them as pending boundaries (always overwrite). - * - All other events: reuse the pending boundaries cached during - * StartExtractingMetadata. - * Finally validates that `extract_from` is older than `extract_to`, failing - * the worker if the platform supplied an inverted window. - * - * @returns void; mutates the event context and `this.sdkState` in place + * On StartExtractingData: stamp `lastSyncStarted` if not already set. + * On StartExtractingMetadata: resolve fresh from the TimeValue objects in the + * event context and cache them as pending boundaries (always overwrite). + * On all other events: reuse the pending boundaries cached during + * StartExtractingMetadata. Finally, validate that extract_from < extract_to. */ resolveExtractionWindow(): void { const sdkState = this.sdkState; @@ -131,14 +122,9 @@ export class ExtractionState extends BaseState { /** * Creates and initializes an `ExtractionState` for an extraction worker. * - * Used by the state dispatcher to build extraction-mode state. The initial state - * is deep-cloned to avoid mutating the caller's object; for non-stateless events - * this fetches persisted state, installs the initial domain mapping if the - * snap-in version changed, then resolves the extraction window (time-value - * resolution + pending boundary reuse) and validates it. - * - * @param params - The state factory parameters of type StateInterface (event, initial connector state, optional domain mapping and worker options) - * @returns Promise resolving to the initialized ExtractionState + * For non-stateless events this fetches persisted state, installs the initial + * domain mapping if the snap-in version changed, then resolves the extraction + * window (time-value resolution + pending boundary reuse) and validates it. */ export async function createExtractionState({ event, diff --git a/src/state/loading-state.ts b/src/state/loading-state.ts index 5b03ecb0..151f5bd9 100644 --- a/src/state/loading-state.ts +++ b/src/state/loading-state.ts @@ -4,13 +4,9 @@ import { BaseState } from './base-state'; import { loadingSdkState, StateInterface } from './state.interfaces'; /** - * Per-mode adapter state for loading workers. - * - * Used to seed the loading SDK state (files-to-load bookkeeping) on top of the - * shared lifecycle provided by `BaseState`. Loading has no extraction-window - * resolution. - * - * @typeParam ConnectorState - the connector-owned state shape + * LoadingState is the per-mode state for loading workers. It seeds the loading + * SDK state (files-to-load bookkeeping) on top of the shared lifecycle provided + * by `BaseState`. Loading has no extraction-window resolution. */ export class LoadingState extends BaseState { constructor(params: StateInterface) { @@ -21,13 +17,8 @@ export class LoadingState extends BaseState { /** * Creates and initializes a `LoadingState` for a loading worker. * - * Used by the state dispatcher to build loading-mode state. The initial state is - * deep-cloned to avoid mutating the caller's object; for non-stateless events - * this fetches persisted state and installs the initial domain mapping if the - * snap-in version changed. - * - * @param params - The state factory parameters of type StateInterface (event, initial connector state, optional domain mapping and worker options) - * @returns Promise resolving to the initialized LoadingState + * For non-stateless events this fetches persisted state and installs the + * initial domain mapping if the snap-in version changed. */ export async function createLoadingState({ event, diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index e3c45cae..9e105772 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -3,13 +3,6 @@ import { AirSyncEvent } from '../types/extraction'; import { FileToLoad } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; -/** - * The SDK-owned portion of the persisted adapter state. - * - * Used to hold bookkeeping the SDK manages itself (extraction-window boundaries, - * attachments/files progress, installed snap-in version) separately from - * connector-owned state, so SDK internals never collide with connector keys. - */ export interface SdkState { /** * @deprecated Use extract_from and extract_to from the event context instead, @@ -27,15 +20,12 @@ export interface SdkState { /** The pending (not yet committed) newest extraction boundary (ISO 8601 timestamp). * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */ pendingWorkersNewest?: string; - /** The committed oldest point of extraction (ISO 8601 timestamp). */ + /** The oldest point of extraction (ISO 8601 timestamp). */ workersOldest?: string; - /** The committed newest point of extraction (ISO 8601 timestamp). */ + /** The newest point of extraction (ISO 8601 timestamp). */ workersNewest?: string; - /** Attachments-extraction bookkeeping (artifact ids, progress cursor). Extraction mode only. */ toDevRev?: ToDevRev; - /** Loading bookkeeping (files still to load into DevRev). Loading mode only. */ fromDevRev?: FromDevRev; - /** The snap-in version id whose initial domain mapping is installed; drives reinstall on change. */ snapInVersionId?: string; } @@ -59,12 +49,6 @@ export interface AdapterStateEnvelope { sdkState: SdkState; } -/** - * SDK-owned attachments-extraction state (external system -> DevRev direction). - * - * Used to track which attachment artifacts have been streamed and how far the - * attachments phase has progressed so it can resume after a timeout. - */ export interface ToDevRev { attachmentsMetadata: { artifactIds: string[]; @@ -81,23 +65,10 @@ export interface ProcessedAttachment { parent_id: string; } -/** - * SDK-owned loading state (DevRev -> external system direction). - * - * Used to track which files still need to be loaded into the external system so - * the loading phase can resume after a timeout. - */ export interface FromDevRev { filesToLoad: FileToLoad[]; } -/** - * Constructor/factory parameters for building an adapter state instance. - * - * Used by `createAdapterState` and the per-mode factories to carry the AirSync - * event, the connector's seed state, and the optional initial domain mapping and - * worker options. - */ export interface StateInterface { event: AirSyncEvent; initialState: ConnectorState; @@ -105,12 +76,6 @@ export interface StateInterface { options?: WorkerAdapterOptions; } -/** - * The initial SDK state seeded for extraction-mode workers. - * - * Used by `ExtractionState` as the baseline `sdkState` (extraction-window - * boundaries plus attachments bookkeeping) before any persisted state is merged in. - */ export const extractionSdkState = { lastSyncStarted: '', lastSuccessfulSyncStarted: '', @@ -128,12 +93,6 @@ export const extractionSdkState = { }, }; -/** - * The initial SDK state seeded for loading-mode workers. - * - * Used by `LoadingState` as the baseline `sdkState` (files-to-load bookkeeping) - * before any persisted state is merged in. - */ export const loadingSdkState = { snapInVersionId: '', fromDevRev: { diff --git a/src/state/state.ts b/src/state/state.ts index bcb48dcb..ab7dbb5c 100644 --- a/src/state/state.ts +++ b/src/state/state.ts @@ -10,13 +10,11 @@ export { ExtractionState, createExtractionState } from './extraction-state'; export { LoadingState, createLoadingState } from './loading-state'; /** - * Creates and initializes the adapter state for the current worker. + * Creates and initializes the adapter state for the current worker, dispatching + * to the extraction or loading state based on the event's sync mode. * - * Used as the single entry point that dispatches to either `createLoadingState` - * or `createExtractionState` based on `event.payload.event_context.mode`. - * - * @param params - The state factory parameters of type StateInterface (event, initial state, optional domain mapping and worker options) - * @returns Promise resolving to the initialized mode-specific state (LoadingState when mode is LOADING, otherwise ExtractionState) + * @param params The state factory parameters (event, initial state, options) + * @returns The initialized mode-specific state */ export async function createAdapterState( params: StateInterface diff --git a/src/types/index.ts b/src/types/index.ts index 9be77e32..f98c7251 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,11 +1,3 @@ -/** - * Public types barrel for the SDK. - * - * Aggregates and re-exports the commonly used types across the SDK domains — common, extraction, - * loading, repo, state, uploader, mappers, and external domain metadata — so consumers can import - * them from a single entry point. - */ - // Common export { AdapterUpdateParams, diff --git a/src/types/loading.ts b/src/types/loading.ts index 7c58c46b..37d2d305 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -2,50 +2,22 @@ import { Mappers } from '../mappers/mappers'; import { ErrorRecord } from './common'; import { AirSyncEvent } from './extraction'; -/** - * Describes a single prepared data file as listed in the loading stats manifest. - * - * Used during loading to enumerate the artifact files produced by extraction, along with their - * item type and record count, so the loader knows what is available to process. - */ export interface StatsFileObject { - /** Identifier of the artifact/file. */ id: string; - /** External item type contained in the file (e.g. the record type being loaded). */ item_type: string; - /** Name of the file. */ file_name: string; - /** Number of records in the file, as a string. */ count: string; } -/** - * Loader-side view of a file to be loaded, tracking its processing progress. - * - * Used to drive and resume loading of a single data file: it records how many lines exist, the next - * line to process, and whether the file has been fully consumed. - */ export interface FileToLoad { - /** Identifier of the artifact/file. */ id: string; - /** Name of the file. */ file_name: string; - /** External item type contained in the file. */ itemType: string; - /** Total number of records in the file. */ count: number; - /** Index of the next line/record to process; used to resume loading across batches. */ lineToProcess: number; - /** Whether all records in the file have been loaded. */ completed: boolean; } -/** - * An attachment to be loaded into the external system, with its source metadata and parent links. - * - * Used by attachment loading to describe a single file (location, type, size, validity window, - * audit fields) and the DevRev/external parent it belongs to. - */ export interface ExternalSystemAttachment { reference_id: DonV2; parent_type: string; @@ -63,28 +35,13 @@ export interface ExternalSystemAttachment { grand_parent_id?: string; } -/** - * A single item to be loaded into the external system. - * - * Used during loading to carry the DevRev (and optional external) identifiers, audit timestamps, - * and the system-specific payload for one record. - * - * Note: this interface is declared twice in this file with identical members (TypeScript merges - * the declarations); the duplicate is redundant — see report. - */ export interface ExternalSystemItem { - /** Identifiers linking this item to DevRev and, when known, the external system. */ id: { - /** DevRev object identifier (DON). */ devrev: DonV2; - /** External system identifier, present once the item exists in the external system. */ external?: string; }; - /** Creation timestamp of the item. */ created_date: string; - /** Last-modified timestamp of the item. */ modified_date: string; - /** System-specific record payload. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any data: any; } @@ -100,195 +57,84 @@ export interface ExternalSystemItem { data: any; } -/** - * Arguments passed to an external-system loading function for a single item. - * - * Used to give create/update handlers the item to load, the ID mappers for resolving DevRev <-> external - * references, and the current AirSync event for auth/context. - * - * @typeParam Type - The shape of the item being loaded. - */ export interface ExternalSystemItemLoadingParams { item: Type; mappers: Mappers; event: AirSyncEvent; } -/** - * Result returned by an external-system loading function for a single item. - * - * Used to report the outcome of a create/update: the resulting external id, an error message, - * the item's modified date, or a delay (in seconds) when the external system is rate limiting. - */ export interface ExternalSystemItemLoadingResponse { - /** External system id of the loaded item, when the operation succeeded. */ id?: string; - /** Error message when the operation failed. */ error?: string; - /** Modified timestamp reported by the external system after the operation. */ modifiedDate?: string; - /** Suggested delay in seconds before retrying, set when rate limited. */ delay?: number; } -/** - * Record of an item that was loaded into the external system. - * - * Used to persist the outcome of a load (external id, error, modified date) for reporting and - * subsequent runs. - */ export interface ExternalSystemItemLoadedItem { - /** External system id of the loaded item. */ id?: string; - /** Error message if loading the item failed. */ error?: string; - /** Modified timestamp reported by the external system. */ modifiedDate?: string; } -/** - * A handler that loads a single item into the external system. - * - * Used to implement the create or update behavior for an item type; receives the item, ID mappers, - * and event, and resolves with the loading outcome. - * - * @typeParam Item - The shape of the item to load. - * @returns Promise resolving to the ExternalSystemItemLoadingResponse for the item. - */ export type ExternalSystemLoadingFunction = ({ item, mappers, event, }: ExternalSystemItemLoadingParams) => Promise; -/** - * Registration of an item type and the functions that load it. - * - * Used to tell the loader, for a given external item type, how to create and update records in the - * external system. - */ export interface ItemTypeToLoad { - /** External item type these handlers apply to. */ itemType: string; - /** Handler that creates a new record in the external system. */ create: ExternalSystemLoadingFunction; - /** Handler that updates an existing record in the external system. */ update: ExternalSystemLoadingFunction; // requiresSecondPass: boolean; } -/** - * Parameters bundling the full set of item-type loaders for a loading run. - * - * Used to pass the configured list of loadable item types into the loading entry point. - */ export interface ItemTypesToLoadParams { - /** The item types to load, each with its create/update handlers. */ itemTypesToLoad: ItemTypeToLoad[]; } -/** - * Per-item-type counters summarizing the outcome of a loading run. - * - * Used to report, for one item type, how many records were created/updated/skipped/deleted/failed. - */ export interface LoaderReport { - /** External item type this report covers. */ item_type: string; - /** Number of records created. */ [ActionType.CREATED]?: number; - /** Number of records updated. */ [ActionType.UPDATED]?: number; - /** Number of records skipped (no-op). */ [ActionType.SKIPPED]?: number; - /** Number of records deleted. */ [ActionType.DELETED]?: number; - /** Number of records that failed to load. */ [ActionType.FAILED]?: number; } -/** - * Signals that the external system is rate limiting and loading should pause. - * - * Used to propagate a back-off duration from a loading function up to the loader. - */ export interface RateLimited { - /** Number of seconds to wait before resuming. */ delay: number; } -/** - * Result of loading a single item, capturing success report, error, or rate-limit signal. - * - * Used internally by the loader to aggregate per-item outcomes. - */ export interface LoadItemResponse { - /** Error record when the item could not be loaded. */ error?: ErrorRecord; - /** Per-type counters contributed by this item. */ report?: LoaderReport; - /** Rate-limit signal when the external system is throttling. */ rateLimit?: RateLimited; } -/** - * Aggregate result of loading one or more item types. - * - * Used to return the per-type reports and the list of processed files at the end of a loading phase. - */ export interface LoadItemTypesResponse { - /** Per-item-type loading reports. */ reports: LoaderReport[]; - /** Names of the data files that were processed. */ processed_files: string[]; } -/** - * The kinds of actions a loader can perform on a record, used as report counter keys. - * - * Used to key {@link LoaderReport} counters and to classify the outcome of each loaded item. - */ export enum ActionType { - /** A new record was created in the external system. */ CREATED = 'created', - /** An existing record was updated. */ UPDATED = 'updated', - /** The record required no change. */ SKIPPED = 'skipped', - /** The record was deleted. */ DELETED = 'deleted', - /** Loading the record failed. */ FAILED = 'failed', } -/** A DevRev object identifier (DON), represented as a string. */ export type DonV2 = string; -/** - * A sync mapper record linking external and DevRev identifiers for one mapping. - * - * Used to track the correspondence between external ids, secondary ids, and DevRev ids, along with - * status, for sync operations. - */ export type SyncMapperRecord = { - /** External system identifiers for the mapped item. */ external_ids: string[]; - /** Secondary external identifiers (e.g. alternate keys). */ secondary_ids: string[]; - /** DevRev object identifiers for the mapped item. */ devrev_ids: string[]; - /** Status values associated with the mapping. */ status: string[]; - /** Input file the record was sourced from, when applicable. */ input_file?: string; }; -/** - * Outgoing event types reported by the loading phases. - * - * Used as the event_type when a loader emits control messages for data loading, attachment loading, - * and loader-state deletion (progress / delayed / done / error), plus a fallback for unrecognized events. - */ export enum LoaderEventType { DataLoadingProgress = 'DATA_LOADING_PROGRESS', DataLoadingDelayed = 'DATA_LOADING_DELAYED', diff --git a/src/uploader/uploader.helpers.ts b/src/uploader/uploader.helpers.ts index 85087db4..ad389a5b 100644 --- a/src/uploader/uploader.helpers.ts +++ b/src/uploader/uploader.helpers.ts @@ -9,12 +9,9 @@ import { import { UploaderResult } from './uploader.interfaces'; /** - * Compresses a JSONL string using gzip. - * - * Used to shrink a serialized JSONL batch before uploading it as an artifact. - * - * @param jsonlObject - The JSONL string to compress. - * @returns An UploaderResult wrapping the gzipped Buffer, or an error on failure. + * Compresses a JSONL string using gzip compression. + * @param {string} jsonlObject - The JSONL string to compress + * @returns {Buffer | void} The compressed buffer or undefined on error */ export function compressGzip(jsonlObject: string): UploaderResult { try { @@ -25,12 +22,9 @@ export function compressGzip(jsonlObject: string): UploaderResult { } /** - * Decompresses a gzipped buffer back into a JSONL string. - * - * Used to restore a downloaded gzipped artifact before parsing it. - * - * @param gzippedJsonlObject - The gzipped Buffer to decompress. - * @returns An UploaderResult wrapping the decompressed JSONL string, or an error on failure. + * Decompresses a gzipped buffer to a JSONL string. + * @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress + * @returns {string | void} The decompressed JSONL string or undefined on error */ export function decompressGzip( gzippedJsonlObject: Buffer @@ -45,11 +39,8 @@ export function decompressGzip( /** * Parses a JSONL string into an array of objects. - * - * Used to turn a decompressed artifact into usable records. - * - * @param jsonlObject - The JSONL string to parse. - * @returns An UploaderResult wrapping the parsed object array, or an error on failure. + * @param {string} jsonlObject - The JSONL string to parse + * @returns {object[] | null} The parsed array of objects or null on error */ export function parseJsonl(jsonlObject: string): UploaderResult { try { @@ -60,14 +51,10 @@ export function parseJsonl(jsonlObject: string): UploaderResult { } /** - * Writes fetched objects to the local file system for local development. - * - * Used to inspect extracted data on disk instead of uploading it when running locally; writes a - * timestamped JSON/JSONL file under the `extracted_files` directory. - * - * @param itemType - The string item type, used to name the output file and pick its extension. - * @param fetchedObjects - The object or array of objects to write, one JSON record per line. - * @returns Promise that resolves once the file is written, or rejects on a write error. + * Downloads fetched objects to the local file system (for local development). + * @param {string} itemType - The type of items being downloaded + * @param {object | object[]} fetchedObjects - The objects to write to file + * @returns {Promise} Resolves when the file is written or rejects on error */ export async function downloadToLocal( itemType: string, @@ -103,13 +90,9 @@ export async function downloadToLocal( } /** - * Truncates a filename that exceeds the platform's maximum length. - * - * Used before requesting an upload URL so the registered file name stays within DevRev limits, - * preserving the extension and inserting an ellipsis in the middle. - * - * @param filename - The string filename to truncate. - * @returns The original filename if within the limit, otherwise a truncated `name...ext` string. + * Truncates a filename if it exceeds the maximum allowed length. + * @param {string} filename - The filename to truncate + * @returns {string} The truncated filename */ export function truncateFilename(filename: string): string { // If the filename is already within the limit, return it as is. diff --git a/src/uploader/uploader.interfaces.ts b/src/uploader/uploader.interfaces.ts index ee8027c0..2224bed7 100644 --- a/src/uploader/uploader.interfaces.ts +++ b/src/uploader/uploader.interfaces.ts @@ -3,15 +3,8 @@ import { AirSyncEvent } from '../types/extraction'; import { ExternalSystemItem, StatsFileObject } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; -/** - * Construction parameters used to create an Uploader instance. - * - * Used to supply the triggering event (platform endpoint, token, request id) and optional adapter options. - */ export interface UploaderFactoryInterface { - /** The AirSync event supplying the DevRev endpoint, service account token, and request id. */ event: AirSyncEvent; - /** Optional worker adapter options (e.g. local development and skip-confirmation flags). */ options?: WorkerAdapterOptions; } @@ -97,26 +90,12 @@ export interface SsorAttachment { inline?: boolean; } -/** - * Result of fetching and parsing a loading stats file artifact. - * - * Used to return the per-item-type stats produced by the loading phase, or an error if it could not be read. - */ export interface StatsFileResponse { - /** Error describing why the stats file could not be retrieved or parsed. */ error?: ErrorRecord; - /** Parsed stats file entries, one per item type. */ statsFile?: StatsFileObject[]; } -/** - * Result of fetching and parsing a transformer file artifact. - * - * Used to return the transformed external system items to be loaded into DevRev, or an error if it could not be read. - */ export interface TransformerFileResponse { - /** Error describing why the transformer file could not be retrieved or parsed. */ error?: ErrorRecord; - /** Parsed external system items to load. */ transformerFile?: ExternalSystemItem[]; } diff --git a/src/uploader/uploader.ts b/src/uploader/uploader.ts index 29ae52c9..fa9833af 100644 --- a/src/uploader/uploader.ts +++ b/src/uploader/uploader.ts @@ -22,12 +22,6 @@ import { UploaderResult, } from './uploader.interfaces'; -/** - * Uploads extraction artifacts to the DevRev platform and reads them back. - * - * Used to compress and upload JSON batches and streamed attachment binaries, obtain upload/download - * URLs, confirm uploads, and download and parse previously uploaded artifacts during sync. - */ export class Uploader { private isLocalDevelopment?: boolean; private devrevApiEndpoint: string; @@ -48,14 +42,10 @@ export class Uploader { } /** - * Uploads fetched objects to the DevRev platform as a single artifact. - * - * Used to compress the objects into a gzipped JSONL file, request an upload URL, push the file, and - * (unless skipped) confirm the upload, returning the resulting artifact descriptor. - * - * @param itemType - The string item type of the objects being uploaded. - * @param fetchedObjects - The object or array of objects to upload. - * @returns Promise resolving to an UploadResponse with the artifact descriptor, or an error message on failure. + * Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform. + * @param {string} itemType - The type of the item to be uploaded + * @param {object[] | object} fetchedObjects - The objects to be uploaded + * @returns {Promise} - The response object containing the artifact information or error information if there was an error */ async upload( itemType: string, @@ -137,14 +127,11 @@ export class Uploader { } /** - * Requests a pre-signed upload URL and form data for a new artifact. - * - * Used before uploading or streaming a file so the binary can be POSTed to the returned URL. - * - * @param filename - The string file name to register (truncated if it exceeds the platform limit). - * @param fileType - The string MIME type of the file. - * @param fileSize - Optional number of bytes; rejected if 0 or less. - * @returns Promise resolving to an UploaderResult wrapping the ArtifactToUpload, or an error on failure. + * Gets the upload URL for an artifact from the DevRev API. + * @param {string} filename - The name of the file to upload + * @param {string} fileType - The MIME type of the file + * @param {number} [fileSize] - Optional file size in bytes + * @returns {Promise} The artifact upload information or undefined on error */ async getArtifactUploadUrl( filename: string, @@ -178,13 +165,10 @@ export class Uploader { } /** - * Uploads an in-memory file buffer to a pre-signed artifact upload URL. - * - * Used to push a fully buffered artifact (e.g. a compressed JSON batch) as multipart form data. - * - * @param artifact - The ArtifactToUpload descriptor holding the upload URL and form fields. - * @param file - The Buffer containing the file contents to upload. - * @returns Promise resolving to an UploaderResult wrapping the AxiosResponse, or an error on failure. + * Uploads an artifact file to the provided upload URL using multipart form data. + * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data + * @param {Buffer} file - The file buffer to upload + * @returns {Promise} The axios response or undefined on error */ async uploadArtifact( artifact: ArtifactToUpload, @@ -209,14 +193,10 @@ export class Uploader { } /** - * Streams a file directly from a source response into a pre-signed artifact upload URL. - * - * Used to upload attachment binaries without buffering them in memory; falls back to the max - * artifact size for Content-Length when the source omits it, and always destroys the source stream. - * - * @param artifact - The ArtifactToUpload descriptor holding the upload URL and form fields. - * @param fileStream - The AxiosResponse whose data stream supplies the file contents. - * @returns Promise resolving to an UploaderResult wrapping the AxiosResponse, or an error on failure. + * Streams an artifact file from an axios response to the upload URL. + * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data + * @param {AxiosResponse} fileStream - The axios response stream containing the file data + * @returns {Promise} The axios response or undefined on error */ async streamArtifact( artifact: ArtifactToUpload, @@ -252,12 +232,9 @@ export class Uploader { } /** - * Confirms with the platform that an artifact upload has finished. - * - * Used after pushing the binary so the platform finalizes and accepts the artifact. - * - * @param artifactId - The string ID of the uploaded artifact to confirm. - * @returns Promise resolving to an object with the AxiosResponse on a 2xx, or an error otherwise. + * Confirms that an artifact upload has been completed successfully. + * @param {string} artifactId - The ID of the artifact to confirm + * @returns {Promise} The axios response or undefined on error */ async confirmArtifactUpload(artifactId: string): Promise<{ response?: AxiosResponse; @@ -296,11 +273,8 @@ export class Uploader { } /** - * Destroys a source stream to prevent resource leaks after streaming an artifact. - * - * Used internally by streamArtifact to close the AxiosResponse data stream on both success and error. - * - * @param fileStream - The AxiosResponse whose underlying data stream should be destroyed/closed. + * Destroys a stream to prevent resource leaks. + * @param {any} fileStream - The axios response stream to destroy */ private destroyStream(fileStream: AxiosResponse): void { try { @@ -318,13 +292,10 @@ export class Uploader { } /** - * Downloads an attachments-metadata artifact and parses it into normalized attachments. - * - * Used during attachment extraction to read back the previously uploaded attachment metadata so its - * binaries can be streamed; resolves the download URL, downloads, gunzips, and parses the JSONL. - * - * @param param0 - Object with `artifact`, the string artifact ID of the attachments-metadata artifact. - * @returns Promise resolving to an object with the NormalizedAttachment array, or an error message on failure. + * Retrieves attachment metadata from an artifact by downloading and parsing it. + * @param {object} param0 - Configuration object + * @param {string} param0.artifact - The artifact ID to download attachments from + * @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object */ async getAttachmentsFromArtifactId({ artifact, @@ -393,12 +364,9 @@ export class Uploader { } /** - * Requests a pre-signed download URL for an artifact from the platform. - * - * Used internally before downloading an artifact's contents back from object storage. - * - * @param artifactId - The string ID of the artifact to download. - * @returns Promise resolving to an UploaderResult wrapping the download URL string, or an error on failure. + * Gets the download URL for an artifact from the DevRev API. + * @param {string} artifactId - The ID of the artifact to download + * @returns {Promise} The download URL or undefined on error */ private async getArtifactDownloadUrl( artifactId: string @@ -423,12 +391,9 @@ export class Uploader { } /** - * Downloads an artifact's raw bytes from a pre-signed URL. - * - * Used internally to fetch artifact contents as a Buffer for later decompression and parsing. - * - * @param artifactUrl - The string pre-signed URL to download the artifact from. - * @returns Promise resolving to an UploaderResult wrapping the file Buffer, or an error on failure. + * Downloads an artifact file from the given URL. + * @param {string} artifactUrl - The URL to download the artifact from + * @returns {Promise} The artifact file buffer or undefined on error */ private async downloadArtifact( artifactUrl: string @@ -445,12 +410,11 @@ export class Uploader { } /** - * Downloads an artifact by ID and parses its JSONL contents into objects. - * - * Used to read back a previously uploaded JSON batch; optionally gunzips the bytes first. - * - * @param param0 - Object with `artifactId` (string artifact ID) and optional `isGzipped` (boolean, default false) flag. - * @returns Promise resolving to an UploaderResult wrapping the parsed object or object array, or an error on failure. + * Retrieves and parses JSON objects from an artifact by artifact ID. + * @param {object} param0 - Configuration object + * @param {string} param0.artifactId - The artifact ID to download and parse + * @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped + * @returns {Promise} The parsed JSON objects or undefined on error */ async getJsonObjectByArtifactId({ artifactId, From 9b2b9758ef0ccb96f5e5281e4d762d79eb62c96f Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:47:23 +0200 Subject: [PATCH 21/30] refactor(v2): post-review cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent cleanups from the v2 self-review: 1. Drop the pre-1.15.2 processed-attachments migration. `migrateProcessedAttachments` only upgraded the legacy `string[]` form of `lastProcessedAttachmentsIdsList` to the structured `ProcessedAttachment[]` form. That `string[]` format predates v1.15.2 (commit c3aa151, Feb 2026); from v1.15.2 on, state is already written as `ProcessedAttachment[]`, and the list is per-cycle scratch state cleared at the end of each completed attachment phase. v2 assumes on-disk state written by >= v1.15.2 (connectors are on >= 1.16, typically 1.19, before moving to v2), so the shim is dead weight. The call site now just initializes the list to [] when absent. 2. Move the `emit` primitive `common/control-protocol.ts` -> `multithreading/emit.ts`. The file's sole export is `emit`, used only by the multithreading layer (base-adapter, spawn) — co-locating it with its consumers and naming it after its content. Import paths updated. 3. Remove the dead `createAdapterState` dispatcher from `state.ts`. After C6 split the single entry point into processExtractionTask/processLoadingTask, process-task.ts calls createExtractionState/createLoadingState directly; the mode dispatcher had zero callers. `state.ts` is now a thin re-export barrel. Build green, lint clean. --- .../attachments-streaming-pool.ts | 37 ------------------- src/multithreading/adapters/base-adapter.ts | 2 +- .../emit.ts} | 2 +- src/multithreading/spawn/spawn.ts | 2 +- src/state/state.ts | 23 ------------ 5 files changed, 3 insertions(+), 63 deletions(-) rename src/{common/control-protocol.ts => multithreading/emit.ts} (95%) diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index 58281e16..59ce3648 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -1,6 +1,5 @@ import { sleep } from '../common/helpers'; import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; -import { ProcessedAttachment } from '../state/state.interfaces'; import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment, @@ -40,35 +39,6 @@ export class AttachmentsStreamingPool { } } - /** - * Migrates processed attachments from the legacy string[] format to the new ProcessedAttachment[] format. - * - * @param attachments - The attachments list to migrate (either string[] or ProcessedAttachment[]) - * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private migrateProcessedAttachments(attachments: any): ProcessedAttachment[] { - // Handle null/undefined - if (!attachments || !Array.isArray(attachments)) { - return []; - } - - // If already migrated (first element is an object), return as-is - if (attachments.length > 0 && typeof attachments[0] === 'object') { - return attachments as ProcessedAttachment[]; - } - - // Migrate old string[] format - if (attachments.length > 0 && typeof attachments[0] === 'string') { - return attachments.map((it) => ({ - id: it as string, - parent_id: '', - })); - } - - return []; - } - async streamAll(): Promise { console.log( `Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.` @@ -90,13 +60,6 @@ export class AttachmentsStreamingPool { []; } - // Migrate old processed attachments to the new format. - this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = - this.migrateProcessedAttachments( - this.adapter.sdkState.toDevRev.attachmentsMetadata - .lastProcessedAttachmentsIdsList - ); - // Start initial batch of promises up to batchSize limit const initialBatchSize = Math.min(this.batchSize, this.attachments.length); const initialPromises = []; diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts index dec92853..ab36c0e6 100644 --- a/src/multithreading/adapters/base-adapter.ts +++ b/src/multithreading/adapters/base-adapter.ts @@ -1,7 +1,7 @@ import { parentPort } from 'node:worker_threads'; import { STATELESS_EVENT_TYPES } from '../../common/constants'; -import { emit } from '../../common/control-protocol'; +import { emit } from '../emit'; import { truncateMessage } from '../../common/helpers'; import { serializeError } from '../../logger/logger'; import { runWithSdkLogContext } from '../../logger/logger.context'; diff --git a/src/common/control-protocol.ts b/src/multithreading/emit.ts similarity index 95% rename from src/common/control-protocol.ts rename to src/multithreading/emit.ts index 0d13be13..9296d87c 100644 --- a/src/common/control-protocol.ts +++ b/src/multithreading/emit.ts @@ -8,7 +8,7 @@ import { LoaderEvent, } from '../types/extraction'; import { LoaderEventType } from '../types/loading'; -import { LIBRARY_VERSION } from './constants'; +import { LIBRARY_VERSION } from '../common/constants'; export interface EmitInterface { event: AirSyncEvent; diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 02176ff0..064d82c5 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -1,7 +1,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import { emit } from '../../common/control-protocol'; +import { emit } from '../emit'; import { getMemoryUsage } from '../../common/helpers'; import { Logger, serializeError } from '../../logger/logger'; import { AirSyncEvent, EventType } from '../../types/extraction'; diff --git a/src/state/state.ts b/src/state/state.ts index ab7dbb5c..7ba15891 100644 --- a/src/state/state.ts +++ b/src/state/state.ts @@ -1,26 +1,3 @@ -import { SyncMode } from '../types/common'; - -import { BaseState } from './base-state'; -import { createExtractionState } from './extraction-state'; -import { createLoadingState } from './loading-state'; -import { StateInterface } from './state.interfaces'; - export { BaseState } from './base-state'; export { ExtractionState, createExtractionState } from './extraction-state'; export { LoadingState, createLoadingState } from './loading-state'; - -/** - * Creates and initializes the adapter state for the current worker, dispatching - * to the extraction or loading state based on the event's sync mode. - * - * @param params The state factory parameters (event, initial state, options) - * @returns The initialized mode-specific state - */ -export async function createAdapterState( - params: StateInterface -): Promise> { - if (params.event.payload.event_context.mode === SyncMode.LOADING) { - return createLoadingState(params); - } - return createExtractionState(params); -} From 4269e30caa22aa1f0c6cc397c8190ab75043658b Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 9 Jun 2026 10:47:57 +0200 Subject: [PATCH 22/30] docs: record post-review cleanup + defer C7 JSDoc to end --- V2_PROGRESS.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md index d9e2f9e7..5094c948 100644 --- a/V2_PROGRESS.md +++ b/V2_PROGRESS.md @@ -181,10 +181,15 @@ commits. Mechanical/structural transforms first (Phase 1), polish + surface-defi emit-based, so the TaskResult return + emitFromResult mapping is authored fresh per the spec above. ### Phase 2 — closing / interactive (batched, done at the end) -- **C7 — JSDoc pass.** Bar = `src/mappers/mappers.ts` style (class block: what+when; method block: - one-line what, "Used to/for…" usage, `@param` w/ type, `@returns`). Public surface + non-obvious - internals (state migration, emit-from-return mapping, attachment streaming pool). Fan out per module, - squash to one `docs:` commit. +- **C7 — JSDoc pass (DEFERRED to the very end).** Bar = `src/mappers/mappers.ts` style (class block: + what+when; method block: one-line what, "Used to/for…" usage, `@param` w/ type, `@returns`). Public + surface + non-obvious internals (state migration, emit-from-return mapping, attachment streaming pool). + FIRST attempt (d05434b, fanned-out subagents) was REVERTED (4fea755): too heavy (902 insertions), some + blocks just restated the obvious, made structural review harder. REDO as the LAST step of the whole + refactor — done once, by hand, against settled code, with an explicit **"skip-the-obvious" rule**: + NO docs on trivial getters/setters, NO restating the method name; document only class-level "what+when", + non-obvious params, and genuinely tricky internals. The high-value blocks (normalizeFetchedState + migration shim, getEventTypeForResult/EVENT_PHASE_MAP, zero-doc types/loading.ts) still get covered. - **C8 — Regenerate api report** (`airsync-sdk.api.md`). - **C9 — Exposure audit (INTERACTIVE with Rado).** Read the regenerated report; decide per-symbol what to keep public vs hide. Empirical floor = anything imported by the 3 connectors (table below). @@ -279,6 +284,12 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: + return-based contract), `processTask` (split), `formatAxiosError` (dropped from index), `AirdropEvent`/`AirdropMessage` (renamed AirSync*), all old `EXTRACTION_*` enum members (deleted). +**v2 on-disk-state baseline assumption (for C11 migrate-v2 note):** v2 assumes persisted state was written by +SDK **>= v1.15.2**. The pre-1.15.2 `string[]` form of `lastProcessedAttachmentsIdsList` is NO LONGER migrated +(shim removed in CLEANUP/9b2b975). The v1→v2 *envelope* migration (`normalizeFetchedState`, C4b) is KEPT — that's +the real v1-vs-v2 boundary (every v1 version, incl. 1.19, writes flat state). Only the ancient intra-v1 string[] +processed-attachments format was dropped. + ## Status | Commit | State | Notes | |--------|-------|-------| @@ -290,7 +301,8 @@ Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: | C4b state envelope | ☑ done | 30ba1b3. { connectorState, sdkState } envelope + v1->v2 migration shim (normalizeFetchedState). adapter.state→connector-only, new adapter.sdkState; ~28 SDK-field access sites moved. SdkState kept combined (narrowing deferred to C5). Reviewer-approved (migration cases verified). | | C5 adapter split | ☑ done | a7a877f. BaseAdapter (template emit + hooks) + ExtractionAdapter + LoadingAdapter; WorkerAdapter→union alias; processTask dispatches by mode (still single entry). worker-adapter.ts deleted; helpers→loading-adapter.helpers. Reviewer-approved (emit equivalence verified). SdkState kept combined (narrowing dropped from scope). | | C6 emit-from-return | ☑ done | 0fb6116. task/onTimeout return TaskResult; SDK maps status→event via getEventTypeForResult and emits once (emitFromResult); emit now protected/internal; processTask→processExtractionTask+processLoadingTask; loader/stream methods return TaskResult. Reviewer-approved (mapping+state-save+no-double-emit verified). NET-NEW design (no oracle). | -| C7 JSDoc | ☑ done | d05434b. Comments-only pass over 25 files to the mappers.ts bar: v2-new code (adapters, state incl. migration shim, process-task/spawn/getEventTypeForResult mapping) + under-documented older modules (repo, uploader, attachments pool, control-protocol, install-IDM, errors, types/loading barrel). Verified every changed line is a comment; build green, lint clean. Fanned out 5 implementer subagents over disjoint file groups. | +| CLEANUP (post-review) | ☑ done | 9b2b975. (1) Removed dead pre-1.15.2 `migrateProcessedAttachments` shim (v2 assumes on-disk state >= v1.15.2); (2) moved emit primitive `common/control-protocol.ts`→`multithreading/emit.ts` (co-located w/ its only consumers base-adapter+spawn); (3) deleted dead `createAdapterState` dispatcher (zero callers post-C6), `state.ts` now a thin barrel. Build green, lint clean. Preceded by 4fea755 (revert of C7). | +| C7 JSDoc | ☐ todo | DEFERRED to very end. First attempt d05434b REVERTED (4fea755) — too heavy/redundant, hurt review. Redo by hand, once, against settled code, with "skip-the-obvious" rule (see C7 plan above). | | C8 api report | ☐ todo | Phase 2 | | C9 exposure audit | ☐ todo | Phase 2, interactive | | C10 tests + baseline | ☐ todo | Phase 2 | From 1d7b1ef35687b53a2f7c886e3491310ebc2da7da Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Mon, 15 Jun 2026 11:49:58 +0200 Subject: [PATCH 23/30] refactor(v2): post-review cleanup - attachments: yield via setImmediate instead of sleep(100) in the streaming progress path; drop the noisy "Sleeping for Nms" log from the sleep() helper. Same soft-timeout observability, ~200s less artificial latency on large syncs. - public API: collapse to a single explicit barrel in src/index.ts (no export *); delete the dead src/http/index.ts and redundant src/types/index.ts; unexport BaseAdapter; promote Mappers/Item/ ItemTypeToLoad onto the root barrel. - timeout: once isTimeout fired, the timeout outcome always wins -- the SDK runs onTimeout (or default progress) and ignores the task return. - types: widen AirSyncEvent.context with user_id/dev_oid/source_id/ service_account_id so connectors stop casting. - docs: add v1 -> v2 MIGRATION.md. Co-Authored-By: Claude Fable 5 --- MIGRATION.md | 462 ++++++++++++++++++ .../attachments-streaming-pool.interfaces.ts | 6 +- .../attachments-streaming-pool.ts | 11 +- src/common/helpers.ts | 1 - src/common/test-utils.ts | 4 + src/http/index.ts | 2 - src/index.ts | 184 ++++++- src/mappers/mappers.interface.ts | 2 +- src/multithreading/process-task.ts | 15 +- src/types/extraction.ts | 8 + src/types/index.ts | 126 ----- 11 files changed, 665 insertions(+), 156 deletions(-) create mode 100644 MIGRATION.md delete mode 100644 src/http/index.ts delete mode 100644 src/types/index.ts diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..5e1bcd3d --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,462 @@ +# Migrating connectors from v1 (`@devrev/ts-adaas`) to v2 (`@devrev/airsync-sdk`) + +This guide covers every breaking change between `@devrev/ts-adaas` v1.x and +`@devrev/airsync-sdk` v2.0.0, with before/after examples taken from real +connectors. The wire protocol (event payloads sent to the platform, API routes, +artifact/upload flow) is **unchanged** — only the connector-facing API changed. + +## TL;DR — what breaks + +| # | Change | Impact | +|---|--------|--------| +| 1 | Package renamed `@devrev/ts-adaas` → `@devrev/airsync-sdk` | every import | +| 2 | `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage` | type annotations | +| 3 | Deprecated event-type enum members deleted | only if you used old `Extraction*` members | +| 4 | `processTask` → `processExtractionTask` / `processLoadingTask` | every worker file | +| 5 | `adapter.emit(...)` removed — tasks **return** a `TaskResult` instead | every worker file | +| 6 | External sync units must be pushed to a repo (no more `external_sync_units` in emit data) | ESU workers | +| 7 | `adapter.state` is connector-state only; SDK fields moved to `adapter.sdkState` | only if you read/wrote SDK fields | +| 8 | `WorkerAdapter` class/type removed → `ExtractionAdapter` / `LoadingAdapter` | helper-function signatures | +| 9 | `loadItemTypes` / `loadAttachments` / `streamAttachments` return a `TaskResult` | loading + attachment workers | +| 10 | Deprecated v1 modules deleted (`Adapter`, `createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, legacy `Uploader`) | only legacy code | +| 11 | Legacy `string[]` form of the processed-attachments dedup list is no longer migrated | in-flight attachment syncs started on SDK < 1.15.2 | + +What does **not** change: `spawn(...)` and all its options (`baseWorkerPath`, +`initialState`, `initialDomainMapping`, `options.batchSize`, `timeout`, +`isLocalDevelopment`, `workerPathOverrides`), default worker paths +(`/workers/data-extraction` etc.), repos (`initializeRepos`, `getRepo`, +`push`), normalization interfaces (`NormalizedItem`, `NormalizedAttachment`, +`RepoInterface`, `ExternalSyncUnit`), the uploader, mappers, logger, +`installInitialDomainMapping`, the `axios`/`axiosClient` re-exports, +`formatAxiosError`/`serializeAxiosError`, `createMockEvent`/`MockServer`, +HTTP retry behavior, and all event-type **string values** on the wire. + +--- + +## 1. Package rename + +```bash +npm uninstall @devrev/ts-adaas +npm install @devrev/airsync-sdk +``` + +Then global-replace the import specifier: + +```ts +// v1 +import { spawn, EventType } from '@devrev/ts-adaas'; +// v2 +import { spawn, EventType } from '@devrev/airsync-sdk'; +``` + +> **Deep imports** like `import { Item } from '@devrev/ts-adaas/dist/repo/repo.interfaces'` +> are fragile and should be replaced with root imports where the symbol is +> exported. If a symbol you need is not exported from the root, request it +> rather than deep-importing. + +## 2. Type renames + +Hard rename, no compatibility alias: + +| v1 | v2 | +|----|----| +| `AirdropEvent` | `AirSyncEvent` | +| `AirdropMessage` | `AirSyncMessage` | + +```ts +// v1 +const run = async (events: AirdropEvent[]) => { ... }; +// v2 +const run = async (events: AirSyncEvent[]) => { ... }; +``` + +Payload shape is identical — only the type name changed. Platform-owned strings +(`/internal/airdrop.*` routes, `'ADaaS'` external system type, +`adaas_library_version` metadata key, `airdrop_*` mapping enum values) are +intentionally unchanged. + +> `AirSyncEvent.context` now declares the identity fields the platform actually +> sends — `user_id`, `dev_oid`, `source_id`, `service_account_id` — alongside the +> existing `secrets`, `snap_in_id`, and `snap_in_version_id`. If you previously +> cast the event to a hand-rolled `CustomAirdropEvent` to read `context.user_id`, +> you can drop that cast and read the fields off `adapter.event.context` directly. + +## 3. Deleted deprecated enum members + +The old/new duplicate enum members were collapsed; only the new names remain. +The **string values of the kept members are byte-identical to v1**, so nothing +changes on the wire — only the TypeScript member names. + +> The **deleted** members carried *different, older* string values +> (e.g. v1 `ExtractionDataStart = 'EXTRACTION_DATA_START'` vs the kept +> `StartExtractingData = 'START_EXTRACTING_DATA'`). The platform stopped sending +> those old strings, and v2 no longer translates them (see the note at the end +> of this section), so the deletion is safe — but don't assume a deleted member +> and its replacement shared a value; they did not. + +**`EventType` (incoming):** + +| Deleted (v1 deprecated) | Use instead | +|--------------------------|-------------| +| `ExtractionExternalSyncUnitsStart` | `StartExtractingExternalSyncUnits` | +| `ExtractionMetadataStart` | `StartExtractingMetadata` | +| `ExtractionDataStart` | `StartExtractingData` | +| `ExtractionDataContinue` | `ContinueExtractingData` | +| `ExtractionDataDelete` | `StartDeletingExtractorState` | +| `ExtractionAttachmentsStart` | `StartExtractingAttachments` | +| `ExtractionAttachmentsContinue` | `ContinueExtractingAttachments` | +| `ExtractionAttachmentsDelete` | `StartDeletingExtractorAttachmentsState` | + +**`ExtractorEventType` (outgoing):** all `Extraction*` members +(`ExtractionDataDone`, `ExtractionDataDelay`, `ExtractionAttachmentsProgress`, …) +are deleted; use the `*Extraction*` members (`DataExtractionDone`, +`DataExtractionDelayed`, `AttachmentExtractionProgress`, …). In practice you +will rarely reference `ExtractorEventType` at all in v2 — see §5. + +**`LoaderEventType`:** the typo/duplicate members `DataLoadingDelay` and +`AttachmentsLoadingProgress/Delayed/Done/Error` are deleted; use +`DataLoadingDelayed` and `AttachmentLoading*`. + +Also note: the SDK no longer translates old event-type strings coming from the +platform — the platform sends only the new strings, and the SDK passes +`event.payload.event_type` through untouched. + +## 4 + 5. The new worker contract: return a `TaskResult` instead of emitting + +This is the core change of v2. In v1 the connector decided *which event* to +emit and called `adapter.emit(...)`. In v2 the connector only reports *how the +phase ended* by **returning** a `TaskResult`; the SDK maps it to the correct +platform event for the current phase and emits it exactly once: + +```ts +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } + | { status: 'error'; error: ErrorRecord }; +``` + +Status → emitted event, per phase: + +| status | resumable phases (data/attachment extraction & loading) | non-resumable (ESU, metadata, state deletion) | +|--------|----------------------------------------------------------|-----------------------------------------------| +| `'success'` | `*Done` | `*Done` | +| `'progress'` | `*Progress` | `*Error` (illegal status; descriptive message) | +| `'delay'` | `*Delayed` (with `delay` seconds) | `*Error` (illegal status) | +| `'error'` | `*Error` (with the error record) | `*Error` | + +`processTask` is split into two typed entry points, and `onTimeout` is now +**optional** (defaults to a `progress` result — see the note below for +non-resumable phases): + +```ts +// v1 +import { processTask, ExtractorEventType } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + // ... extract ... + await adapter.emit(ExtractorEventType.DataExtractionDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(ExtractorEventType.DataExtractionProgress, { progress: 50 }); + }, +}); +``` + +```ts +// v2 +import { processExtractionTask } from '@devrev/airsync-sdk'; + +processExtractionTask({ + task: async ({ adapter }) => { + // ... extract ... + return { status: 'success' }; + }, + // onTimeout can be omitted entirely: the SDK emits progress by default. +}); +``` + +Loading workers use `processLoadingTask` the same way. + +### Emit-call → return-value translation table + +| v1 | v2 | +|----|----| +| `await adapter.emit(XxxDone)` | `return { status: 'success' }` | +| `await adapter.emit(XxxProgress, { progress: n })` | `return { status: 'progress' }` (the `progress` % was already deprecated/ignored) | +| `await adapter.emit(XxxDelayed, { delay: seconds })` | `return { status: 'delay', delaySeconds: seconds }` | +| `await adapter.emit(XxxError, { error: { message } })` | `return { status: 'error', error: { message } }` | +| `await adapter.emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to repo + `return { status: 'success' }` — see §6 | + +Everything the SDK attached automatically in v1 is still attached +automatically: artifacts of uploaded repos (extraction events), `reports` and +`processed_files` (loading events), and state is still saved before every +non-stateless emit. An explicit `await adapter.postState()` before returning is +no longer needed for the emit path (it still works and is harmless). + +### Emits buried in helper functions + +A common v1 pattern is emitting deep inside helpers and signalling the caller +to stop: + +```ts +// v1 — helper emits, returns false to abort +async function extractList(adapter: WorkerAdapter, ...) { + if (rateLimited) { + await adapter.emit(ExtractorEventType.DataExtractionDelayed, { delay }); + return false; + } + ... +} +``` + +In v2 the helper must **bubble the outcome up** instead, because only the +task's return value reaches the SDK: + +```ts +// v2 — helper returns the outcome; task returns it to the SDK +async function extractList(adapter: ExtractionAdapter, ...): Promise { + if (rateLimited) return { status: 'delay', delaySeconds: delay }; + ... + return null; // keep going +} + +processExtractionTask({ + task: async ({ adapter }) => { + for (const itemType of itemTypes) { + const stop = await extractList(adapter, itemType); + if (stop) return stop; + } + return { status: 'success' }; + }, +}); +``` + +### Timeout handling inside the task + +Checking `adapter.isTimeout` in your extraction loop still works exactly like +v1 — but instead of emitting progress and exiting, return progress: + +```ts +// v2 +if (adapter.isTimeout) { + return { status: 'progress' }; // platform will send CONTINUE_* next +} +``` + +> **Behavior note — the timeout outcome always wins.** Once the soft timeout has +> fired, the SDK emits the `onTimeout` result (or the default `progress`) and +> **ignores whatever the task returned**, by design: a phase that ran out of time +> must hand off for continuation rather than report itself complete. So if you +> have cleanup or a specific outcome that must survive a timeout, put it in +> `onTimeout`, not in the task's return value. + +> **Non-resumable phases (ESU, metadata, state deletion):** the default +> `onTimeout` produces a `progress` result, which is illegal for these phases +> and is emitted as an error with a generic message. If you want a clean error +> message on timeout for these workers, provide an explicit `onTimeout`: +> `onTimeout: async () => ({ status: 'error', error: { message: 'Lambda timeout.' } })`. + +## 6. External sync units go through a repo + +In v1 the SDK accepted `external_sync_units` in the emit data and internally +uploaded them via a repo. With emit gone, push them to the +`EXTERNAL_SYNC_UNITS` repo yourself: + +```ts +// v1 +await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { + external_sync_units: externalSyncUnits, +}); +``` + +```ts +// v2 +import { AirSyncDefaultItemTypes, processExtractionTask } from '@devrev/airsync-sdk'; + +processExtractionTask({ + task: async ({ adapter }) => { + const externalSyncUnits = await fetchExternalSyncUnits(); + + adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + // match the batching the v1 SDK used internally for ESUs + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, + ]); + await adapter + .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) + ?.push(externalSyncUnits); + + return { status: 'success' }; + }, +}); +``` + +The repo is uploaded automatically before the `Done` event is emitted (same as +all repos). + +## 7. State split: `adapter.state` vs `adapter.sdkState` + +In v1, connector state and SDK bookkeeping lived in one flat object, persisted +as one blob, and `adapter.state` exposed both. In v2: + +- `adapter.state` is **connector state only** — exactly the shape of the + `initialState` you pass to `spawn`. Reads and writes work as before. +- SDK bookkeeping (`lastSyncStarted`, `lastSuccessfulSyncStarted`, + `workersOldest/Newest`, `pendingWorkers*`, `toDevRev`, `fromDevRev`, + `snapInVersionId`) moved to `adapter.sdkState`. It is SDK-internal; do not + write to it. If you were *reading* one of these fields (e.g. + `adapter.state.lastSuccessfulSyncStarted` for incremental sync), prefer + `event.payload.event_context.extract_from` / `extract_to`, which the SDK now + resolves for you. +- On disk, state is persisted as `{ connectorState, sdkState }`. The SDK + **migrates v1 flat blobs automatically on first read** (known SDK keys are + split out; everything else becomes connector state), so in-flight syncs + survive the upgrade. The migration is **not** version-gated — any flat blob is + split, regardless of which SDK version wrote it. (The one thing that is *not* + back-migrated is the legacy `string[]` form of the attachment dedup list — see + §9.) +- Edge case: if your v1 connector state had a top-level key literally named + `connectorState` or `sdkState`, the migration will misread it — rename it + before upgrading. + +```ts +// v1 — SDK fields visible on state (don't carry this over) +if (adapter.state.lastSuccessfulSyncStarted) { ... } + +// v2 — use the resolved extraction window instead +const { extract_from, extract_to } = adapter.event.payload.event_context; +``` + +Pure connector-state usage is unaffected: + +```ts +// works identically in v1 and v2 +adapter.state[itemType].cursor = nextCursor; +adapter.state[itemType].complete = true; +``` + +## 8. `WorkerAdapter` → `ExtractionAdapter` / `LoadingAdapter` + +The `WorkerAdapter` class no longer exists. Helper signatures change to the +mode-specific adapter: + +```ts +// v1 +async function extractList(adapter: WorkerAdapter, ...) { ... } +// v2 +import { ExtractionAdapter } from '@devrev/airsync-sdk'; +async function extractList(adapter: ExtractionAdapter, ...) { ... } +``` + +Extraction surface (`initializeRepos`, `getRepo`, `artifacts`, +`streamAttachments`, `processAttachment`, `shouldExtract`) lives on +`ExtractionAdapter`; loading surface (`loadItemTypes`, `loadAttachments`, +`loadItem`, `loadAttachment`, `mappers`, `reports`, `processedFiles`) lives on +`LoadingAdapter`. Shared: `event`, `state`, `sdkState`, `extractionScope`, +`postState`, `isTimeout`. + +> Only the `WorkerAdapter` **class** was removed. The **types** +> `WorkerAdapterInterface` and `WorkerAdapterOptions` still exist and are still +> exported, unchanged. Don't do a blind global rename of every `WorkerAdapter` +> token — replace the class/`WorkerAdapter` annotations only. + +> Symbols that previously required a deep `dist/` import are now on the root +> barrel: `Mappers` (class), `ItemTypeToLoad`, and `Item` all import from +> `@devrev/airsync-sdk` directly. `ToDevRev` is **not** exported — it is +> SDK-internal now (drop any use of it; see §7). + +## 9. SDK loading/streaming methods return a `TaskResult` + +`loadItemTypes`, `loadAttachments` and `streamAttachments` no longer emit or +exit mid-flight — they return a `TaskResult` you simply pass through: + +```ts +// v1 +processTask({ + task: async ({ adapter }) => { + const { reports, processed_files } = await adapter.loadItemTypes({ itemTypesToLoad }); + await adapter.emit(LoaderEventType.DataLoadingDone, { reports, processed_files }); + }, + onTimeout: async ({ adapter }) => { + await adapter.emit(LoaderEventType.DataLoadingProgress, { + reports: adapter.reports, + processed_files: adapter.processedFiles, + }); + }, +}); +``` + +```ts +// v2 +processLoadingTask({ + task: async ({ adapter }) => { + return adapter.loadItemTypes({ itemTypesToLoad }); + }, +}); +``` + +Rate limits (→ `delay`), timeouts (→ `progress`), errors (→ `error`) and +completion (→ `success`) are all encoded in the returned result; `reports` and +`processed_files` are attached to the emitted event automatically. + +Attachment extraction: + +```ts +// v1 +const response = await adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +if (response?.delay) { + await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { delay: response.delay }); +} else if (response?.error) { + await adapter.emit(ExtractorEventType.AttachmentExtractionError, { error: response.error }); +} else { + await adapter.emit(ExtractorEventType.AttachmentExtractionDone); +} +``` + +```ts +// v2 +return adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +``` + +Custom attachment `processors` (reducer/iterator) are still supported with the +same signatures. + +> **Edge regression — very old in-flight attachment syncs.** In v1 the SDK +> migrated the legacy `string[]` form of the processed-attachments dedup list +> (`sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList`) to the +> current `{ id, parent_id }[]` form on read. v2 removed that conversion. The +> `string[]` form only exists in state written by SDK **< 1.15.2**. If an +> attachment-extraction phase started on a pre-1.15.2 SDK and is *still mid-flight* +> when the connector upgrades to v2, the v2 dedup check (`it.id === …`) silently +> fails to match the bare-string entries, so attachments already downloaded in +> that sync get re-uploaded once. New syncs (and any sync started on ≥ 1.15.2) are +> unaffected. The duplicates are deduplicated downstream by the platform, so the +> only cost is wasted re-download/upload work on that one continuation. + +## 10. Deleted legacy modules + +Everything under the v1 `deprecated/` tree is gone: `Adapter`, +`createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, and the +legacy `Uploader` export. Replacements: the v2 adapters + `process*Task` for +`Adapter`/`createAdapter`; `axiosClient` for `HTTPClient`; repos for the legacy +uploader. (`formatAxiosError` is still exported.) + +## 11. Migration checklist + +1. `npm uninstall @devrev/ts-adaas && npm install @devrev/airsync-sdk`; replace import specifiers. +2. Rename `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage`. +3. Replace deleted enum members with their new names (§3) — values unchanged. +4. Split workers: extraction files use `processExtractionTask`, loading files use `processLoadingTask`. +5. Convert every `adapter.emit(...)` into a returned `TaskResult` (§4–5); bubble outcomes up from helpers. +6. ESU workers: push external sync units to the `EXTERNAL_SYNC_UNITS` repo (§6). +7. Replace `WorkerAdapter` annotations with `ExtractionAdapter` / `LoadingAdapter`. +8. Remove reads/writes of SDK-owned state fields from `adapter.state`; use the event context's `extract_from`/`extract_to` (§7). +9. Pass through the `TaskResult` from `loadItemTypes`/`loadAttachments`/`streamAttachments` (§9). +10. Decide per worker whether you need an explicit `onTimeout` (recommended for ESU/metadata to control the error message; usually omit elsewhere). +11. Remove deep `dist/**` imports and any usage of deleted legacy modules. +12. Update jest mocks of the SDK module (they hardcode the v1 shape: `processTask`, `WorkerAdapter`, old enum members). diff --git a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts index 9a22ea08..03472492 100644 --- a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts +++ b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts @@ -1,7 +1,5 @@ -import { - ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, -} from '../types'; +import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; export interface AttachmentsStreamingPoolParams { diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index 59ce3648..39f2348a 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -1,10 +1,9 @@ -import { sleep } from '../common/helpers'; import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, ProcessAttachmentReturnType, -} from '../types'; +} from '../types/extraction'; import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces'; export class AttachmentsStreamingPool { @@ -34,8 +33,10 @@ export class AttachmentsStreamingPool { this.totalProcessedCount++; if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) { console.info(`Processed ${this.totalProcessedCount} attachments so far.`); - // Sleep for 100ms to avoid blocking the event loop - await sleep(100); + // Yield once to the event loop so a pending soft-timeout message + // (WorkerMessageExit) can be delivered and adapter.isTimeout can flip + // before the next batch of attachments is processed. + await new Promise((resolve) => setImmediate(resolve)); } } diff --git a/src/common/helpers.ts b/src/common/helpers.ts index e4abeef9..afb1b464 100644 --- a/src/common/helpers.ts +++ b/src/common/helpers.ts @@ -37,7 +37,6 @@ export function getLibraryVersion() { * @returns {Promise} A promise that resolves after the given number of milliseconds */ export async function sleep(ms: number) { - console.log(`Sleeping for ${ms}ms.`); return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/src/common/test-utils.ts b/src/common/test-utils.ts index c9b8b2ba..9e4372ed 100644 --- a/src/common/test-utils.ts +++ b/src/common/test-utils.ts @@ -63,6 +63,10 @@ export function createMockEvent( }, snap_in_version_id: 'test_snap_in_version_id', snap_in_id: 'test_snap_in_id', + user_id: 'test_user_id', + dev_oid: 'test_dev_oid', + source_id: 'test_source_id', + service_account_id: 'test_service_account_id', }, payload: { connection_data: { diff --git a/src/http/index.ts b/src/http/index.ts deleted file mode 100644 index 4edc2c55..00000000 --- a/src/http/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './axios-client'; -export * from './types'; diff --git a/src/index.ts b/src/index.ts index f079f05e..ebc9dd6b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,23 +1,183 @@ +// ──────────────────────────────────────────────────────────────────────────── +// Public API barrel for @devrev/airsync-sdk. +// +// This is the SINGLE source of the public surface — every exported symbol is +// named explicitly (no `export *`), so any change to the public API shows up as +// a diff here. Adding a symbol to the package's public contract means adding it +// to this file. +// ──────────────────────────────────────────────────────────────────────────── + +// ── Entry points & adapters ── +export { + processExtractionTask, + processLoadingTask, +} from './multithreading/process-task'; +export { spawn } from './multithreading/spawn/spawn'; +export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter'; +export { LoadingAdapter } from './multithreading/adapters/loading-adapter'; + +// ── Worker contract types ── +export type { + ProcessTaskInterface, + TaskAdapterInterface, + TaskResult, + TaskStatus, + WorkerAdapterInterface, + WorkerAdapterOptions, + SpawnFactoryInterface, + SpawnInterface, + ExtractionScope, + WorkerPathOverrides, +} from './types/workers'; + +// ── Constants & enums ── export { AirSyncDefaultItemTypes } from './common/constants'; +export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants'; export { ExtractionCommonError } from './common/errors'; -export * from './common/install-initial-domain-mapping'; -export * from './http'; + +// ── Domain mapping install ── +export { installInitialDomainMapping } from './common/install-initial-domain-mapping'; + +// ── HTTP client ── +export { axios, axiosClient } from './http/axios-client'; + +// ── Error formatting ── export { formatAxiosError, serializeAxiosError } from './logger/logger'; + +// ── Common types ── +export { ErrorLevel, SyncMode } from './types/common'; +export type { + AdapterUpdateParams, + ErrorRecord, + InitialDomainMapping, + LogRecord, +} from './types/common'; + +// ── Extraction types ── +export { + EventType, + ExtractionMode, + ExtractorEventType, + InitialSyncScope, + TimeUnit, + TimeValueType, +} from './types/extraction'; +export type { + AirSyncEvent, + AirSyncMessage, + ConnectionData, + DomainObjectState, + EventContext, + EventContextIn, + EventContextOut, + EventData, + ExternalProcessAttachmentFunction, + ExternalSyncUnit, + ExternalSystemAttachmentIteratorFunction, + ExternalSystemAttachmentReducerFunction, + ExternalSystemAttachmentStreamingFunction, + ExternalSystemAttachmentStreamingParams, + ExternalSystemAttachmentStreamingResponse, + ExtractorEvent, + ProcessAttachmentReturnType, + TimeValue, +} from './types/extraction'; + +// ── Loading types ── +export { LoaderEventType } from './types/loading'; +export type { + ExternalSystemAttachment, + ExternalSystemItem, + ExternalSystemItemLoadingParams, + ExternalSystemItemLoadingResponse, + ItemTypeToLoad, +} from './types/loading'; + +// ── Repo types ── +export type { + Item, + NormalizedAttachment, + NormalizedItem, + RepoInterface, +} from './repo/repo.interfaces'; + +// ── Mappers ── +export { Mappers } from './mappers/mappers'; +export { + SyncMapperRecordStatus, + SyncMapperRecordTargetType, +} from './mappers/mappers.interface'; +export type { + MappersCreateParams, + MappersGetByExternalIdParams, + MappersGetByTargetIdParams, + MappersUpdateParams, +} from './mappers/mappers.interface'; + +// ── State types ── +export type { AdapterState } from './state/state.interfaces'; + +// ── Uploader types ── +export type { + Artifact, + ArtifactsPrepareResponse, + SsorAttachment, + StreamAttachmentsResponse, + StreamResponse, + UploadResponse, +} from './uploader/uploader.interfaces'; + +// ── External domain metadata types ── +export type { + CollectionData, + ConditionalPrivilegeData, + CustomLinkData, + CustomLinkNames, + CustomStage, + CustomState, + EnumData, + EnumValue, + EnumValueKey, + ExternalDomainMetadata, + Field, + FieldCondition, + FieldConditionComparator, + FieldConditionEffect, + FieldConditions, + FieldKey, + FieldPrivilegeData, + FieldReferenceData, + FieldType, + FloatData, + IntData, + PermissionData, + RecordType, + RecordTypeCategory, + RecordTypeCategoryKey, + RecordTypeKey, + RecordTypePrivilegeData, + RecordTypeScope, + ReferenceData, + ReferenceDetail, + ReferenceType, + SchemaVersion, + StageKey, + StageDiagram, + StateKey, + StructData, + StructTypeKey, + StructType, + TargetTypeKeyData, + TextData, + TypedReferenceData, +} from './types/external-domain-metadata'; + +// ── Testing utilities (public test-support surface) ── export { MockServer } from './mock-server/mock-server'; export type { RequestInfo, RetryConfig, RouteConfig, } from './mock-server/mock-server.interfaces'; -export { - processExtractionTask, - processLoadingTask, -} from './multithreading/process-task'; -export { spawn } from './multithreading/spawn/spawn'; -export { BaseAdapter } from './multithreading/adapters/base-adapter'; -export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter'; -export { LoadingAdapter } from './multithreading/adapters/loading-adapter'; export { createMockEvent, MOCK_SERVER_DEFAULT_URL } from './common/test-utils'; export type { DeepPartial } from './common/test-utils'; -export * from './types'; -export * from './types/workers'; diff --git a/src/mappers/mappers.interface.ts b/src/mappers/mappers.interface.ts index 7d03c60b..f601ef98 100644 --- a/src/mappers/mappers.interface.ts +++ b/src/mappers/mappers.interface.ts @@ -1,4 +1,4 @@ -import { AirSyncEvent } from '../types'; +import { AirSyncEvent } from '../types/extraction'; import { DonV2 } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index a43debbb..e59a789c 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -30,6 +30,11 @@ import { LoadingAdapter } from './adapters/loading-adapter'; * If `onTimeout` is omitted, the SDK emits a phase-appropriate default on * timeout: `progress` (resumable phases) or `error` (non-resumable phases) is * handled by the status->event mapping when we emit a `progress` result. + * + * On timeout the timeout outcome always wins: once the soft timeout has fired, + * the SDK emits the `onTimeout` result (or the default `progress`) and ignores + * whatever the task returned, because a phase that ran out of time must hand off + * for continuation rather than report itself complete. */ async function runWorkerTask>( buildAdapter: () => Promise, @@ -51,16 +56,16 @@ async function runWorkerTask>( task({ adapter }) ); - // On timeout, hand off to onTimeout (or default to a progress result). - if (adapter.isTimeout && !adapter.hasWorkerEmitted) { + // If the soft timeout fired, the timeout outcome wins regardless of what + // the task returned: hand off via onTimeout (or the default progress + // result) so the phase continues in a fresh invocation. + if (adapter.isTimeout) { result = onTimeout ? await runWithUserLogContext(async () => onTimeout({ adapter })) : { status: 'progress' }; } - if (!adapter.hasWorkerEmitted) { - await adapter.emitFromResult(result); - } + await adapter.emitFromResult(result); process.exit(0); } catch (error) { diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 9b3a7c70..95ad3789 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -349,6 +349,14 @@ export interface AirSyncEvent { }; snap_in_version_id: string; snap_in_id: string; + /** DevRev identity of the user who triggered the sync. */ + user_id: string; + /** DevRev org id (don:identity:.../devo/...). */ + dev_oid: string; + /** External source identity, when the platform provides one. */ + source_id: string; + /** DevRev service-account identity used for the sync. */ + service_account_id: string; }; payload: AirSyncMessage; execution_metadata: { diff --git a/src/types/index.ts b/src/types/index.ts deleted file mode 100644 index f98c7251..00000000 --- a/src/types/index.ts +++ /dev/null @@ -1,126 +0,0 @@ -// Common -export { - AdapterUpdateParams, - ErrorLevel, - ErrorRecord, - InitialDomainMapping, - LogRecord, - SyncMode, -} from './common'; - -// Extraction -export { - AirSyncEvent, - AirSyncMessage, - ConnectionData, - DomainObjectState, - EventContextIn, - EventContextOut, - EventContext, - EventData, - EventType, - ExternalProcessAttachmentFunction, - ExternalSyncUnit, - ExternalSystemAttachmentIteratorFunction, - ExternalSystemAttachmentReducerFunction, - ExternalSystemAttachmentStreamingFunction, - ExternalSystemAttachmentStreamingParams, - ExternalSystemAttachmentStreamingResponse, - ExtractionMode, - ExtractorEvent, - ExtractorEventType, - InitialSyncScope, - ProcessAttachmentReturnType, - TimeUnit, - TimeValue, - TimeValueType, -} from './extraction'; - -// Loading -export { - ExternalSystemAttachment, - ExternalSystemItem, - ExternalSystemItemLoadingParams, - ExternalSystemItemLoadingResponse, - LoaderEventType, -} from './loading'; - -// Repo -export { - NormalizedAttachment, - NormalizedItem, - RepoInterface, -} from '../repo/repo.interfaces'; - -// State -export { AdapterState } from '../state/state.interfaces'; - -export { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; - -// Uploader -export { - Artifact, - ArtifactsPrepareResponse, - SsorAttachment, - StreamAttachmentsResponse, - StreamResponse, - UploadResponse, -} from '../uploader/uploader.interfaces'; - -// Mappers -export type { - MappersCreateParams, - MappersGetByExternalIdParams, - MappersGetByTargetIdParams, - MappersUpdateParams, -} from '../mappers/mappers.interface'; - -export { - SyncMapperRecordStatus, - SyncMapperRecordTargetType, -} from '../mappers/mappers.interface'; - -// External Domain Metadata -export type { - CollectionData, - ConditionalPrivilegeData, - CustomLinkData, - CustomLinkNames, - CustomStage, - CustomState, - EnumData, - EnumValue, - EnumValueKey, - ExternalDomainMetadata, - Field, - FieldCondition, - FieldConditionComparator, - FieldConditionEffect, - FieldConditions, - FieldKey, - FieldPrivilegeData, - FieldReferenceData, - FieldType, - FloatData, - IntData, - PermissionData, - RecordType, - RecordTypeCategory, - RecordTypeCategoryKey, - RecordTypeKey, - RecordTypePrivilegeData, - RecordTypeScope, - ReferenceData, - ReferenceDetail, - ReferenceType, - SchemaVersion, - StageKey, - StageDiagram, - StateKey, - StructData, - StructTypeKey, - StructType, - TargetTypeKeyData, - TextData, - TypedReferenceData, -} from './external-domain-metadata'; From 034f8028baf48f5cf8e4ad13b38453d253cc2fef Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 16 Jun 2026 12:57:46 +0200 Subject: [PATCH 24/30] refactor(v2): remove axios from the public surface Stop exposing axios as public API while keeping it as an internal dependency. The internal retry-enabled client (http/axios-client-internal) is unchanged and still backs uploader, emit, state, mappers and IDM. Public-surface changes (all v2 breaking): - Delete the public `axios`/`axiosClient` exports and the src/http/axios-client.ts file. Connectors needing axios import it directly. - Replace `formatAxiosError`/`serializeAxiosError` barrel exports with a single `serializeError(unknown): string`. `serializeAxiosError` stays exported off-barrel (used internally + by its unit tests). - Introduce `HttpStreamResponse` (minimal structural supertype of AxiosResponse: `{ data; headers }`) and use it on the public `ExternalSystemAttachmentStreamingResponse.httpStream` and the Uploader/ExtractionAdapter stream params, so AxiosResponse no longer leaks through public type signatures. - Mappers methods now unwrap `.data` and return the payload types directly instead of AxiosResponse envelopes; loading-adapter call sites updated. Drive-by: remove the dead duplicate `truncateFilename` from common/helpers (the live copy is in uploader/uploader.helpers), and delete the unused http/constants.ts and http/types.ts (no remaining importers). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/common/helpers.ts | 30 -------- src/http/axios-client.ts | 68 ------------------- src/http/constants.ts | 3 - src/http/types.ts | 16 ----- src/index.ts | 6 +- src/logger/logger.ts | 11 --- src/mappers/mappers.ts | 34 +++++----- .../adapters/extraction-adapter.ts | 5 +- .../adapters/loading-adapter.ts | 42 ++++++------ src/types/extraction.ts | 10 ++- src/uploader/uploader.ts | 11 +-- 11 files changed, 54 insertions(+), 182 deletions(-) delete mode 100644 src/http/axios-client.ts delete mode 100644 src/http/constants.ts delete mode 100644 src/http/types.ts diff --git a/src/common/helpers.ts b/src/common/helpers.ts index afb1b464..bd73b8c8 100644 --- a/src/common/helpers.ts +++ b/src/common/helpers.ts @@ -2,10 +2,6 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import * as v8 from 'v8'; -import { - MAX_DEVREV_FILENAME_EXTENSION_LENGTH, - MAX_DEVREV_FILENAME_LENGTH, -} from './constants'; import { MAX_LOG_STRING_LENGTH } from '../logger/logger.constants'; /** @@ -40,32 +36,6 @@ export async function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } -/** - * Truncates a filename if it exceeds the maximum allowed length. - * @param {string} filename - The filename to truncate - * @returns {string} The truncated filename - */ -export function truncateFilename(filename: string): string { - // If the filename is already within the limit, return it as is. - if (filename.length <= MAX_DEVREV_FILENAME_LENGTH) { - return filename; - } - - console.warn( - `Filename length exceeds the maximum limit of ${MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.` - ); - - const extension = filename.slice(-MAX_DEVREV_FILENAME_EXTENSION_LENGTH); - // Calculate how many characters are available for the name part after accounting for the extension and "..." - const availableNameLength = - MAX_DEVREV_FILENAME_LENGTH - MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..." - - // Truncate the name part and add an ellipsis - const truncatedFilename = filename.slice(0, availableNameLength); - - return `${truncatedFilename}...${extension}`; -} - /** * MemoryInfo is an interface that represents the memory usage information. * @interface MemoryInfo diff --git a/src/http/axios-client.ts b/src/http/axios-client.ts deleted file mode 100644 index 703b1e59..00000000 --- a/src/http/axios-client.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Axios client setup with retry capabilities using axios-retry. - * - * This module exports an Axios client instance (`axiosClient`) that is configured to automatically retry - * failed requests under certain conditions. - * - * Retry Conditions: - * 1. Network errors (where no response is received). - * 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT). - * 3. All 5xx server errors. - * - * Retry Strategy: - * - A maximum of 5 retries are attempted. - * - Exponential backoff delay is applied between retries, increasing with each retry attempt. - * - * Additional Features: - * - When the maximum number of retry attempts is reached, sensitive headers (like authorization) - * are removed from error logs for security reasons. - * - * Exported: - * - `axios`: Original axios instance for additional customizations or direct use. - * - `axiosClient`: Configured axios instance with retry logic. - */ - -import axios, { AxiosError } from 'axios'; -import axiosRetry from 'axios-retry'; -import { runWithUserLogContext } from '../logger/logger.context'; - -const axiosClient = axios.create(); - -axiosRetry(axiosClient, { - retries: 5, - retryDelay: (retryCount, error) => { - // exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms - const delay = axiosRetry.exponentialDelay(retryCount, error, 1000); - - runWithUserLogContext(() => - console.warn( - `Request to ${error.config?.url} failed with response status code ${ - error.response?.status - }. Method ${ - error.config?.method - }. Retry count: ${retryCount}. Retrying in ${Math.round( - delay / 1000 - )}s.` - ) - ); - - return delay; - }, - retryCondition: (error: AxiosError) => { - return ( - (axiosRetry.isNetworkOrIdempotentRequestError(error) && - error.response?.status !== 429) || - (error.response?.status ?? 0) >= 500 - ); - }, - onMaxRetryTimesExceeded(error: AxiosError) { - delete error.config?.headers?.authorization; - delete error.config?.headers?.Authorization; - delete error.request._header; - runWithUserLogContext(() => - console.warn('Max retry times exceeded. Error', error) - ); - }, -}); - -export { axios, axiosClient }; diff --git a/src/http/constants.ts b/src/http/constants.ts deleted file mode 100644 index 25cfbf91..00000000 --- a/src/http/constants.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const RATE_LIMIT_EXCEEDED = 'LIMIT_EXCEEDED'; -export const LAMBDA_LIMIT_EXCEEDED = 'LAMBDA_LIMIT_EXCEEDED'; -export const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429; diff --git a/src/http/types.ts b/src/http/types.ts deleted file mode 100644 index 3206229a..00000000 --- a/src/http/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * HTTP Response type - * @deprecated - */ -export type HTTPResponse = { - success: boolean; - message: string; - data: Data; -}; - -interface Data { - records: object[]; // List of records of the entity - delay: number; // Delay in seconds(used for ratelimiting), Time to wait before next call - nextPage?: number; // The next page of the entity to be processed - metadata?: object; // Other information that should be returned -} diff --git a/src/index.ts b/src/index.ts index ebc9dd6b..c9d81310 100644 --- a/src/index.ts +++ b/src/index.ts @@ -38,11 +38,8 @@ export { ExtractionCommonError } from './common/errors'; // ── Domain mapping install ── export { installInitialDomainMapping } from './common/install-initial-domain-mapping'; -// ── HTTP client ── -export { axios, axiosClient } from './http/axios-client'; - // ── Error formatting ── -export { formatAxiosError, serializeAxiosError } from './logger/logger'; +export { serializeError } from './logger/logger'; // ── Common types ── export { ErrorLevel, SyncMode } from './types/common'; @@ -79,6 +76,7 @@ export type { ExternalSystemAttachmentStreamingParams, ExternalSystemAttachmentStreamingResponse, ExtractorEvent, + HttpStreamResponse, ProcessAttachmentReturnType, TimeValue, } from './types/extraction'; diff --git a/src/logger/logger.ts b/src/logger/logger.ts index 6d06ef09..a7204d3f 100644 --- a/src/logger/logger.ts +++ b/src/logger/logger.ts @@ -232,14 +232,3 @@ export function serializeAxiosError(error: AxiosError): AxiosErrorResponse { return serializedAxiosError; } - -/** - * Formats an Axios error to a printable format. - * - * @param error - Axios error to format - * @returns Formatted error object - * @deprecated Use {@link serializeAxiosError} instead - */ -export function formatAxiosError(error: AxiosError): object { - return serializeAxiosError(error); -} diff --git a/src/mappers/mappers.ts b/src/mappers/mappers.ts index c552b02c..1238c5c9 100644 --- a/src/mappers/mappers.ts +++ b/src/mappers/mappers.ts @@ -1,5 +1,3 @@ -import { AxiosResponse } from 'axios'; - import { axiosClient } from '../http/axios-client-internal'; import { @@ -34,13 +32,13 @@ export class Mappers { * Used to find the mapping when you know the DevRev ID and want to find the external system ID. * * @param params - Query parameters of type MappersGetByTargetIdParams - * @returns Promise with response data containing the sync mapper record + * @returns Promise resolving to the sync mapper record */ async getByTargetId( params: MappersGetByTargetIdParams - ): Promise> { + ): Promise { const { sync_unit, target } = params; - return axiosClient.get( + const response = await axiosClient.get( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-target`, { headers: { @@ -49,6 +47,7 @@ export class Mappers { params: { sync_unit, target }, } ); + return response.data; } /** @@ -57,13 +56,13 @@ export class Mappers { * Used to find the mapping when you know the external system ID and want to find the DevRev ID. * * @param params - Query parameters of type MappersGetByExternalIdParams - * @returns Promise with response data containing the sync mapper record + * @returns Promise resolving to the sync mapper record */ async getByExternalId( params: MappersGetByExternalIdParams - ): Promise> { + ): Promise { const { sync_unit, external_id, target_type } = params; - return axiosClient.get( + const response = await axiosClient.get( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-external-id`, { headers: { @@ -72,6 +71,7 @@ export class Mappers { params: { sync_unit, external_id, target_type }, } ); + return response.data; } /** @@ -82,12 +82,10 @@ export class Mappers { * the mapping for future synchronization operations. * * @param params - Creation parameters of type MappersCreateParams - * @returns Promise with response data containing the created sync mapper record + * @returns Promise resolving to the created sync mapper record */ - async create( - params: MappersCreateParams - ): Promise> { - return axiosClient.post( + async create(params: MappersCreateParams): Promise { + const response = await axiosClient.post( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.create`, params, { @@ -96,6 +94,7 @@ export class Mappers { }, } ); + return response.data; } /** @@ -105,12 +104,10 @@ export class Mappers { * additional DevRev entities need to be associated. * * @param params - Update parameters of type MappersUpdateParams - * @returns Promise with response data containing the updated sync mapper record + * @returns Promise resolving to the updated sync mapper record */ - async update( - params: MappersUpdateParams - ): Promise> { - return axiosClient.post( + async update(params: MappersUpdateParams): Promise { + const response = await axiosClient.post( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.update`, params, { @@ -119,5 +116,6 @@ export class Mappers { }, } ); + return response.data; } } diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index 5bbbffa0..ea4f21ac 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -1,5 +1,3 @@ -import { AxiosResponse } from 'axios'; - import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; import { AirSyncDefaultItemTypes, @@ -22,6 +20,7 @@ import { ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, + HttpStreamResponse, ProcessAttachmentReturnType, } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; @@ -319,7 +318,7 @@ export class ExtractionAdapter< * Destroys a stream to prevent memory leaks. * @param httpStream - The axios response stream to destroy */ - private destroyHttpStream(httpStream: AxiosResponse): void { + private destroyHttpStream(httpStream: HttpStreamResponse): void { try { if (httpStream && httpStream.data) { if (typeof httpStream.data.destroy === 'function') { diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts index 49fdd2bf..6493d9be 100644 --- a/src/multithreading/adapters/loading-adapter.ts +++ b/src/multithreading/adapters/loading-adapter.ts @@ -355,12 +355,11 @@ export class LoadingAdapter< const devrevId = item.id.devrev; try { - const syncMapperRecordResponse = await this._mappers.getByTargetId({ + const syncMapperRecord = await this._mappers.getByTargetId({ sync_unit: this.event.payload.event_context.sync_unit, target: devrevId, }); - const syncMapperRecord = syncMapperRecordResponse.data; if (!syncMapperRecord) { console.warn('Failed to get sync mapper record from response.'); return { @@ -383,7 +382,7 @@ export class LoadingAdapter< if (id) { try { - const syncMapperRecordUpdateResponse = await this._mappers.update({ + const syncMapperRecordUpdate = await this._mappers.update({ id: syncMapperRecord.sync_mapper_record.id, sync_unit: this.event.payload.event_context.sync_unit, status: SyncMapperRecordStatus.OPERATIONAL, @@ -407,7 +406,7 @@ export class LoadingAdapter< console.log( 'Successfully updated sync mapper record.', - syncMapperRecordUpdateResponse.data + syncMapperRecordUpdate ); } catch (error) { console.warn( @@ -465,25 +464,24 @@ export class LoadingAdapter< if (id) { // Create mapper try { - const syncMapperRecordCreateResponse = - await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - external_ids: [id], - targets: [devrevId], - ...(modifiedDate && { - external_versions: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }), - }); + const syncMapperRecordCreate = await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + external_ids: [id], + targets: [devrevId], + ...(modifiedDate && { + external_versions: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }), + }); console.log( 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data + syncMapperRecordCreate ); return { @@ -575,7 +573,7 @@ export class LoadingAdapter< }; } else if (id) { try { - const syncMapperRecordCreateResponse = await this._mappers.create({ + const syncMapperRecordCreate = await this._mappers.create({ sync_unit: this.event.payload.event_context.sync_unit, external_ids: [id], targets: [item.reference_id], @@ -584,7 +582,7 @@ export class LoadingAdapter< console.log( 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data + syncMapperRecordCreate ); } catch (error) { console.warn( diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 95ad3789..fba35a1b 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -4,7 +4,6 @@ import { Artifact } from '../uploader/uploader.interfaces'; import { ErrorRecord } from './common'; -import { AxiosResponse } from 'axios'; import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; import { DonV2, LoaderReport, RateLimited } from './loading'; @@ -406,8 +405,15 @@ export interface ExternalSystemAttachmentStreamingParams { event: AirSyncEvent; } +export interface HttpStreamResponse { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + headers: Record; +} + export interface ExternalSystemAttachmentStreamingResponse { - httpStream?: AxiosResponse; + httpStream?: HttpStreamResponse; error?: ErrorRecord; delay?: number; } diff --git a/src/uploader/uploader.ts b/src/uploader/uploader.ts index fa9833af..f35612cf 100644 --- a/src/uploader/uploader.ts +++ b/src/uploader/uploader.ts @@ -5,6 +5,7 @@ import { axiosClient } from '../http/axios-client-internal'; import { MAX_DEVREV_ARTIFACT_SIZE } from '../common/constants'; import { NormalizedAttachment } from '../repo/repo.interfaces'; +import { HttpStreamResponse } from '../types/extraction'; import { serializeError } from '../logger/logger'; import { @@ -193,14 +194,14 @@ export class Uploader { } /** - * Streams an artifact file from an axios response to the upload URL. + * Streams an artifact file from an HTTP streaming response to the upload URL. * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data - * @param {AxiosResponse} fileStream - The axios response stream containing the file data + * @param {HttpStreamResponse} fileStream - The HTTP streaming response containing the file data * @returns {Promise} The axios response or undefined on error */ async streamArtifact( artifact: ArtifactToUpload, - fileStream: AxiosResponse + fileStream: HttpStreamResponse ): Promise> { const formData = new FormData(); for (const field in artifact.form_data) { @@ -274,9 +275,9 @@ export class Uploader { /** * Destroys a stream to prevent resource leaks. - * @param {any} fileStream - The axios response stream to destroy + * @param {HttpStreamResponse} fileStream - The HTTP streaming response to destroy */ - private destroyStream(fileStream: AxiosResponse): void { + private destroyStream(fileStream: HttpStreamResponse): void { try { if (fileStream && fileStream.data) { // For axios response streams, the data property contains the actual stream From f0de116fd7e07b7c3c22a229c652dc29081d4584 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 16 Jun 2026 13:13:22 +0200 Subject: [PATCH 25/30] =?UTF-8?q?refactor(v2):=20structural=20cleanup=20?= =?UTF-8?q?=E2=80=94=20moves,=20renames,=20common/=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure file moves/renames + import-path rewrites. No behavior changes; the public barrel's exported symbol set is unchanged (paths only). Renames: - http/axios-client-internal.ts -> http/client.ts (+ .test) (drop redundant '-internal'; barrel-absence already means internal) - logger/logger.context.ts -> logger/logger-context.ts - logger/logger.constants.ts -> logger/logger-constants.ts - mappers/mappers.interface.ts -> mappers/mappers.interfaces.ts common/ decomposition (by single-owner rule; common/ now = constants + helpers): - install-initial-domain-mapping.{ts,test} -> state/ - time-value-resolver.{ts,test} -> state/ - errors.ts -> types/errors.ts New testing/ module (consolidates all public test-support surface): - mock-server/{mock-server,mock-server.interfaces,README} -> testing/ - common/test-utils.ts split: createMockEvent + DeepPartial -> testing/mock-event.ts; MOCK_SERVER_DEFAULT_URL -> testing/mock-server.ts Deleted: src/multithreading/worker-adapter/ (6 dead test files referencing the pre-refactor WorkerAdapter/State/AirdropEvent symbols removed in the adapter split; replacements get written in the deferred test pass). Allowed companion-file suffixes are now exactly {.interfaces,.helpers,.test}; the one-off .context/.constants suffixes are gone. Verified: production source typechecks and builds clean; no stale references to any moved path. Tests remain intentionally broken from the prior adapter split (they reference deleted modules/symbols) and are fixed in a later pass; this commit introduces no new module-resolution breakage. Co-Authored-By: Claude Opus 4.8 (1M context) --- jest.config.cjs | 4 +- src/common/helpers.ts | 2 +- ...client-internal.test.ts => client.test.ts} | 2 +- .../{axios-client-internal.ts => client.ts} | 0 src/index.ts | 16 +- ...ogger.constants.ts => logger-constants.ts} | 0 .../{logger.context.ts => logger-context.ts} | 0 src/logger/logger.test.ts | 4 +- src/logger/logger.ts | 4 +- ...ers.interface.ts => mappers.interfaces.ts} | 0 src/mappers/mappers.test.ts | 9 +- src/mappers/mappers.ts | 4 +- src/multithreading/adapters/base-adapter.ts | 2 +- .../adapters/extraction-adapter.ts | 2 +- .../adapters/loading-adapter.ts | 4 +- src/multithreading/create-worker.test.ts | 2 +- src/multithreading/emit.ts | 2 +- src/multithreading/process-task.test.ts | 4 +- src/multithreading/process-task.ts | 2 +- src/multithreading/spawn/spawn.test.ts | 2 +- .../worker-adapter.artifacts.test.ts | 224 ---- .../worker-adapter.emit.test.ts | 593 ---------- .../worker-adapter.extraction.test.ts | 1034 ----------------- .../worker-adapter.helpers.test.ts | 592 ---------- .../worker-adapter.loading.test.ts | 729 ------------ .../worker-adapter.serialization.test.ts | 83 -- src/multithreading/worker.js | 2 +- src/repo/repo.test.ts | 2 +- src/repo/repo.ts | 2 +- src/state/base-state.ts | 4 +- src/state/extraction-state.ts | 2 +- .../install-initial-domain-mapping.test.ts | 8 +- .../install-initial-domain-mapping.ts | 2 +- src/state/state.extract-window.test.ts | 2 +- src/state/state.pending-boundaries.test.ts | 4 +- src/state/state.post-state.test.ts | 4 +- src/state/state.test.ts | 4 +- src/state/state.time-value-resolution.test.ts | 2 +- .../time-value-resolver.test.ts | 2 +- src/{common => state}/time-value-resolver.ts | 2 +- src/{mock-server => testing}/README.md | 0 .../test-utils.ts => testing/mock-event.ts} | 2 +- .../mock-server.interfaces.ts | 0 src/{mock-server => testing}/mock-server.ts | 6 + .../control-protocol-reconciliation.test.ts | 2 +- .../dummy-connector/data-extraction.test.ts | 2 +- .../external-sync-units-extraction.test.ts | 2 +- .../metadata-extraction.test.ts | 2 +- .../size-limit-1.test.ts | 2 +- src/tests/extract-from-collision.test.ts | 2 +- src/tests/jest.setup.ts | 2 +- .../spawn-worker/delete-event-type.test.ts | 2 +- .../spawn-worker/unknown-event-type.test.ts | 2 +- src/tests/test-helpers.interfaces.ts | 2 +- src/tests/timeout-handling/no-timeout.test.ts | 2 +- .../timeout-handling/timeout-blocked.test.ts | 2 +- .../timeout-handling/timeout-graceful.test.ts | 2 +- .../timeout-unblocked.test.ts | 2 +- src/{common => types}/errors.ts | 0 src/types/extraction.test.ts | 2 +- src/uploader/uploader.test.ts | 6 +- src/uploader/uploader.ts | 2 +- 62 files changed, 80 insertions(+), 3328 deletions(-) rename src/http/{axios-client-internal.test.ts => client.test.ts} (99%) rename src/http/{axios-client-internal.ts => client.ts} (100%) rename src/logger/{logger.constants.ts => logger-constants.ts} (100%) rename src/logger/{logger.context.ts => logger-context.ts} (100%) rename src/mappers/{mappers.interface.ts => mappers.interfaces.ts} (100%) delete mode 100644 src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts delete mode 100644 src/multithreading/worker-adapter/worker-adapter.emit.test.ts delete mode 100644 src/multithreading/worker-adapter/worker-adapter.extraction.test.ts delete mode 100644 src/multithreading/worker-adapter/worker-adapter.helpers.test.ts delete mode 100644 src/multithreading/worker-adapter/worker-adapter.loading.test.ts delete mode 100644 src/multithreading/worker-adapter/worker-adapter.serialization.test.ts rename src/{common => state}/install-initial-domain-mapping.test.ts (97%) rename src/{common => state}/install-initial-domain-mapping.ts (97%) rename src/{common => state}/time-value-resolver.test.ts (99%) rename src/{common => state}/time-value-resolver.ts (98%) rename src/{mock-server => testing}/README.md (100%) rename src/{common/test-utils.ts => testing/mock-event.ts} (98%) rename src/{mock-server => testing}/mock-server.interfaces.ts (100%) rename src/{mock-server => testing}/mock-server.ts (97%) rename src/{common => types}/errors.ts (100%) diff --git a/jest.config.cjs b/jest.config.cjs index be2cf47a..fab696c9 100644 --- a/jest.config.cjs +++ b/jest.config.cjs @@ -11,7 +11,7 @@ module.exports = { 'src/tests/timeout-handling/.*.ts', // These tests are slow (10-15s per) 'src/tests/dummy-connector/metadata-extraction.test.ts', - 'src/http/axios-client-internal.test.ts', + 'src/http/client.test.ts', 'src/tests/event-data-size-limit/.*.test.ts', ], }, @@ -31,7 +31,7 @@ module.exports = { preset: 'ts-jest', testMatch: [ '/src/tests/dummy-connector/metadata-extraction.test.ts', - '/src/http/axios-client-internal.test.ts', + '/src/http/client.test.ts', '/src/tests/event-data-size-limit/size-limit-1.test.ts', ], }, diff --git a/src/common/helpers.ts b/src/common/helpers.ts index bd73b8c8..5e4d2785 100644 --- a/src/common/helpers.ts +++ b/src/common/helpers.ts @@ -2,7 +2,7 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import * as v8 from 'v8'; -import { MAX_LOG_STRING_LENGTH } from '../logger/logger.constants'; +import { MAX_LOG_STRING_LENGTH } from '../logger/logger-constants'; /** * Gets the library version from the package.json file. diff --git a/src/http/axios-client-internal.test.ts b/src/http/client.test.ts similarity index 99% rename from src/http/axios-client-internal.test.ts rename to src/http/client.test.ts index daa9e07f..8f220cfc 100644 --- a/src/http/axios-client-internal.test.ts +++ b/src/http/client.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { axiosClient } from './axios-client-internal'; +import { axiosClient } from './client'; jest.setTimeout(60000); diff --git a/src/http/axios-client-internal.ts b/src/http/client.ts similarity index 100% rename from src/http/axios-client-internal.ts rename to src/http/client.ts diff --git a/src/index.ts b/src/index.ts index c9d81310..aa4878ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,10 +33,10 @@ export type { // ── Constants & enums ── export { AirSyncDefaultItemTypes } from './common/constants'; export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants'; -export { ExtractionCommonError } from './common/errors'; +export { ExtractionCommonError } from './types/errors'; // ── Domain mapping install ── -export { installInitialDomainMapping } from './common/install-initial-domain-mapping'; +export { installInitialDomainMapping } from './state/install-initial-domain-mapping'; // ── Error formatting ── export { serializeError } from './logger/logger'; @@ -104,13 +104,13 @@ export { Mappers } from './mappers/mappers'; export { SyncMapperRecordStatus, SyncMapperRecordTargetType, -} from './mappers/mappers.interface'; +} from './mappers/mappers.interfaces'; export type { MappersCreateParams, MappersGetByExternalIdParams, MappersGetByTargetIdParams, MappersUpdateParams, -} from './mappers/mappers.interface'; +} from './mappers/mappers.interfaces'; // ── State types ── export type { AdapterState } from './state/state.interfaces'; @@ -171,11 +171,11 @@ export type { } from './types/external-domain-metadata'; // ── Testing utilities (public test-support surface) ── -export { MockServer } from './mock-server/mock-server'; +export { MockServer, MOCK_SERVER_DEFAULT_URL } from './testing/mock-server'; export type { RequestInfo, RetryConfig, RouteConfig, -} from './mock-server/mock-server.interfaces'; -export { createMockEvent, MOCK_SERVER_DEFAULT_URL } from './common/test-utils'; -export type { DeepPartial } from './common/test-utils'; +} from './testing/mock-server.interfaces'; +export { createMockEvent } from './testing/mock-event'; +export type { DeepPartial } from './testing/mock-event'; diff --git a/src/logger/logger.constants.ts b/src/logger/logger-constants.ts similarity index 100% rename from src/logger/logger.constants.ts rename to src/logger/logger-constants.ts diff --git a/src/logger/logger.context.ts b/src/logger/logger-context.ts similarity index 100% rename from src/logger/logger.context.ts rename to src/logger/logger-context.ts diff --git a/src/logger/logger.test.ts b/src/logger/logger.test.ts index 71374ec7..2f882505 100644 --- a/src/logger/logger.test.ts +++ b/src/logger/logger.test.ts @@ -2,7 +2,7 @@ import { AxiosError } from 'axios'; import { inspect } from 'node:util'; import { LIBRARY_VERSION } from '../common/constants'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { AirdropEvent, EventType } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; import { @@ -14,7 +14,7 @@ import { import { INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, MAX_LOG_STRING_LENGTH, -} from './logger.constants'; +} from './logger-constants'; // Mock console methods const mockConsoleInfo = jest.spyOn(console, 'info').mockImplementation(); diff --git a/src/logger/logger.ts b/src/logger/logger.ts index a7204d3f..fb7b1e10 100644 --- a/src/logger/logger.ts +++ b/src/logger/logger.ts @@ -7,8 +7,8 @@ import { isMainThread, parentPort } from 'node:worker_threads'; import { LIBRARY_VERSION } from '../common/constants'; import { WorkerAdapterOptions, WorkerMessageSubject } from '../types/workers'; -import { INSPECT_OPTIONS } from './logger.constants'; -import { getSdkLogContextValue } from './logger.context'; +import { INSPECT_OPTIONS } from './logger-constants'; +import { getSdkLogContextValue } from './logger-context'; import { AxiosErrorResponse, LoggerFactoryInterface, diff --git a/src/mappers/mappers.interface.ts b/src/mappers/mappers.interfaces.ts similarity index 100% rename from src/mappers/mappers.interface.ts rename to src/mappers/mappers.interfaces.ts diff --git a/src/mappers/mappers.test.ts b/src/mappers/mappers.test.ts index 68a8ef23..54e8f3d4 100644 --- a/src/mappers/mappers.test.ts +++ b/src/mappers/mappers.test.ts @@ -1,5 +1,6 @@ -import { createMockEvent, MOCK_SERVER_DEFAULT_URL } from '../common/test-utils'; -import { axiosClient } from '../http/axios-client-internal'; +import { createMockEvent } from '../testing/mock-event'; +import { MOCK_SERVER_DEFAULT_URL } from '../testing/mock-server'; +import { axiosClient } from '../http/client'; import { EventType } from '../types/extraction'; import { Mappers } from './mappers'; import { @@ -9,10 +10,10 @@ import { MappersUpdateParams, SyncMapperRecordStatus, SyncMapperRecordTargetType, -} from './mappers.interface'; +} from './mappers.interfaces'; // Mock the axios client -jest.mock('../http/axios-client-internal'); +jest.mock('../http/client'); const mockAxiosClient = axiosClient as jest.Mocked; describe(Mappers.name, () => { diff --git a/src/mappers/mappers.ts b/src/mappers/mappers.ts index 1238c5c9..1982c353 100644 --- a/src/mappers/mappers.ts +++ b/src/mappers/mappers.ts @@ -1,4 +1,4 @@ -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { MappersCreateParams, @@ -10,7 +10,7 @@ import { MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse, -} from './mappers.interface'; +} from './mappers.interfaces'; /** * Manages sync mapper records that link external system items to DevRev items. diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts index ab36c0e6..c5f46188 100644 --- a/src/multithreading/adapters/base-adapter.ts +++ b/src/multithreading/adapters/base-adapter.ts @@ -4,7 +4,7 @@ import { STATELESS_EVENT_TYPES } from '../../common/constants'; import { emit } from '../emit'; import { truncateMessage } from '../../common/helpers'; import { serializeError } from '../../logger/logger'; -import { runWithSdkLogContext } from '../../logger/logger.context'; +import { runWithSdkLogContext } from '../../logger/logger-context'; import { BaseState } from '../../state/state'; import { SdkState } from '../../state/state.interfaces'; import { diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index ea4f21ac..367171ba 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -8,7 +8,7 @@ import { serializeError } from '../../logger/logger'; import { runWithSdkLogContext, runWithUserLogContext, -} from '../../logger/logger.context'; +} from '../../logger/logger-context'; import { Repo } from '../../repo/repo'; import { NormalizedAttachment, diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts index 6493d9be..68e7b5a8 100644 --- a/src/multithreading/adapters/loading-adapter.ts +++ b/src/multithreading/adapters/loading-adapter.ts @@ -4,9 +4,9 @@ import { serializeError } from '../../logger/logger'; import { runWithSdkLogContext, runWithUserLogContext, -} from '../../logger/logger.context'; +} from '../../logger/logger-context'; import { Mappers } from '../../mappers/mappers'; -import { SyncMapperRecordStatus } from '../../mappers/mappers.interface'; +import { SyncMapperRecordStatus } from '../../mappers/mappers.interfaces'; import { BaseState } from '../../state/state'; import { AirSyncEvent, diff --git a/src/multithreading/create-worker.test.ts b/src/multithreading/create-worker.test.ts index 0c9eb07e..b1b069f6 100644 --- a/src/multithreading/create-worker.test.ts +++ b/src/multithreading/create-worker.test.ts @@ -1,7 +1,7 @@ import { isMainThread, Worker } from 'worker_threads'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; import { createWorker } from './create-worker'; diff --git a/src/multithreading/emit.ts b/src/multithreading/emit.ts index 9296d87c..f3e41ac3 100644 --- a/src/multithreading/emit.ts +++ b/src/multithreading/emit.ts @@ -1,5 +1,5 @@ import { AxiosResponse } from 'axios'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { AirSyncEvent, EventData, diff --git a/src/multithreading/process-task.test.ts b/src/multithreading/process-task.test.ts index 015c5612..2c5b06d0 100644 --- a/src/multithreading/process-task.test.ts +++ b/src/multithreading/process-task.test.ts @@ -49,7 +49,7 @@ jest.mock('../logger/logger', () => ({ serializeError: jest.fn((e: unknown) => String(e)), })); -jest.mock('../logger/logger.context', () => ({ +jest.mock('../logger/logger-context', () => ({ runWithSdkLogContext: jest.fn((fn: () => unknown) => fn()), runWithUserLogContext: jest.fn((fn: () => unknown) => fn()), })); @@ -69,7 +69,7 @@ import { processTask } from './process-task'; import { translateIncomingEventType } from '../common/event-type-translation'; import { createAdapterState } from '../state/state'; import { WorkerAdapter } from './worker-adapter/worker-adapter'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; function setWorkerData(data: Record) { (global as Record).__workerData__ = data; diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index e59a789c..6c59250d 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -4,7 +4,7 @@ import { Logger, serializeError } from '../logger/logger'; import { runWithSdkLogContext, runWithUserLogContext, -} from '../logger/logger.context'; +} from '../logger/logger-context'; import { createExtractionState } from '../state/extraction-state'; import { createLoadingState } from '../state/loading-state'; import { diff --git a/src/multithreading/spawn/spawn.test.ts b/src/multithreading/spawn/spawn.test.ts index 9445c20f..43ac5f45 100644 --- a/src/multithreading/spawn/spawn.test.ts +++ b/src/multithreading/spawn/spawn.test.ts @@ -2,7 +2,7 @@ import { EventEmitter } from 'events'; import { DEFAULT_LAMBDA_TIMEOUT } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { WorkerEvent, WorkerMessageSubject } from '../../types/workers'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; // --------------------------------------------------------------------------- // Mocks diff --git a/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts b/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts deleted file mode 100644 index f3bef85e..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { AirSyncDefaultItemTypes } from '../../common/constants'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createItems } from '../../tests/test-helpers'; -import { createMockEvent } from '../../common/test-utils'; -import { Artifact, EventType } from '../../types'; -import { ExternalSyncUnit } from '../../types/extraction'; -import { WorkerAdapter } from './worker-adapter'; - -// 1. Create a mock function for the method you want to override. -const mockUpload = (itemType: string, objects: object[]) => { - return { - error: null, - artifact: { - id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`, - item_type: itemType, - item_count: objects.length, - }, - }; -}; - -// 2. Mock the entire 'uploader' module. -// The factory function () => { ... } returns the mock implementation. -jest.mock('../../uploader/uploader', () => { - return { - // The mocked Uploader class - Uploader: jest.fn().mockImplementation(() => { - // The constructor of the mocked Uploader returns an object - // with the methods you want to control. - return { - upload: mockUpload, - }; - }), - }; -}); - -function checkArtifactOrder( - artifacts: Artifact[], - expectedOrder: { itemType: string }[] -): boolean { - let outerIndex = 0; - for (const artifact of artifacts) { - try { - // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown - while (artifact.item_type != expectedOrder[outerIndex].itemType) { - outerIndex++; - } - } catch (e) { - console.error('Error finding artifact type in repos:', e); - return false; - } - } - return true; -} - -describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => { - interface TestState { - attachments: { completed: boolean }; - } - let testAdapter: WorkerAdapter; - - beforeEach(() => { - // Create a fresh adapter instance for this test to avoid mocking conflicts - const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.StartExtractingData }, - }); - const mockAdapterState = new State({ - event: mockEvent, - initialState: { attachments: { completed: false } }, - }); - - testAdapter = new WorkerAdapter({ - event: mockEvent, - adapterState: mockAdapterState, - options: { - batchSize: 50, - }, - }); - }); - - it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => { - const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(5)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(105)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(4); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should work with more than 2 repos', async () => { - const repos = [ - { itemType: 'ItemTypeA' }, - { itemType: 'ItemTypeB' }, - { itemType: 'ItemTypeC' }, - { itemType: 'ItemTypeD' }, - ]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeC')?.push(createItems(103)); - await testAdapter.getRepo('ItemTypeD')?.push(createItems(104)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(12); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should maintain order with multiple pushes and uploads', async () => { - const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.upload(); - await testAdapter.getRepo('ItemTypeB')?.upload(); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(20); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should not count artifacts if 0 items are pushed to the repo', async () => { - const repos = [{ itemType: 'ItemTypeA' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push([]); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(0); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); -}); - -describe('External sync units splitting into artifacts', () => { - let testAdapter: WorkerAdapter>; - - beforeEach(() => { - const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.StartExtractingExternalSyncUnits }, - }); - const mockAdapterState = new State>({ - event: mockEvent, - initialState: {}, - }); - - testAdapter = new WorkerAdapter({ - event: mockEvent, - adapterState: mockAdapterState, - }); - }); - - it('should split 125k external sync units into 5 artifacts', async () => { - const BATCH_SIZE = 25_000; - const TOTAL_UNITS = 125_000; - - const externalSyncUnits: ExternalSyncUnit[] = Array.from( - { length: TOTAL_UNITS }, - (_, i) => ({ - id: String(i), - name: `Unit ${i}`, - description: `Description ${i}`, - }) - ); - - testAdapter.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: BATCH_SIZE, - skipConfirmation: true, - }, - }, - ]); - - const repo = testAdapter.getRepo( - AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS - ); - const chunkSize = 10_000; - for (let i = 0; i < TOTAL_UNITS; i += chunkSize) { - await repo?.push(externalSyncUnits.slice(i, i + chunkSize)); - } - - await testAdapter.uploadAllRepos(); - - expect(testAdapter.artifacts.length).toBe( - TOTAL_UNITS / BATCH_SIZE // 5 - ); - expect( - testAdapter.artifacts.every( - (a) => a.item_type === AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS - ) - ).toBe(true); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts b/src/multithreading/worker-adapter/worker-adapter.emit.test.ts deleted file mode 100644 index 8afef41c..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts +++ /dev/null @@ -1,593 +0,0 @@ -import { UNBOUNDED_DATE_TIME_VALUE } from '../../common/constants'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - Artifact, - EventType, - ExtractorEventType, - LoaderEventType, -} from '../../types'; -import { ActionType, LoaderReport } from '../../types/loading'; -import { WorkerAdapter } from './worker-adapter'; - -/* eslint-disable @typescript-eslint/no-require-imports */ - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType = EventType.StartExtractingData): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -describe(`${WorkerAdapter.name}.emit`, () => { - let adapter: WorkerAdapter; - let mockPostMessage: jest.Mock; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - - const workerThreads = require('node:worker_threads'); - mockPostMessage = jest.fn(); - if (workerThreads.parentPort) { - jest - .spyOn(workerThreads.parentPort, 'postMessage') - .mockImplementation(mockPostMessage); - } else { - workerThreads.parentPort = { postMessage: mockPostMessage }; - } - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should emit only one event when multiple events of same type are sent', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should emit only once even when a different event type follows', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.DataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.AttachmentExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should correctly emit one event even if postState errors', async () => { - // Arrange - adapter['adapterState'].postState = jest - .fn() - .mockRejectedValue(new Error('postState error')); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should correctly emit one event even if uploadAllRepos errors', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest - .fn() - .mockRejectedValue(new Error('uploadAllRepos error')); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should include artifacts in data for extraction events', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['_artifacts'] = [ - { id: 'art-1', item_count: 10, item_type: 'issues' }, - ] as Artifact[]; - - // Act - await adapter.emit(ExtractorEventType.DataExtractionDone); - - // Assert - expect(mockEmit).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - artifacts: expect.arrayContaining([ - expect.objectContaining({ id: 'art-1' }), - ]), - }), - }) - ); - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('reports'); - expect(callData).not.toHaveProperty('processed_files'); - }); - - it('should include reports and processed_files in data for loader events', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['loaderReports'] = [ - { item_type: 'tasks', [ActionType.CREATED]: 5 }, - ] as LoaderReport[]; - adapter['_processedFiles'] = ['file-1', 'file-2']; - - // Act - await adapter.emit(LoaderEventType.DataLoadingDone); - - // Assert - expect(mockEmit).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - reports: expect.arrayContaining([ - expect.objectContaining({ item_type: 'tasks' }), - ]), - processed_files: ['file-1', 'file-2'], - }), - }) - ); - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('artifacts'); - }); - - it('should not include artifacts, reports, or processed_files for unknown event types', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['_artifacts'] = [ - { id: 'art-1', item_count: 10, item_type: 'issues' }, - ] as Artifact[]; - adapter['loaderReports'] = [ - { item_type: 'tasks', [ActionType.CREATED]: 5 }, - ] as LoaderReport[]; - adapter['_processedFiles'] = ['file-1']; - - // Act - await adapter.emit('SOME_UNKNOWN_EVENT' as ExtractorEventType); - - // Assert - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('artifacts'); - expect(callData).not.toHaveProperty('reports'); - expect(callData).not.toHaveProperty('processed_files'); - }); - - it('should include artifacts for all ExtractorEventType values', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - const extractorEvents = [ - ExtractorEventType.DataExtractionDone, - ExtractorEventType.DataExtractionProgress, - ExtractorEventType.DataExtractionError, - ExtractorEventType.AttachmentExtractionDone, - ExtractorEventType.AttachmentExtractionProgress, - ]; - - for (const eventType of extractorEvents) { - jest.clearAllMocks(); - adapter.hasWorkerEmitted = false; - adapter['adapterState'].postState = jest - .fn() - .mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(eventType); - - // Assert - const callData = mockEmit.mock.calls[0]?.[0]?.data; - expect(callData).toHaveProperty('artifacts'); - expect(callData).not.toHaveProperty('reports'); - } - }); - - it('should include reports and processed_files for all LoaderEventType values', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - const loaderEvents = [ - LoaderEventType.DataLoadingDone, - LoaderEventType.DataLoadingProgress, - LoaderEventType.DataLoadingError, - LoaderEventType.AttachmentLoadingDone, - LoaderEventType.AttachmentLoadingProgress, - ]; - - for (const eventType of loaderEvents) { - jest.clearAllMocks(); - adapter.hasWorkerEmitted = false; - adapter['adapterState'].postState = jest - .fn() - .mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(eventType); - - // Assert - const callData = mockEmit.mock.calls[0]?.[0]?.data; - expect(callData).toHaveProperty('reports'); - expect(callData).toHaveProperty('processed_files'); - expect(callData).not.toHaveProperty('artifacts'); - } - }); - - it('should truncate a long error message, preserving the original prefix', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - const longMessage = 'E'.repeat(20_000); - - // Act - await adapter.emit(ExtractorEventType.DataExtractionError, { - error: { message: longMessage }, - }); - - // Assert - const { emit: mockEmit } = require('../../common/control-protocol'); - const emittedMessage = mockEmit.mock.calls[0][0].data?.error - ?.message as string; - expect(emittedMessage.length).toBeLessThan(longMessage.length); - expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true); - }); -}); - -describe(`${WorkerAdapter.name}.emit — ExternalSyncUnitExtractionDone legacy path`, () => { - it('should upload ESUs via a repo and strip external_sync_units from the emitted payload', async () => { - // Arrange - const { adapter } = makeAdapter(EventType.StartExtractingExternalSyncUnits); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - const pushMock = jest.fn().mockResolvedValue(undefined); - jest.spyOn(adapter, 'initializeRepos'); - jest.spyOn(adapter, 'getRepo').mockReturnValue({ push: pushMock } as never); - const esus = [{ id: 'esu-1' }, { id: 'esu-2' }] as never; - - // Act - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: esus, - }); - - // Assert - expect(pushMock).toHaveBeenCalledWith(esus); - // external_sync_units must NOT appear in the payload sent to the platform - // (it would be too large for SQS — that is the entire reason this path exists). - const { emit: mockEmit } = require('../../common/control-protocol'); - const emittedData = mockEmit.mock.calls[0][0].data as Record< - string, - unknown - >; - expect(emittedData).not.toHaveProperty('external_sync_units'); - }); -}); - -describe('WorkerAdapter — workersOldest / workersNewest boundary updates', () => { - let adapter: WorkerAdapter; - let mockPostMessage: jest.Mock; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - - const workerThreads = require('node:worker_threads'); - mockPostMessage = jest.fn(); - if (workerThreads.parentPort) { - jest - .spyOn(workerThreads.parentPort, 'postMessage') - .mockImplementation(mockPostMessage); - } else { - workerThreads.parentPort = { postMessage: mockPostMessage }; - } - - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - async function emitDone( - adapterInstance: WorkerAdapter, - extractionStart: string | undefined, - extractionEnd: string | undefined - ) { - adapterInstance.event.payload.event_context.extract_from = extractionStart; - adapterInstance.event.payload.event_context.extract_to = extractionEnd; - // Reset the emit guard so we can emit multiple times within one test. - adapterInstance['hasWorkerEmitted'] = false; - - await adapterInstance.emit(ExtractorEventType.AttachmentExtractionDone, { - reports: [], - processed_files: [], - }); - } - - describe('initial import with UNBOUNDED start', () => { - it('should set workersOldest to UNBOUNDED_DATE_TIME_VALUE and workersNewest to extraction end', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation after UNBOUNDED initial import', () => { - it('should NOT overwrite workersOldest when reconciliation start is later than sentinel', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - - it('should NOT overwrite workersOldest even when reconciliation start is very early', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '1980-01-01T00:00:00.000Z', - '1990-01-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - }); - - describe('forward sync after UNBOUNDED initial import', () => { - it('should expand workersNewest forward while preserving workersOldest', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-06-01T00:00:00.000Z', - '2025-07-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-07-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation with end beyond current newest', () => { - it('should expand workersNewest when reconciliation end is later', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-01-01T00:00:00.000Z', - '2025-08-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-08-01T00:00:00.000Z'); - }); - }); - - describe('first sync with absolute dates (no UNBOUNDED)', () => { - it('should set both boundaries from the extraction range', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation after absolute initial sync', () => { - it('should expand workersOldest backward when reconciliation start is earlier', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-06-01T00:00:00.000Z', - '2025-02-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2024-06-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - - it('should NOT change boundaries when reconciliation is within existing range', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-01-15T00:00:00.000Z', - '2025-02-15T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - - it('should expand both boundaries when reconciliation exceeds both', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-06-01T00:00:00.000Z', - '2025-09-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2024-06-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-09-01T00:00:00.000Z'); - }); - }); - - describe('multiple forward syncs', () => { - it('should progressively expand workersNewest while preserving workersOldest', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-06-01T00:00:00.000Z', - '2025-07-01T00:00:00.000Z' - ); - expect(adapter.state.workersNewest).toBe('2025-07-01T00:00:00.000Z'); - - await emitDone( - adapter, - '2025-07-01T00:00:00.000Z', - '2025-08-01T00:00:00.000Z' - ); - expect(adapter.state.workersNewest).toBe('2025-08-01T00:00:00.000Z'); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - }); - }); - - describe('non-AttachmentExtractionDone events should NOT update boundaries', () => { - it.each([ - ExtractorEventType.DataExtractionDone, - ExtractorEventType.DataExtractionProgress, - ExtractorEventType.MetadataExtractionError, - ExtractorEventType.AttachmentExtractionError, - ])('should not update boundaries on %s', async (eventType) => { - adapter.state.workersOldest = '2025-01-01T00:00:00.000Z'; - adapter.state.workersNewest = '2025-03-01T00:00:00.000Z'; - adapter.event.payload.event_context.extract_from = - '2024-01-01T00:00:00.000Z'; - adapter.event.payload.event_context.extract_to = - '2025-12-01T00:00:00.000Z'; - - await adapter.emit(eventType, { - reports: [], - processed_files: [], - }); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts b/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts deleted file mode 100644 index 55b38e88..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts +++ /dev/null @@ -1,1034 +0,0 @@ -import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - Artifact, - EventType, - ExtractorEventType, -} from '../../types'; -import { WorkerAdapter } from './worker-adapter'; - -/* eslint-disable @typescript-eslint/no-require-imports */ - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType = EventType.StartExtractingData): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -describe(`${WorkerAdapter.name}.streamAttachments`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - }); - - it('should process all artifact batches successfully', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - { - url: 'http://example.com/file2.pdf', - id: 'attachment2', - file_name: 'file2.pdf', - parent_id: 'parent2', - }, - ], - }) - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file3.pdf', - id: 'attachment3', - file_name: 'file3.pdf', - parent_id: 'parent3', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.initializeRepos).toHaveBeenCalledWith([ - { itemType: 'ssor_attachment' }, - ]); - expect(adapter.initializeRepos).toHaveBeenCalledTimes(1); - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(2); - - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]); - expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0); - expect(result).toBeUndefined(); - }); - - it('[edge] should handle invalid batch size by using 1 instead', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - batchSize: 0, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - batchSize: 100, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should handle empty attachments metadata artifact IDs', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - }, - }; - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should handle errors when getting attachments', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - const mockError = new Error('Failed to get attachments'); - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - error: mockError, - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ - error: mockError, - }); - }); - - it('[edge] should handle empty attachments array from artifact', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]); - expect(result).toBeUndefined(); - }); - - it('should use custom processors when provided', async () => { - // Arrange - const mockStream = jest.fn(); - const mockReducer = jest.fn().mockReturnValue(['custom-reduced']); - const mockIterator = jest.fn().mockResolvedValue({}); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - processors: { - reducer: mockReducer, - iterator: mockIterator, - }, - }); - - // Assert - expect(mockReducer).toHaveBeenCalledWith({ - attachments: [{ id: 'attachment1' }], - adapter: adapter, - batchSize: 1, - }); - expect(mockIterator).toHaveBeenCalledWith({ - reducedAttachments: ['custom-reduced'], - adapter: adapter, - stream: mockStream, - }); - expect(result).toBeUndefined(); - }); - - it('should handle rate limiting from iterator', async () => { - // Arrange - const mockStream = jest.fn(); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({ delay: 30 }), - })); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ delay: 30 }); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - ]); - }); - - it('should handle error from iterator', async () => { - // Arrange - const mockStream = jest.fn(); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({ - error: 'Mock error', - }), - })); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ error: 'Mock error' }); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - ]); - }); - - it('should emit progress event and exit process on timeout, preserving state for resumption', async () => { - // Arrange - const mockStream = jest.fn(); - - const exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2', 'artifact3'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockImplementation(() => { - adapter.isTimeout = true; - return {}; - }), - })); - - adapter.initializeRepos = jest.fn(); - - const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - - // Act - await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - ExtractorEventType.AttachmentExtractionProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - 'artifact2', - 'artifact3', - ]); - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(1); - - exitSpy.mockRestore(); - }); - - it('should stop after the timeout flips between batches and preserve unprocessed artifacts for resumption', async () => { - // Arrange: three artifacts. The first batch's streamAll completes - // successfully; the second sets isTimeout=true mid-run. The third batch - // must never be reached. - const mockStream = jest.fn(); - const exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2', 'artifact3'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file.pdf', - id: 'attachment-x', - file_name: 'file.pdf', - parent_id: 'parent-x', - }, - ], - }); - - // First call: clean streamAll. Second call: flip isTimeout AFTER streaming. - (AttachmentsStreamingPool as jest.Mock) - .mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({}), - })) - .mockImplementationOnce(() => ({ - streamAll: jest.fn().mockImplementation(() => { - adapter.isTimeout = true; - return {}; - }), - })); - - adapter.initializeRepos = jest.fn(); - const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - - // Act - await adapter.streamAttachments({ stream: mockStream }); - - // Assert - // - Fetched attachments for the first two artifacts only; the third never ran - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(2); - // - Progress emitted and process.exit(0) called once the timeout was detected - expect(emitSpy).toHaveBeenCalledWith( - ExtractorEventType.AttachmentExtractionProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - // - Artifact 1 was shifted out cleanly; artifact 2 remains (timeout caught - // before its shift) along with the untouched artifact 3 - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact2', - 'artifact3', - ]); - - exitSpy.mockRestore(); - }); - - it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => { - // Arrange - const mockStream = jest.fn(); - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - { - url: 'http://example.com/file2.pdf', - id: 'attachment2', - file_name: 'file2.pdf', - parent_id: 'parent2', - }, - { - url: 'http://example.com/file3.pdf', - id: 'attachment3', - file_name: 'file3.pdf', - parent_id: 'parent3', - }, - ], - }); - - adapter.processAttachment = jest.fn().mockResolvedValue(null); - - // Act - await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength( - 0 - ); - expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0); - }); -}); - -describe(`${WorkerAdapter.name}.processAttachment`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.StartExtractingAttachments)); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - const createMockHttpStream = (headers: Record = {}) => ({ - headers, - data: { destroy: jest.fn() }, - }); - - const makeAttachment = (overrides = {}) => ({ - id: 'att-1', - url: 'https://example.com/file.pdf', - file_name: 'file.pdf', - parent_id: 'parent-1', - content_type: 'application/pdf', - ...overrides, - }); - - function setupUploaderHappyPath() { - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art_1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ response: {} }); - adapter['uploader'].confirmArtifactUpload = jest - .fn() - .mockResolvedValue({ response: {} }); - - const pushMock = jest.fn().mockResolvedValue(undefined); - adapter.getRepo = jest.fn().mockReturnValue({ push: pushMock }); - return pushMock; - } - - // ---- content-type resolution ---- - it('should use attachment.content_type when provided, ignoring HTTP header', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({ - 'content-type': 'text/plain', - 'content-length': '100', - }), - }); - - // Act - await adapter.processAttachment( - makeAttachment({ content_type: 'application/pdf' }) as never, - mockStream - ); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'file.pdf', - 'application/pdf', - 100 - ); - }); - - it('should use HTTP header content-type when attachment.content_type is not set', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({ - 'content-type': 'image/jpeg', - 'content-length': '200', - }), - }); - - const attachment = { - id: 'att-2', - url: 'https://example.com/photo.jpg', - file_name: 'photo.jpg', - parent_id: 'parent-2', - }; - - // Act - await adapter.processAttachment(attachment as never, mockStream); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'photo.jpg', - 'image/jpeg', - 200 - ); - }); - - it('should fall back to application/octet-stream when neither content_type nor HTTP header is set', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({}), - }); - - const attachment = { - id: 'att-3', - url: 'https://example.com/file.bin', - file_name: 'file.bin', - parent_id: 'parent-3', - }; - - // Act - await adapter.processAttachment(attachment as never, mockStream); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'file.bin', - 'application/octet-stream', - undefined - ); - }); - - // ---- error paths ---- - it('should return the stream error directly when the stream function returns an error', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ error: new Error('stream failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error).toBeDefined(); - }); - - it('should propagate a rate-limit delay from the stream function', async () => { - // Arrange - const stream = jest.fn().mockResolvedValue({ delay: 5 }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.delay).toBe(5); - }); - - it('should return an error containing the attachment ID when getArtifactUploadUrl fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest - .fn() - .mockResolvedValue({ error: new Error('upload url failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('att-1'); - expect(result?.error?.message).toContain('preparing artifact'); - }); - - it('should return an error when streamArtifact fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art-1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ error: new Error('stream failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('streaming to artifact'); - }); - - it('should return an error when confirmArtifactUpload fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art-1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ response: {} }); - adapter['uploader'].confirmArtifactUpload = jest - .fn() - .mockResolvedValue({ error: new Error('confirm failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('confirming upload'); - }); - - it.each([ - { inline: true, expected: true }, - { inline: false, expected: false }, - ])( - 'should set inline=$expected on the ssorAttachment when attachment.inline=$inline', - async ({ inline, expected }) => { - // Arrange - const pushMock = setupUploaderHappyPath(); - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - - // Act - await adapter.processAttachment( - makeAttachment({ inline }) as never, - stream - ); - - // Assert - const ssorItem = pushMock.mock.calls[0][0][0] as Record; - expect(ssorItem.inline).toBe(expected); - } - ); - - it('should return a descriptive error when the stream function returns no httpStream', async () => { - // Arrange - const stream = jest.fn().mockResolvedValue({ httpStream: null }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain( - 'Error while opening attachment stream' - ); - }); -}); - -describe(`${WorkerAdapter.name}.initializeRepos — event size threshold`, () => { - it('should set isTimeout=true once the cumulative artifact payload exceeds EVENT_SIZE_THRESHOLD_BYTES', () => { - // Arrange - const { adapter } = makeAdapter(); - - let capturedOnUpload: ((artifact: Artifact) => void) | undefined; - const { Repo } = require('../../repo/repo'); - (Repo as jest.Mock).mockImplementationOnce( - (opts: { onUpload: (a: Artifact) => void }) => { - capturedOnUpload = opts.onUpload; - return { itemType: 'issues', upload: jest.fn(), uploadedArtifacts: [] }; - } - ); - - // Act - adapter.initializeRepos([{ itemType: 'issues' }]); - expect(capturedOnUpload).toBeDefined(); - capturedOnUpload!({ - id: 'artifact-x', - item_count: 1, - item_type: 'x'.repeat(200_000), - }); - - // Assert - expect(adapter.isTimeout).toBe(true); - }); -}); - -describe(`${WorkerAdapter.name}.getRepo`, () => { - it('should return undefined when the requested repo was never initialised', () => { - // Arrange - const { adapter } = makeAdapter(); - - // Act - const result = adapter.getRepo('non-existent-type'); - - // Assert - expect(result).toBeUndefined(); - }); -}); - -describe(`${WorkerAdapter.name}.destroyHttpStream`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - ({ adapter } = makeAdapter()); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it.each([ - { - label: 'calls destroy() when available', - data: { destroy: jest.fn(), close: jest.fn() }, - expectDestroy: true, - expectClose: false, - }, - { - label: 'calls close() when destroy is not present', - data: { close: jest.fn() }, - expectDestroy: false, - expectClose: true, - }, - { - label: 'does not throw when neither method is present', - data: {}, - expectDestroy: false, - expectClose: false, - }, - { - label: 'does not throw when data is null', - data: null, - expectDestroy: false, - expectClose: false, - }, - ])('$label', ({ data, expectDestroy, expectClose }) => { - // Arrange - const httpStream = { data } as never; - - // Act & Assert - expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow(); - - if (expectDestroy) { - expect((data as { destroy: jest.Mock }).destroy).toHaveBeenCalled(); - } - if (expectClose) { - expect((data as { close: jest.Mock }).close).toHaveBeenCalled(); - } - }); - - it('should not re-throw when destroy() itself throws', () => { - // Arrange - const httpStream = { - data: { - destroy: () => { - throw new Error('stream error'); - }, - }, - }; - - // Act & Assert - expect(() => - adapter['destroyHttpStream'](httpStream as never) - ).not.toThrow(); - }); -}); - -describe(`${WorkerAdapter.name} — extractionScope`, () => { - it('should return empty object by default', () => { - const { adapter } = makeAdapter(); - expect(adapter.extractionScope).toEqual({}); - }); - - it('should return extraction scope from adapter state', () => { - const { adapter, adapterState } = makeAdapter(); - const extractionScope = { - tasks: { extract: true }, - users: { extract: false }, - }; - - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = extractionScope; - - expect(adapter.extractionScope).toEqual(extractionScope); - }); -}); - -describe(`${WorkerAdapter.name} — shouldExtract`, () => { - it('should return true when extraction scope is empty', () => { - const { adapter } = makeAdapter(); - expect(adapter.shouldExtract('tasks')).toBe(true); - expect(adapter.shouldExtract('users')).toBe(true); - }); - - it('should return true when item type is not in scope', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: true }, - }; - expect(adapter.shouldExtract('users')).toBe(true); - }); - - it('should return true when item type has extract: true', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: true }, - }; - expect(adapter.shouldExtract('tasks')).toBe(true); - }); - - it('should return false when item type has extract: false', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: false }, - users: { extract: true }, - }; - expect(adapter.shouldExtract('tasks')).toBe(false); - expect(adapter.shouldExtract('users')).toBe(true); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts b/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts deleted file mode 100644 index df151d48..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts +++ /dev/null @@ -1,592 +0,0 @@ -import { - ActionType, - ItemTypeToLoad, - LoaderReport, - StatsFileObject, -} from '../../types/loading'; - -import { - addReportToLoaderReport, - getFilesToLoad, -} from './worker-adapter.helpers'; - -describe(getFilesToLoad.name, () => { - let statsFile: StatsFileObject[]; - - beforeEach(() => { - statsFile = [ - { - id: 'test-artifact-1', - file_name: 'test_file_1.json.gz', - item_type: 'issues', - count: '79', - }, - { - id: 'test-artifact-2', - file_name: 'test_file_2.json.gz', - item_type: 'comments', - count: '1079', - }, - { - id: 'test-artifact-3', - file_name: 'test_file_3.json.gz', - item_type: 'issues', - count: '1921', - }, - { - id: 'test-artifact-4', - file_name: 'test_file_4.json.gz', - item_type: 'comments', - count: '921', - }, - { - id: 'test-artifact-5', - file_name: 'test_file_5.json.gz', - item_type: 'attachments', - count: '50', - }, - { - id: 'test-artifact-6', - file_name: 'test_file_6.json.gz', - item_type: 'unknown', - count: '50', - }, - { - id: 'test-artifact-7', - file_name: 'test_file_7.json.gz', - item_type: 'issues', - count: '32', - }, - ]; - }); - - it('should filter files by supported item types and order them correctly', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'attachments', create: jest.fn(), update: jest.fn() }, - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - ]; - const expectedResult = [ - { - id: 'test-artifact-5', - itemType: 'attachments', - count: 50, - file_name: 'test_file_5.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-1', - itemType: 'issues', - count: 79, - file_name: 'test_file_1.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-3', - itemType: 'issues', - count: 1921, - file_name: 'test_file_3.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-7', - itemType: 'issues', - count: 32, - file_name: 'test_file_7.json.gz', - completed: false, - lineToProcess: 0, - }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual(expectedResult); - }); - - it('should ignore files with unrecognized item types in statsFile', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - { itemType: 'unrecognized', create: jest.fn(), update: jest.fn() }, - ]; - const expectedResult = [ - { - id: 'test-artifact-1', - itemType: 'issues', - count: 79, - file_name: 'test_file_1.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-3', - itemType: 'issues', - count: 1921, - file_name: 'test_file_3.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-7', - itemType: 'issues', - count: 32, - file_name: 'test_file_7.json.gz', - completed: false, - lineToProcess: 0, - }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual(expectedResult); - }); - - it('should parse count string to number', () => { - // Arrange - const singleItemStatsFile: StatsFileObject[] = [ - { - id: 'test-artifact-single', - file_name: 'test_file_single.json.gz', - item_type: 'issues', - count: '12345', - }, - ]; - const supportedItemTypes = ['issues']; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: singleItemStatsFile, - }); - - // Assert - expect(result[0].count).toBe(12345); - expect(typeof result[0].count).toBe('number'); - }); - - it('should initialize completed as false and lineToProcess as 0', () => { - // Arrange - const singleItemStatsFile: StatsFileObject[] = [ - { - id: 'test-artifact-init', - file_name: 'test_file_init.json.gz', - item_type: 'issues', - count: '100', - }, - ]; - const supportedItemTypes = ['issues']; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: singleItemStatsFile, - }); - - // Assert - expect(result[0].completed).toBe(false); - expect(result[0].lineToProcess).toBe(0); - }); - - it('[edge] should return an empty array when statsFile is empty', () => { - // Arrange - const emptyStatsFile: StatsFileObject[] = []; - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile: emptyStatsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when supportedItemTypes is empty', () => { - // Arrange - const supportedItemTypes: string[] = []; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when statsFile has no matching items', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'users', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when both statsFile and supportedItemTypes are empty', () => { - // Arrange - const emptyStatsFile: StatsFileObject[] = []; - const supportedItemTypes: string[] = []; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: emptyStatsFile, - }); - - // Assert - expect(result).toEqual([]); - }); -}); - -describe(addReportToLoaderReport.name, () => { - it('should add a new report when no existing report for the item type', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }); - }); - - it('should merge created counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(15); - }); - - it('should merge updated counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.UPDATED]: 20, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.UPDATED]: 8, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.UPDATED]).toBe(28); - }); - - it('should merge failed counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.FAILED]: 3, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.FAILED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.FAILED]).toBe(5); - }); - - it('should merge all action types when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 20, - [ActionType.FAILED]: 3, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 5, - [ActionType.UPDATED]: 8, - [ActionType.FAILED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 15, - [ActionType.UPDATED]: 28, - [ActionType.FAILED]: 5, - }); - }); - - it('should add reports for different item types separately', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'comments', - [ActionType.CREATED]: 50, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(2); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 10, - }); - expect(result[1]).toEqual({ - item_type: 'comments', - [ActionType.CREATED]: 50, - }); - }); - - it('should preserve existing count when new report has undefined for an action type', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(13); - expect(result[0][ActionType.UPDATED]).toBe(5); - }); - - it('should use new report count when existing report has undefined for an action type', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - [ActionType.UPDATED]: 7, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(13); - expect(result[0][ActionType.UPDATED]).toBe(7); - }); - - it('should mutate and return the same loaderReports array', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 10, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toBe(loaderReports); - }); - - it('[edge] should handle report with only item_type and no action counts', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ item_type: 'issues' }); - }); - - it('[edge] should handle merging when both reports have zero counts', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 0, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 0, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(0); - }); - - it('[edge] should handle empty loaderReports array', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'attachments', - [ActionType.CREATED]: 25, - [ActionType.FAILED]: 1, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'attachments', - [ActionType.CREATED]: 25, - [ActionType.FAILED]: 1, - }); - }); - - it('[edge] should preserve existing created count when new report has undefined created', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.UPDATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(10); - expect(result[0][ActionType.UPDATED]).toBe(5); - }); - - it('[edge] should preserve existing updated count when new report has undefined updated', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.UPDATED]: 15, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.UPDATED]).toBe(15); - expect(result[0][ActionType.CREATED]).toBe(3); - }); - - it('[edge] should preserve existing failed count when new report has undefined failed', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.FAILED]: 7, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.FAILED]).toBe(7); - expect(result[0][ActionType.CREATED]).toBe(2); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.loading.test.ts b/src/multithreading/worker-adapter/worker-adapter.loading.test.ts deleted file mode 100644 index 92486ab6..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.loading.test.ts +++ /dev/null @@ -1,729 +0,0 @@ -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - EventType, - LoaderEventType, -} from '../../types'; -import { - ActionType, - ExternalSystemAttachment, - ExternalSystemItem, -} from '../../types/loading'; -import { WorkerAdapter } from './worker-adapter'; - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -function makeLoaderItem(devrevId = 'dev-1'): ExternalSystemItem { - return { - id: { devrev: devrevId, external: 'ext-1' }, - created_date: '', - modified_date: '', - data: {}, - }; -} - -function setupLoaderFile( - adapter: WorkerAdapter, - items: ExternalSystemItem[], - itemType = 'tasks' -) { - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'file.json', - itemType, - count: items.length, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: items }); -} - -describe(`${WorkerAdapter.name}.loadItemTypes — timeout and unexpected errors`, () => { - let adapter: WorkerAdapter; - let exitSpy: jest.SpyInstance; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should emit DataLoadingProgress and exit on timeout', async () => { - // Arrange - const items = [makeLoaderItem('dev-1'), makeLoaderItem('dev-2')]; - setupLoaderFile(adapter, items); - adapter.isTimeout = true; - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith(LoaderEventType.DataLoadingProgress); - expect(exitSpy).toHaveBeenCalledWith(0); - }); - - it('should emit DataLoadingProgress mid-loop when timeout arrives between items', async () => { - // Arrange - const items = [ - makeLoaderItem('dev-1'), - makeLoaderItem('dev-2'), - makeLoaderItem('dev-3'), - ]; - setupLoaderFile(adapter, items); - exitSpy.mockRestore(); - exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => { - throw new Error('process.exit'); - }) as never); - let loadItemCallCount = 0; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/require-await - jest.spyOn(adapter as any, 'loadItem').mockImplementation(async () => { - loadItemCallCount++; - if (loadItemCallCount === 1) { - adapter.isTimeout = true; - } - return { report: { item_type: 'tasks', updated: 1 } }; - }); - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act & Assert - await expect(adapter.loadItemTypes({ itemTypesToLoad })).rejects.toThrow( - 'process.exit' - ); - expect(loadItemCallCount).toBe(1); - expect(emitSpy).toHaveBeenCalledWith(LoaderEventType.DataLoadingProgress); - }); - - it('should emit DataLoadingError and exit(1) on unexpected error', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'file1.json', - itemType: 'tasks', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockRejectedValue(new Error('Unexpected network failure')); - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Error during data loading'), - }), - }) - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); -}); - -describe(`${WorkerAdapter.name}.loadItemTypes — loadItem branch coverage via public API`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - let exitSpy: jest.SpyInstance; - - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - itemTypesToLoad[0].create = jest.fn(); - itemTypesToLoad[0].update = jest.fn(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should accumulate an UPDATED report when the connector updates the item and the mapper sync succeeds', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-1')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - adapter['_mappers'].update = jest.fn().mockResolvedValue({ data: {} }); - itemTypesToLoad[0].update = jest - .fn() - .mockResolvedValue({ id: 'ext-updated-1' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(reports).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - item_type: 'tasks', - [ActionType.UPDATED]: 1, - }), - ]) - ); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should fall back to create and accumulate a CREATED report when the mapper record does not exist (404)', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-2')]); - const axiosError = { isAxiosError: true, response: { status: 404 } }; - adapter['_mappers'].getByTargetId = jest.fn().mockRejectedValue(axiosError); - adapter['_mappers'].create = jest.fn().mockResolvedValue({ data: {} }); - itemTypesToLoad[0].create = jest - .fn() - .mockResolvedValue({ id: 'new-ext-id' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(reports).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - item_type: 'tasks', - [ActionType.CREATED]: 1, - }), - ]) - ); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit DataLoadingDelayed and stop processing when the connector signals a rate-limit delay', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-3')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - itemTypesToLoad[0].update = jest.fn().mockResolvedValue({ delay: 15 }); - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingDelayed, - expect.objectContaining({ delay: 15 }) - ); - }); - - it('should count the item as FAILED when the update succeeds but the mapper sync throws', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-4')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - adapter['_mappers'].update = jest - .fn() - .mockRejectedValue(new Error('mapper down')); - itemTypesToLoad[0].update = jest.fn().mockResolvedValue({ id: 'ext-id' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - expect(reports).toBeDefined(); - }); - - it('should not emit for a non-404 Axios error from the mapper (recorded as item-level error)', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-5')]); - const axiosError = { - isAxiosError: true, - message: 'internal server error', - response: { status: 500 }, - }; - adapter['_mappers'].getByTargetId = jest.fn().mockRejectedValue(axiosError); - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should handle a null sync_mapper_record gracefully and continue loading', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-6')]); - adapter['_mappers'].getByTargetId = jest - .fn() - .mockResolvedValue({ data: null }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - expect(reports).toBeDefined(); - }); -}); - -describe(`${WorkerAdapter.name}.loadItemTypes — additional branches`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should return immediately with empty reports when filesToLoad is empty', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { filesToLoad: [] }; - - // Act - const result = await adapter.loadItemTypes({ - itemTypesToLoad: [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ], - }); - - // Assert - expect(result.reports).toEqual([]); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit DataLoadingError when a file references an item type not in itemTypesToLoad', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'art-1', - file_name: 'file.json', - itemType: 'unknown-type', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: [makeLoaderItem()] }); - - // Act - await adapter.loadItemTypes({ - itemTypesToLoad: [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ], - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('unknown-type'), - }), - }) - ); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachments — timeout, transformer errors, unexpected errors`, () => { - let adapter: WorkerAdapter; - let exitSpy: jest.SpyInstance; - let emitSpy: jest.SpyInstance; - - function setupFilesToLoad( - a: WorkerAdapter, - items: ExternalSystemAttachment[] - ) { - a['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'attachments.json', - itemType: 'attachment', - count: items.length, - lineToProcess: 0, - completed: false, - }, - ], - }; - - a['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: items }); - } - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should emit AttachmentLoadingProgress and exit on timeout', async () => { - // Arrange - const items = [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ] as ExternalSystemAttachment[]; - setupFilesToLoad(adapter, items); - adapter.isTimeout = true; - - // Act - await adapter.loadAttachments({ - create: jest.fn(), - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - }); - - it('should emit AttachmentLoadingError on transformer file error', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'bad-artifact', - file_name: 'attachments.json', - itemType: 'attachment', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ - response: null, - error: new Error('Artifact not found'), - }); - - // Act - await adapter.loadAttachments({ - create: jest.fn(), - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Transformer file not found'), - }), - }) - ); - }); - - it('should emit AttachmentLoadingError and exit(1) on unexpected error', async () => { - // Arrange - const items = [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ] as ExternalSystemAttachment[]; - setupFilesToLoad(adapter, items); - const mockCreate = jest - .fn() - .mockRejectedValue(new Error('Unexpected API failure')); - - // Act - await adapter.loadAttachments({ create: mockCreate }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Error during attachment loading'), - }), - }) - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachments — additional branches`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should return immediately with empty reports when fromDevRev is not set', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = undefined; - - // Act - const result = await adapter.loadAttachments({ create: jest.fn() }); - - // Assert - expect(result.reports).toEqual([]); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit AttachmentLoadingDelayed and stop the loop when the connector signals a rate-limit delay', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'art-1', - file_name: 'attachments.json', - itemType: 'attachment', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ - response: [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ], - }); - jest - // eslint-disable-next-line @typescript-eslint/no-explicit-any - .spyOn(adapter as any, 'loadAttachment') - .mockResolvedValue({ rateLimit: { delay: 20 } } as never); - - // Act - await adapter.loadAttachments({ create: jest.fn() }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingDelayed, - expect.objectContaining({ delay: 20 }) - ); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachment`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - function makeAttachment(): ExternalSystemAttachment { - return { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - } as ExternalSystemAttachment; - } - - it('should return a CREATED report when create succeeds', async () => { - // Arrange - adapter['_mappers'].create = jest.fn().mockResolvedValue({ data: {} }); - const create = jest.fn().mockResolvedValue({ id: 'att-ext-1' }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.item_type).toBe('attachment'); - expect(result.report?.[ActionType.CREATED]).toBe(1); - }); - - it('should still return CREATED even when mapper create fails — attachment loading is resilient', async () => { - // Arrange - adapter['_mappers'].create = jest - .fn() - .mockRejectedValue(new Error('mapper failed')); - const create = jest.fn().mockResolvedValue({ id: 'att-ext-1' }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.[ActionType.CREATED]).toBe(1); - }); - - it('should propagate rate-limit delay when the connector signals one', async () => { - // Arrange - const create = jest.fn().mockResolvedValue({ delay: 30 }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.rateLimit?.delay).toBe(30); - }); - - it('should return a FAILED report when create returns neither id nor delay', async () => { - // Arrange - const create = jest.fn().mockResolvedValue({ id: null, delay: null }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.[ActionType.FAILED]).toBe(1); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts b/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts deleted file mode 100644 index bf488fd7..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { jsonl } from 'js-jsonl'; - -// Pin the serialization contract for items that reach the uploader. -// -// The SDK uploads items via Uploader.upload(), which calls jsonl.stringify() on -// the input. That means user `create`/`update` callbacks and normalizers can -// silently produce inputs that fail or lose information at the wire boundary: -// -// - Circular references throw "Converting circular structure to JSON" -// - BigInt values throw "Do not know how to serialize a BigInt" -// - Date objects are converted to ISO strings (information loss: no Date on -// the other side, just a string) -// - undefined fields are dropped (null fields are preserved) -// -// These tests exist to catch regressions in the serialization layer (e.g., -// silently switching to a different serializer that masks BigInt or mangles -// Dates) before they reach production. - -describe('serialization boundary for items uploaded via jsonl', () => { - it('throws when an item contains a circular reference', () => { - // Arrange - const item: Record = { id: 'a' }; - item.self = item; - - // Act & Assert - expect(() => jsonl.stringify([item])).toThrow(/circular/i); - }); - - it('throws when an item contains a BigInt field', () => { - // Arrange - const item = { id: 'a', counter: BigInt(1) }; - - // Act & Assert - expect(() => jsonl.stringify([item])).toThrow(/BigInt/i); - }); - - it('serializes Date instances to ISO strings (information loss — consumer receives a string)', () => { - // Arrange - const item = { - id: 'a', - created: new Date('2025-01-01T00:00:00.000Z'), - }; - - // Act - const output = jsonl.stringify([item]); - const parsed = JSON.parse(output) as Record; - - // Assert - expect(parsed.created).toBe('2025-01-01T00:00:00.000Z'); - expect(typeof parsed.created).toBe('string'); - }); - - it('drops undefined fields but preserves null fields', () => { - // Arrange - const item = { - id: 'a', - present: null, - missing: undefined, - }; - - // Act - const output = jsonl.stringify([item]); - const parsed = JSON.parse(output) as Record; - - // Assert - expect(parsed).toEqual({ id: 'a', present: null }); - expect(parsed).not.toHaveProperty('missing'); - }); - - it('emits one newline-terminated line per item (jsonl format)', () => { - // Arrange - const items = [{ id: 'a' }, { id: 'b' }, { id: 'c' }]; - - // Act - const output = jsonl.stringify(items); - const lines = output.split('\n').filter((l) => l.length > 0); - - // Assert - expect(lines).toHaveLength(3); - expect(JSON.parse(lines[0])).toEqual({ id: 'a' }); - expect(JSON.parse(lines[2])).toEqual({ id: 'c' }); - }); -}); diff --git a/src/multithreading/worker.js b/src/multithreading/worker.js index f19e44c6..784df889 100644 --- a/src/multithreading/worker.js +++ b/src/multithreading/worker.js @@ -3,7 +3,7 @@ const { workerData } = require('node:worker_threads'); require('ts-node').register(); const { Logger } = require('../logger/logger'); -const { runWithUserLogContext } = require('../logger/logger.context'); +const { runWithUserLogContext } = require('../logger/logger-context'); console = new Logger({ event: workerData.event, options: workerData.options }); diff --git a/src/repo/repo.test.ts b/src/repo/repo.test.ts index b1d93c76..148bb58a 100644 --- a/src/repo/repo.test.ts +++ b/src/repo/repo.test.ts @@ -1,7 +1,7 @@ import { AirSyncDefaultItemTypes, SSOR_ATTACHMENT } from '../common/constants'; import { createItems, normalizeItem } from '../tests/test-helpers'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types'; import { Repo } from './repo'; diff --git a/src/repo/repo.ts b/src/repo/repo.ts index d119eea8..df2687ac 100644 --- a/src/repo/repo.ts +++ b/src/repo/repo.ts @@ -9,7 +9,7 @@ import { Uploader } from '../uploader/uploader'; import { Artifact } from '../uploader/uploader.interfaces'; import { WorkerAdapterOptions } from '../types/workers'; -import { runWithUserLogContext } from '../logger/logger.context'; +import { runWithUserLogContext } from '../logger/logger-context'; import { NormalizedAttachment, NormalizedItem, diff --git a/src/state/base-state.ts b/src/state/base-state.ts index 06db8959..0ce50a94 100644 --- a/src/state/base-state.ts +++ b/src/state/base-state.ts @@ -1,8 +1,8 @@ import axios from 'axios'; import { parentPort } from 'node:worker_threads'; -import { installInitialDomainMapping } from '../common/install-initial-domain-mapping'; -import { axiosClient } from '../http/axios-client-internal'; +import { installInitialDomainMapping } from './install-initial-domain-mapping'; +import { axiosClient } from '../http/client'; import { getPrintableState, serializeError } from '../logger/logger'; import { InitialDomainMapping } from '../types/common'; import { AirSyncEvent } from '../types/extraction'; diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts index b78dbbc0..24d51083 100644 --- a/src/state/extraction-state.ts +++ b/src/state/extraction-state.ts @@ -1,7 +1,7 @@ import { parentPort } from 'node:worker_threads'; import { STATELESS_EVENT_TYPES } from '../common/constants'; -import { resolveTimeValue } from '../common/time-value-resolver'; +import { resolveTimeValue } from './time-value-resolver'; import { serializeError } from '../logger/logger'; import { EventType } from '../types/extraction'; import { WorkerMessageSubject } from '../types/workers'; diff --git a/src/common/install-initial-domain-mapping.test.ts b/src/state/install-initial-domain-mapping.test.ts similarity index 97% rename from src/common/install-initial-domain-mapping.test.ts rename to src/state/install-initial-domain-mapping.test.ts index ec56b02c..bbed61bb 100644 --- a/src/common/install-initial-domain-mapping.test.ts +++ b/src/state/install-initial-domain-mapping.test.ts @@ -1,7 +1,7 @@ import axios from 'axios'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from './test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { InitialDomainMapping } from '../types'; import { EventType } from '../types/extraction'; import { installInitialDomainMapping } from './install-initial-domain-mapping'; @@ -11,8 +11,8 @@ jest.mock('axios', () => ({ ...jest.requireActual('axios'), isAxiosError: jest.fn(), })); -jest.mock('../http/axios-client-internal', () => { - const originalModule = jest.requireActual('../http/axios-client-internal'); +jest.mock('../http/client', () => { + const originalModule = jest.requireActual('../http/client'); return { ...originalModule, axiosClient: { diff --git a/src/common/install-initial-domain-mapping.ts b/src/state/install-initial-domain-mapping.ts similarity index 97% rename from src/common/install-initial-domain-mapping.ts rename to src/state/install-initial-domain-mapping.ts index 5a8ec436..9cfffbd0 100644 --- a/src/common/install-initial-domain-mapping.ts +++ b/src/state/install-initial-domain-mapping.ts @@ -1,4 +1,4 @@ -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { AirSyncEvent } from '../types/extraction'; import { serializeError } from '../logger/logger'; diff --git a/src/state/state.extract-window.test.ts b/src/state/state.extract-window.test.ts index 74d76452..c0cd0f68 100644 --- a/src/state/state.extract-window.test.ts +++ b/src/state/state.extract-window.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; import { State, createAdapterState } from './state'; diff --git a/src/state/state.pending-boundaries.test.ts b/src/state/state.pending-boundaries.test.ts index 78c00858..4576e73b 100644 --- a/src/state/state.pending-boundaries.test.ts +++ b/src/state/state.pending-boundaries.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; import { State, createAdapterState } from './state'; @@ -19,7 +19,7 @@ describe('State — pending extraction boundaries', () => { postStateSpy = jest.spyOn(State.prototype, 'postState'); fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( - require('../common/install-initial-domain-mapping'), + require('./install-initial-domain-mapping'), 'installInitialDomainMapping' ); jest.spyOn(process, 'exit').mockImplementation(() => { diff --git a/src/state/state.post-state.test.ts b/src/state/state.post-state.test.ts index 83ff39e0..dc2a75d0 100644 --- a/src/state/state.post-state.test.ts +++ b/src/state/state.post-state.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; import { State, createAdapterState } from './state'; @@ -77,7 +77,7 @@ describe('State.postState', () => { }); // Mock axiosClient.post directly to bypass the retry backoff - const axiosClientModule = require('../http/axios-client-internal'); + const axiosClientModule = require('../http/client'); const axiosPostSpy = jest .spyOn(axiosClientModule.axiosClient, 'post') .mockRejectedValue(new Error('network error')); diff --git a/src/state/state.test.ts b/src/state/state.test.ts index e8a2e5ce..2c0b33e9 100644 --- a/src/state/state.test.ts +++ b/src/state/state.test.ts @@ -3,7 +3,7 @@ import { STATELESS_EVENT_TYPES, } from '../common/constants'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; import { State, createAdapterState } from './state'; import { extractionSdkState } from './state.interfaces'; @@ -25,7 +25,7 @@ describe(State.name, () => { postStateSpy = jest.spyOn(State.prototype, 'postState'); fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( - require('../common/install-initial-domain-mapping'), + require('./install-initial-domain-mapping'), 'installInitialDomainMapping' ); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { diff --git a/src/state/state.time-value-resolution.test.ts b/src/state/state.time-value-resolution.test.ts index a2265a58..9c1cb814 100644 --- a/src/state/state.time-value-resolution.test.ts +++ b/src/state/state.time-value-resolution.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValue, TimeValueType } from '../types/extraction'; import { State, createAdapterState } from './state'; diff --git a/src/common/time-value-resolver.test.ts b/src/state/time-value-resolver.test.ts similarity index 99% rename from src/common/time-value-resolver.test.ts rename to src/state/time-value-resolver.test.ts index f84a3dab..43ccf7ef 100644 --- a/src/common/time-value-resolver.test.ts +++ b/src/state/time-value-resolver.test.ts @@ -1,6 +1,6 @@ import { TimeValueType } from '../types/extraction'; import { SdkState } from '../state/state.interfaces'; -import { UNBOUNDED_DATE_TIME_VALUE } from './constants'; +import { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; import { parseDuration, applyDuration, diff --git a/src/common/time-value-resolver.ts b/src/state/time-value-resolver.ts similarity index 98% rename from src/common/time-value-resolver.ts rename to src/state/time-value-resolver.ts index ce2494d8..9a8f9cfe 100644 --- a/src/common/time-value-resolver.ts +++ b/src/state/time-value-resolver.ts @@ -1,6 +1,6 @@ import { TimeUnit, TimeValue, TimeValueType } from '../types/extraction'; import { SdkState } from '../state/state.interfaces'; -import { UNBOUNDED_DATE_TIME_VALUE } from './constants'; +import { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; /** * Parses a shorthand duration string into its numeric value and unit. diff --git a/src/mock-server/README.md b/src/testing/README.md similarity index 100% rename from src/mock-server/README.md rename to src/testing/README.md diff --git a/src/common/test-utils.ts b/src/testing/mock-event.ts similarity index 98% rename from src/common/test-utils.ts rename to src/testing/mock-event.ts index 9e4372ed..e503bf65 100644 --- a/src/common/test-utils.ts +++ b/src/testing/mock-event.ts @@ -1,6 +1,6 @@ import { AirSyncEvent, EventType } from '../types/extraction'; -export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; +import { MOCK_SERVER_DEFAULT_URL } from './mock-server'; /** * Recursively makes all properties of T optional. diff --git a/src/mock-server/mock-server.interfaces.ts b/src/testing/mock-server.interfaces.ts similarity index 100% rename from src/mock-server/mock-server.interfaces.ts rename to src/testing/mock-server.interfaces.ts diff --git a/src/mock-server/mock-server.ts b/src/testing/mock-server.ts similarity index 97% rename from src/mock-server/mock-server.ts rename to src/testing/mock-server.ts index 7c196996..eaa45dad 100644 --- a/src/mock-server/mock-server.ts +++ b/src/testing/mock-server.ts @@ -10,6 +10,12 @@ import { RouteHandlers, } from './mock-server.interfaces'; +/** + * Default base URL for the mock server. The port `0` lets the OS assign a free + * port at listen time; tests read the resolved URL from `mockServer.baseUrl`. + */ +export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; + const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10mb /** diff --git a/src/tests/control-protocol-reconciliation.test.ts b/src/tests/control-protocol-reconciliation.test.ts index 26752f2f..9cee1039 100644 --- a/src/tests/control-protocol-reconciliation.test.ts +++ b/src/tests/control-protocol-reconciliation.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; describe('Enhanced Control Protocol', () => { diff --git a/src/tests/dummy-connector/data-extraction.test.ts b/src/tests/dummy-connector/data-extraction.test.ts index 82b812a1..09a4d331 100644 --- a/src/tests/dummy-connector/data-extraction.test.ts +++ b/src/tests/dummy-connector/data-extraction.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/dummy-connector/external-sync-units-extraction.test.ts b/src/tests/dummy-connector/external-sync-units-extraction.test.ts index d0f87c50..a3aeaf14 100644 --- a/src/tests/dummy-connector/external-sync-units-extraction.test.ts +++ b/src/tests/dummy-connector/external-sync-units-extraction.test.ts @@ -1,6 +1,6 @@ import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/dummy-connector/metadata-extraction.test.ts b/src/tests/dummy-connector/metadata-extraction.test.ts index 18ac043a..02d19d6d 100644 --- a/src/tests/dummy-connector/metadata-extraction.test.ts +++ b/src/tests/dummy-connector/metadata-extraction.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/event-data-size-limit/size-limit-1.test.ts b/src/tests/event-data-size-limit/size-limit-1.test.ts index 72850e8e..0b1ea7aa 100644 --- a/src/tests/event-data-size-limit/size-limit-1.test.ts +++ b/src/tests/event-data-size-limit/size-limit-1.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; // Increase timeout for this test since we're doing many uploads diff --git a/src/tests/extract-from-collision.test.ts b/src/tests/extract-from-collision.test.ts index a5e8dbb5..b6106648 100644 --- a/src/tests/extract-from-collision.test.ts +++ b/src/tests/extract-from-collision.test.ts @@ -1,5 +1,5 @@ import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { EventContext, EventType, TimeValueType } from '../types/extraction'; /** diff --git a/src/tests/jest.setup.ts b/src/tests/jest.setup.ts index 87c86273..8887f19b 100644 --- a/src/tests/jest.setup.ts +++ b/src/tests/jest.setup.ts @@ -1,4 +1,4 @@ -import { MockServer } from '../mock-server/mock-server'; +import { MockServer } from '../testing/mock-server'; // Use port 0 for dynamic port allocation, enabling parallel test execution export const mockServer = new MockServer(0); diff --git a/src/tests/spawn-worker/delete-event-type.test.ts b/src/tests/spawn-worker/delete-event-type.test.ts index b2b2e445..c65f6c77 100644 --- a/src/tests/spawn-worker/delete-event-type.test.ts +++ b/src/tests/spawn-worker/delete-event-type.test.ts @@ -1,6 +1,6 @@ import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/spawn-worker/unknown-event-type.test.ts b/src/tests/spawn-worker/unknown-event-type.test.ts index 00ca5f0a..7d04f8e0 100644 --- a/src/tests/spawn-worker/unknown-event-type.test.ts +++ b/src/tests/spawn-worker/unknown-event-type.test.ts @@ -1,6 +1,6 @@ import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/test-helpers.interfaces.ts b/src/tests/test-helpers.interfaces.ts index 085d7c3a..987c0827 100644 --- a/src/tests/test-helpers.interfaces.ts +++ b/src/tests/test-helpers.interfaces.ts @@ -1,4 +1,4 @@ -import { DeepPartial } from '../common/test-utils'; +import { DeepPartial } from '../testing/mock-event'; import { AirdropEvent } from '../types/extraction'; /** diff --git a/src/tests/timeout-handling/no-timeout.test.ts b/src/tests/timeout-handling/no-timeout.test.ts index 7b9df7dc..22582eed 100644 --- a/src/tests/timeout-handling/no-timeout.test.ts +++ b/src/tests/timeout-handling/no-timeout.test.ts @@ -5,7 +5,7 @@ import { } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/timeout-handling/timeout-blocked.test.ts b/src/tests/timeout-handling/timeout-blocked.test.ts index ec66e516..daecc18c 100644 --- a/src/tests/timeout-handling/timeout-blocked.test.ts +++ b/src/tests/timeout-handling/timeout-blocked.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/timeout-handling/timeout-graceful.test.ts b/src/tests/timeout-handling/timeout-graceful.test.ts index f37a5062..1a5071da 100644 --- a/src/tests/timeout-handling/timeout-graceful.test.ts +++ b/src/tests/timeout-handling/timeout-graceful.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/tests/timeout-handling/timeout-unblocked.test.ts b/src/tests/timeout-handling/timeout-unblocked.test.ts index 8f4fe8b5..3747fe24 100644 --- a/src/tests/timeout-handling/timeout-unblocked.test.ts +++ b/src/tests/timeout-handling/timeout-unblocked.test.ts @@ -4,7 +4,7 @@ import { ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; +import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; diff --git a/src/common/errors.ts b/src/types/errors.ts similarity index 100% rename from src/common/errors.ts rename to src/types/errors.ts diff --git a/src/types/extraction.test.ts b/src/types/extraction.test.ts index 1dd9cb9e..5453ee81 100644 --- a/src/types/extraction.test.ts +++ b/src/types/extraction.test.ts @@ -1,4 +1,4 @@ -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; import { EventType, InitialSyncScope, TimeValueType } from './extraction'; diff --git a/src/uploader/uploader.test.ts b/src/uploader/uploader.test.ts index 41e97775..a2546ee8 100644 --- a/src/uploader/uploader.test.ts +++ b/src/uploader/uploader.test.ts @@ -3,7 +3,7 @@ import FormData from 'form-data'; import { jsonl } from 'js-jsonl'; import zlib from 'zlib'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { callPrivateMethod, createArtifact, @@ -14,13 +14,13 @@ import { spyOnPrivateMethod, } from '../tests/test-helpers'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { compressGzip, downloadToLocal } from './uploader.helpers'; import { ArtifactToUpload, UploaderResult } from './uploader.interfaces'; import { Uploader } from './uploader'; -jest.mock('../http/axios-client-internal'); +jest.mock('../http/client'); jest.mock('./uploader.helpers', () => ({ ...jest.requireActual('./uploader.helpers'), downloadToLocal: jest.fn(), diff --git a/src/uploader/uploader.ts b/src/uploader/uploader.ts index f35612cf..f729cbd9 100644 --- a/src/uploader/uploader.ts +++ b/src/uploader/uploader.ts @@ -1,7 +1,7 @@ import { AxiosResponse } from 'axios'; import FormData from 'form-data'; import { jsonl } from 'js-jsonl'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { MAX_DEVREV_ARTIFACT_SIZE } from '../common/constants'; import { NormalizedAttachment } from '../repo/repo.interfaces'; From 253cb5cb3bbeb14e8b7138e8ecae1da43b5f2b47 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 16 Jun 2026 14:01:52 +0200 Subject: [PATCH 26/30] refactor(v2): fold logger-context + logger-constants into logger.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge the two one-off logger peer files back into logger.ts and condense their JSDoc. Both were rejected for folding in the earlier structure pass on cycle/importer-count grounds; re-checking the live tree showed the fold is clean once one misfiled helper moves: - logger-context.ts is a pure leaf (only node:async_hooks) and 5 of its 6 importers already import logger.ts; the 6th (repo.ts) pulls it transitively via Uploader. No cycle. - logger-constants.ts alone would cycle: common/helpers.ts:truncateMessage reads MAX_LOG_STRING_LENGTH and logger.ts imports truncateMessage. Resolved by moving truncateMessage itself into logger.ts — it is a log-formatting helper misfiled in the common/ grab-bag (only callers: logger.ts + base-adapter.ts). This severs the cycle and improves cohesion. Importers repointed to ./logger for runWith*/getSdkLogContextValue/ serializeError/truncateMessage. logger/ is now logger.ts + logger.interfaces.ts + logger.test.ts. Public barrel unchanged (these symbols were always internal). Production typecheck, lint, and build clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/common/helpers.ts | 18 ---- src/logger/logger-constants.ts | 13 --- src/logger/logger-context.ts | 85 ------------------- src/logger/logger.test.ts | 6 +- src/logger/logger.ts | 62 +++++++++++++- src/multithreading/adapters/base-adapter.ts | 8 +- .../adapters/extraction-adapter.ts | 4 +- .../adapters/loading-adapter.ts | 4 +- src/multithreading/process-task.test.ts | 3 - src/multithreading/process-task.ts | 5 +- src/multithreading/spawn/spawn.test.ts | 2 - src/multithreading/worker.js | 3 +- src/repo/repo.ts | 2 +- 13 files changed, 74 insertions(+), 141 deletions(-) delete mode 100644 src/logger/logger-constants.ts delete mode 100644 src/logger/logger-context.ts diff --git a/src/common/helpers.ts b/src/common/helpers.ts index 5e4d2785..d4ef030b 100644 --- a/src/common/helpers.ts +++ b/src/common/helpers.ts @@ -2,8 +2,6 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import * as v8 from 'v8'; -import { MAX_LOG_STRING_LENGTH } from '../logger/logger-constants'; - /** * Gets the library version from the package.json file. * @returns {string} The library version @@ -111,19 +109,3 @@ export function getMemoryUsage(): MemoryInfo { throw err; } } - -/** - * Truncates a message if it exceeds the maximum allowed length. - * Adds a suffix indicating how many characters were omitted. - * - * @param message - The message to truncate - * @returns Truncated message or original if within limits - */ -export function truncateMessage(message: string): string { - if (message.length > MAX_LOG_STRING_LENGTH) { - return `${message.substring(0, MAX_LOG_STRING_LENGTH)}... ${ - message.length - MAX_LOG_STRING_LENGTH - } more characters`; - } - return message; -} diff --git a/src/logger/logger-constants.ts b/src/logger/logger-constants.ts deleted file mode 100644 index b183cb43..00000000 --- a/src/logger/logger-constants.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { InspectOptions } from 'node:util'; - -export const MAX_LOG_STRING_LENGTH = 10000; -export const MAX_LOG_DEPTH = 10; -export const MAX_LOG_ARRAY_LENGTH = 100; - -export const INSPECT_OPTIONS: InspectOptions = { - compact: false, - breakLength: Infinity, - depth: MAX_LOG_DEPTH, - maxArrayLength: MAX_LOG_ARRAY_LENGTH, - maxStringLength: MAX_LOG_STRING_LENGTH, -}; diff --git a/src/logger/logger-context.ts b/src/logger/logger-context.ts deleted file mode 100644 index 59378788..00000000 --- a/src/logger/logger-context.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { AsyncLocalStorage } from 'node:async_hooks'; - -/** - * Async-aware context storage for tracking whether code is executing within SDK or user context. - * - * AsyncLocalStorage is used instead of a simple global variable because it automatically - * propagates the context across async boundaries (await, Promise.then, setTimeout, etc.). - * This ensures that even after multiple await calls in user code, the logging context - * remains correct without requiring explicit context passing through function parameters. - * - * The stored boolean value indicates: - * - `true`: Code is executing within SDK internals (logs tagged as is_sdk_log: true) - * - `false`: Code is executing within user-provided handlers (logs tagged as is_sdk_log: false) - */ -const sdkLogContext = new AsyncLocalStorage(); - -/** - * Executes a function within user log context, marking all logs as user-originated. - * - * Use this to wrap user-provided callback functions (e.g., task handlers, event callbacks). - * The context automatically propagates through any async operations within the function, - * ensuring that all console.log calls inside are correctly tagged as user logs. - * - * @template T - The return type of the function - * @param fn - The function to execute within user context (can be sync or async) - * @returns The result of the function execution - * - * @example - * ```typescript - * await runWithUserLogContext(async () => { - * console.log('This is a user log'); // is_sdk_log: false - * await someAsyncOperation(); - * console.log('Still a user log'); // is_sdk_log: false (context preserved) - * }); - * ``` - */ -export function runWithUserLogContext(fn: () => T): T { - return sdkLogContext.run(false, fn); -} - -/** - * Executes a function within SDK log context, marking all logs as SDK-originated. - * - * Use this to wrap SDK internal operations (e.g., emit, postState, adapter methods). - * The context automatically propagates through any async operations within the function, - * ensuring that all console.log calls inside are correctly tagged as SDK logs. - * - * This allows proper nesting: SDK code can call user code via runWithUserLogContext, - * and when control returns to SDK code, logs are correctly attributed. - * - * @template T - The return type of the function - * @param fn - The function to execute within SDK context (can be sync or async) - * @returns The result of the function execution - * - * @example - * ```typescript - * await runWithSdkLogContext(async () => { - * console.log('SDK internal log'); // is_sdk_log: true - * runWithUserLogContext(() => { - * console.log('User handler log'); // is_sdk_log: false - * }); - * console.log('Back to SDK log'); // is_sdk_log: true - * }); - * ``` - */ -export function runWithSdkLogContext(fn: () => T): T { - return sdkLogContext.run(true, fn); -} - -/** - * Retrieves the current SDK log context value. - * - * Returns whether the current execution context is within SDK code (true) or user code (false). - * If no context has been set (e.g., during testing or edge cases), returns the provided default. - * - * @param defaultValue - The value to return if no context is currently set - * @returns `true` if in SDK context, `false` if in user context, or defaultValue if unset - */ -export function getSdkLogContextValue(defaultValue: boolean): boolean { - const storeValue = sdkLogContext.getStore(); - if (typeof storeValue === 'boolean') { - return storeValue; - } - return defaultValue; -} diff --git a/src/logger/logger.test.ts b/src/logger/logger.test.ts index 2f882505..be30fcf6 100644 --- a/src/logger/logger.test.ts +++ b/src/logger/logger.test.ts @@ -7,14 +7,12 @@ import { AirdropEvent, EventType } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; import { getPrintableState, + INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, Logger, + MAX_LOG_STRING_LENGTH, serializeAxiosError, serializeError, } from './logger'; -import { - INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, - MAX_LOG_STRING_LENGTH, -} from './logger-constants'; // Mock console methods const mockConsoleInfo = jest.spyOn(console, 'info').mockImplementation(); diff --git a/src/logger/logger.ts b/src/logger/logger.ts index fb7b1e10..3d1c5432 100644 --- a/src/logger/logger.ts +++ b/src/logger/logger.ts @@ -1,14 +1,13 @@ import { AxiosError, isAxiosError, RawAxiosResponseHeaders } from 'axios'; +import { AsyncLocalStorage } from 'node:async_hooks'; import { Console } from 'node:console'; -import { inspect } from 'node:util'; +import { inspect, InspectOptions } from 'node:util'; import { isMainThread, parentPort } from 'node:worker_threads'; import { LIBRARY_VERSION } from '../common/constants'; import { WorkerAdapterOptions, WorkerMessageSubject } from '../types/workers'; -import { INSPECT_OPTIONS } from './logger-constants'; -import { getSdkLogContextValue } from './logger-context'; import { AxiosErrorResponse, LoggerFactoryInterface, @@ -17,7 +16,62 @@ import { PrintableArray, PrintableState, } from './logger.interfaces'; -import { truncateMessage } from '../common/helpers'; + +// ── Log formatting ── + +export const MAX_LOG_STRING_LENGTH = 10000; +const MAX_LOG_DEPTH = 10; +const MAX_LOG_ARRAY_LENGTH = 100; + +export const INSPECT_OPTIONS: InspectOptions = { + compact: false, + breakLength: Infinity, + depth: MAX_LOG_DEPTH, + maxArrayLength: MAX_LOG_ARRAY_LENGTH, + maxStringLength: MAX_LOG_STRING_LENGTH, +}; + +/** + * Truncates a message that exceeds the maximum length, appending a suffix + * noting how many characters were omitted. + */ +export function truncateMessage(message: string): string { + if (message.length > MAX_LOG_STRING_LENGTH) { + return `${message.substring(0, MAX_LOG_STRING_LENGTH)}... ${ + message.length - MAX_LOG_STRING_LENGTH + } more characters`; + } + return message; +} + +// ── SDK log context ── + +/** + * Async-aware flag tracking whether code is running in SDK internals (`true`) + * or user-provided handlers (`false`). AsyncLocalStorage propagates the value + * across await/Promise/timer boundaries, so logs stay correctly attributed + * without threading the flag through call signatures. + */ +const sdkLogContext = new AsyncLocalStorage(); + +/** Runs `fn` with logs marked as user-originated (`is_sdk_log: false`). */ +export function runWithUserLogContext(fn: () => T): T { + return sdkLogContext.run(false, fn); +} + +/** Runs `fn` with logs marked as SDK-originated (`is_sdk_log: true`). */ +export function runWithSdkLogContext(fn: () => T): T { + return sdkLogContext.run(true, fn); +} + +/** Returns the current context flag, or `defaultValue` if none is set. */ +export function getSdkLogContextValue(defaultValue: boolean): boolean { + const storeValue = sdkLogContext.getStore(); + if (typeof storeValue === 'boolean') { + return storeValue; + } + return defaultValue; +} /** * Custom logger that extends Node.js Console with context-aware logging. diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts index c5f46188..0cca182f 100644 --- a/src/multithreading/adapters/base-adapter.ts +++ b/src/multithreading/adapters/base-adapter.ts @@ -2,9 +2,11 @@ import { parentPort } from 'node:worker_threads'; import { STATELESS_EVENT_TYPES } from '../../common/constants'; import { emit } from '../emit'; -import { truncateMessage } from '../../common/helpers'; -import { serializeError } from '../../logger/logger'; -import { runWithSdkLogContext } from '../../logger/logger-context'; +import { + runWithSdkLogContext, + serializeError, + truncateMessage, +} from '../../logger/logger'; import { BaseState } from '../../state/state'; import { SdkState } from '../../state/state.interfaces'; import { diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index 367171ba..7ae5a062 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -4,11 +4,11 @@ import { EVENT_SIZE_THRESHOLD_BYTES, SSOR_ATTACHMENT, } from '../../common/constants'; -import { serializeError } from '../../logger/logger'; import { runWithSdkLogContext, runWithUserLogContext, -} from '../../logger/logger-context'; + serializeError, +} from '../../logger/logger'; import { Repo } from '../../repo/repo'; import { NormalizedAttachment, diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts index 68e7b5a8..3bf1551b 100644 --- a/src/multithreading/adapters/loading-adapter.ts +++ b/src/multithreading/adapters/loading-adapter.ts @@ -1,10 +1,10 @@ import axios from 'axios'; -import { serializeError } from '../../logger/logger'; import { runWithSdkLogContext, runWithUserLogContext, -} from '../../logger/logger-context'; + serializeError, +} from '../../logger/logger'; import { Mappers } from '../../mappers/mappers'; import { SyncMapperRecordStatus } from '../../mappers/mappers.interfaces'; import { BaseState } from '../../state/state'; diff --git a/src/multithreading/process-task.test.ts b/src/multithreading/process-task.test.ts index 2c5b06d0..04dcce38 100644 --- a/src/multithreading/process-task.test.ts +++ b/src/multithreading/process-task.test.ts @@ -47,9 +47,6 @@ jest.mock('../logger/logger', () => ({ logFn: jest.fn(), })), serializeError: jest.fn((e: unknown) => String(e)), -})); - -jest.mock('../logger/logger-context', () => ({ runWithSdkLogContext: jest.fn((fn: () => unknown) => fn()), runWithUserLogContext: jest.fn((fn: () => unknown) => fn()), })); diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 6c59250d..f6a594f0 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -1,10 +1,11 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads'; -import { Logger, serializeError } from '../logger/logger'; import { + Logger, runWithSdkLogContext, runWithUserLogContext, -} from '../logger/logger-context'; + serializeError, +} from '../logger/logger'; import { createExtractionState } from '../state/extraction-state'; import { createLoadingState } from '../state/loading-state'; import { diff --git a/src/multithreading/spawn/spawn.test.ts b/src/multithreading/spawn/spawn.test.ts index 43ac5f45..320725be 100644 --- a/src/multithreading/spawn/spawn.test.ts +++ b/src/multithreading/spawn/spawn.test.ts @@ -38,8 +38,6 @@ jest.mock('../../common/helpers', () => ({ arrayBuffersMB: '5.00', }), sleep: jest.fn(), - truncateFilename: jest.fn((f: string) => f), - truncateMessage: jest.fn((m: string) => m), })); // --------------------------------------------------------------------------- diff --git a/src/multithreading/worker.js b/src/multithreading/worker.js index 784df889..7f35818b 100644 --- a/src/multithreading/worker.js +++ b/src/multithreading/worker.js @@ -2,8 +2,7 @@ const { workerData } = require('node:worker_threads'); require('ts-node').register(); -const { Logger } = require('../logger/logger'); -const { runWithUserLogContext } = require('../logger/logger-context'); +const { Logger, runWithUserLogContext } = require('../logger/logger'); console = new Logger({ event: workerData.event, options: workerData.options }); diff --git a/src/repo/repo.ts b/src/repo/repo.ts index df2687ac..a54e30a4 100644 --- a/src/repo/repo.ts +++ b/src/repo/repo.ts @@ -9,7 +9,7 @@ import { Uploader } from '../uploader/uploader'; import { Artifact } from '../uploader/uploader.interfaces'; import { WorkerAdapterOptions } from '../types/workers'; -import { runWithUserLogContext } from '../logger/logger-context'; +import { runWithUserLogContext } from '../logger/logger'; import { NormalizedAttachment, NormalizedItem, From 2721f317d87466e1fc36bef7f75629e6f32dc6a3 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Tue, 16 Jun 2026 14:01:59 +0200 Subject: [PATCH 27/30] docs(v2): remove stale root docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete three root markdown files that had drifted out of sync with the v2 public surface, and drop README's link to the removed reference: - V2_PROGRESS.md — internal refactor-tracking scaffold, spent. - REFERENCE.md — 1337-line hand-written API doc, ~52 dead-symbol references. - MIGRATION.md — contained false claims about axios still being exported. A correct v1->v2 guide will be written in the deferred docs pass once the public surface is settled and the api report regenerates. Root markdown is now README.md + CONTRIBUTING.md (both accurate). Co-Authored-By: Claude Opus 4.8 (1M context) --- MIGRATION.md | 462 ----------------- README.md | 4 - REFERENCE.md | 1337 ------------------------------------------------ V2_PROGRESS.md | 309 ----------- 4 files changed, 2112 deletions(-) delete mode 100644 MIGRATION.md delete mode 100644 REFERENCE.md delete mode 100644 V2_PROGRESS.md diff --git a/MIGRATION.md b/MIGRATION.md deleted file mode 100644 index 5e1bcd3d..00000000 --- a/MIGRATION.md +++ /dev/null @@ -1,462 +0,0 @@ -# Migrating connectors from v1 (`@devrev/ts-adaas`) to v2 (`@devrev/airsync-sdk`) - -This guide covers every breaking change between `@devrev/ts-adaas` v1.x and -`@devrev/airsync-sdk` v2.0.0, with before/after examples taken from real -connectors. The wire protocol (event payloads sent to the platform, API routes, -artifact/upload flow) is **unchanged** — only the connector-facing API changed. - -## TL;DR — what breaks - -| # | Change | Impact | -|---|--------|--------| -| 1 | Package renamed `@devrev/ts-adaas` → `@devrev/airsync-sdk` | every import | -| 2 | `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage` | type annotations | -| 3 | Deprecated event-type enum members deleted | only if you used old `Extraction*` members | -| 4 | `processTask` → `processExtractionTask` / `processLoadingTask` | every worker file | -| 5 | `adapter.emit(...)` removed — tasks **return** a `TaskResult` instead | every worker file | -| 6 | External sync units must be pushed to a repo (no more `external_sync_units` in emit data) | ESU workers | -| 7 | `adapter.state` is connector-state only; SDK fields moved to `adapter.sdkState` | only if you read/wrote SDK fields | -| 8 | `WorkerAdapter` class/type removed → `ExtractionAdapter` / `LoadingAdapter` | helper-function signatures | -| 9 | `loadItemTypes` / `loadAttachments` / `streamAttachments` return a `TaskResult` | loading + attachment workers | -| 10 | Deprecated v1 modules deleted (`Adapter`, `createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, legacy `Uploader`) | only legacy code | -| 11 | Legacy `string[]` form of the processed-attachments dedup list is no longer migrated | in-flight attachment syncs started on SDK < 1.15.2 | - -What does **not** change: `spawn(...)` and all its options (`baseWorkerPath`, -`initialState`, `initialDomainMapping`, `options.batchSize`, `timeout`, -`isLocalDevelopment`, `workerPathOverrides`), default worker paths -(`/workers/data-extraction` etc.), repos (`initializeRepos`, `getRepo`, -`push`), normalization interfaces (`NormalizedItem`, `NormalizedAttachment`, -`RepoInterface`, `ExternalSyncUnit`), the uploader, mappers, logger, -`installInitialDomainMapping`, the `axios`/`axiosClient` re-exports, -`formatAxiosError`/`serializeAxiosError`, `createMockEvent`/`MockServer`, -HTTP retry behavior, and all event-type **string values** on the wire. - ---- - -## 1. Package rename - -```bash -npm uninstall @devrev/ts-adaas -npm install @devrev/airsync-sdk -``` - -Then global-replace the import specifier: - -```ts -// v1 -import { spawn, EventType } from '@devrev/ts-adaas'; -// v2 -import { spawn, EventType } from '@devrev/airsync-sdk'; -``` - -> **Deep imports** like `import { Item } from '@devrev/ts-adaas/dist/repo/repo.interfaces'` -> are fragile and should be replaced with root imports where the symbol is -> exported. If a symbol you need is not exported from the root, request it -> rather than deep-importing. - -## 2. Type renames - -Hard rename, no compatibility alias: - -| v1 | v2 | -|----|----| -| `AirdropEvent` | `AirSyncEvent` | -| `AirdropMessage` | `AirSyncMessage` | - -```ts -// v1 -const run = async (events: AirdropEvent[]) => { ... }; -// v2 -const run = async (events: AirSyncEvent[]) => { ... }; -``` - -Payload shape is identical — only the type name changed. Platform-owned strings -(`/internal/airdrop.*` routes, `'ADaaS'` external system type, -`adaas_library_version` metadata key, `airdrop_*` mapping enum values) are -intentionally unchanged. - -> `AirSyncEvent.context` now declares the identity fields the platform actually -> sends — `user_id`, `dev_oid`, `source_id`, `service_account_id` — alongside the -> existing `secrets`, `snap_in_id`, and `snap_in_version_id`. If you previously -> cast the event to a hand-rolled `CustomAirdropEvent` to read `context.user_id`, -> you can drop that cast and read the fields off `adapter.event.context` directly. - -## 3. Deleted deprecated enum members - -The old/new duplicate enum members were collapsed; only the new names remain. -The **string values of the kept members are byte-identical to v1**, so nothing -changes on the wire — only the TypeScript member names. - -> The **deleted** members carried *different, older* string values -> (e.g. v1 `ExtractionDataStart = 'EXTRACTION_DATA_START'` vs the kept -> `StartExtractingData = 'START_EXTRACTING_DATA'`). The platform stopped sending -> those old strings, and v2 no longer translates them (see the note at the end -> of this section), so the deletion is safe — but don't assume a deleted member -> and its replacement shared a value; they did not. - -**`EventType` (incoming):** - -| Deleted (v1 deprecated) | Use instead | -|--------------------------|-------------| -| `ExtractionExternalSyncUnitsStart` | `StartExtractingExternalSyncUnits` | -| `ExtractionMetadataStart` | `StartExtractingMetadata` | -| `ExtractionDataStart` | `StartExtractingData` | -| `ExtractionDataContinue` | `ContinueExtractingData` | -| `ExtractionDataDelete` | `StartDeletingExtractorState` | -| `ExtractionAttachmentsStart` | `StartExtractingAttachments` | -| `ExtractionAttachmentsContinue` | `ContinueExtractingAttachments` | -| `ExtractionAttachmentsDelete` | `StartDeletingExtractorAttachmentsState` | - -**`ExtractorEventType` (outgoing):** all `Extraction*` members -(`ExtractionDataDone`, `ExtractionDataDelay`, `ExtractionAttachmentsProgress`, …) -are deleted; use the `*Extraction*` members (`DataExtractionDone`, -`DataExtractionDelayed`, `AttachmentExtractionProgress`, …). In practice you -will rarely reference `ExtractorEventType` at all in v2 — see §5. - -**`LoaderEventType`:** the typo/duplicate members `DataLoadingDelay` and -`AttachmentsLoadingProgress/Delayed/Done/Error` are deleted; use -`DataLoadingDelayed` and `AttachmentLoading*`. - -Also note: the SDK no longer translates old event-type strings coming from the -platform — the platform sends only the new strings, and the SDK passes -`event.payload.event_type` through untouched. - -## 4 + 5. The new worker contract: return a `TaskResult` instead of emitting - -This is the core change of v2. In v1 the connector decided *which event* to -emit and called `adapter.emit(...)`. In v2 the connector only reports *how the -phase ended* by **returning** a `TaskResult`; the SDK maps it to the correct -platform event for the current phase and emits it exactly once: - -```ts -export type TaskResult = - | { status: 'success' } - | { status: 'progress' } - | { status: 'delay'; delaySeconds: number } - | { status: 'error'; error: ErrorRecord }; -``` - -Status → emitted event, per phase: - -| status | resumable phases (data/attachment extraction & loading) | non-resumable (ESU, metadata, state deletion) | -|--------|----------------------------------------------------------|-----------------------------------------------| -| `'success'` | `*Done` | `*Done` | -| `'progress'` | `*Progress` | `*Error` (illegal status; descriptive message) | -| `'delay'` | `*Delayed` (with `delay` seconds) | `*Error` (illegal status) | -| `'error'` | `*Error` (with the error record) | `*Error` | - -`processTask` is split into two typed entry points, and `onTimeout` is now -**optional** (defaults to a `progress` result — see the note below for -non-resumable phases): - -```ts -// v1 -import { processTask, ExtractorEventType } from '@devrev/ts-adaas'; - -processTask({ - task: async ({ adapter }) => { - // ... extract ... - await adapter.emit(ExtractorEventType.DataExtractionDone); - }, - onTimeout: async ({ adapter }) => { - await adapter.postState(); - await adapter.emit(ExtractorEventType.DataExtractionProgress, { progress: 50 }); - }, -}); -``` - -```ts -// v2 -import { processExtractionTask } from '@devrev/airsync-sdk'; - -processExtractionTask({ - task: async ({ adapter }) => { - // ... extract ... - return { status: 'success' }; - }, - // onTimeout can be omitted entirely: the SDK emits progress by default. -}); -``` - -Loading workers use `processLoadingTask` the same way. - -### Emit-call → return-value translation table - -| v1 | v2 | -|----|----| -| `await adapter.emit(XxxDone)` | `return { status: 'success' }` | -| `await adapter.emit(XxxProgress, { progress: n })` | `return { status: 'progress' }` (the `progress` % was already deprecated/ignored) | -| `await adapter.emit(XxxDelayed, { delay: seconds })` | `return { status: 'delay', delaySeconds: seconds }` | -| `await adapter.emit(XxxError, { error: { message } })` | `return { status: 'error', error: { message } }` | -| `await adapter.emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to repo + `return { status: 'success' }` — see §6 | - -Everything the SDK attached automatically in v1 is still attached -automatically: artifacts of uploaded repos (extraction events), `reports` and -`processed_files` (loading events), and state is still saved before every -non-stateless emit. An explicit `await adapter.postState()` before returning is -no longer needed for the emit path (it still works and is harmless). - -### Emits buried in helper functions - -A common v1 pattern is emitting deep inside helpers and signalling the caller -to stop: - -```ts -// v1 — helper emits, returns false to abort -async function extractList(adapter: WorkerAdapter, ...) { - if (rateLimited) { - await adapter.emit(ExtractorEventType.DataExtractionDelayed, { delay }); - return false; - } - ... -} -``` - -In v2 the helper must **bubble the outcome up** instead, because only the -task's return value reaches the SDK: - -```ts -// v2 — helper returns the outcome; task returns it to the SDK -async function extractList(adapter: ExtractionAdapter, ...): Promise { - if (rateLimited) return { status: 'delay', delaySeconds: delay }; - ... - return null; // keep going -} - -processExtractionTask({ - task: async ({ adapter }) => { - for (const itemType of itemTypes) { - const stop = await extractList(adapter, itemType); - if (stop) return stop; - } - return { status: 'success' }; - }, -}); -``` - -### Timeout handling inside the task - -Checking `adapter.isTimeout` in your extraction loop still works exactly like -v1 — but instead of emitting progress and exiting, return progress: - -```ts -// v2 -if (adapter.isTimeout) { - return { status: 'progress' }; // platform will send CONTINUE_* next -} -``` - -> **Behavior note — the timeout outcome always wins.** Once the soft timeout has -> fired, the SDK emits the `onTimeout` result (or the default `progress`) and -> **ignores whatever the task returned**, by design: a phase that ran out of time -> must hand off for continuation rather than report itself complete. So if you -> have cleanup or a specific outcome that must survive a timeout, put it in -> `onTimeout`, not in the task's return value. - -> **Non-resumable phases (ESU, metadata, state deletion):** the default -> `onTimeout` produces a `progress` result, which is illegal for these phases -> and is emitted as an error with a generic message. If you want a clean error -> message on timeout for these workers, provide an explicit `onTimeout`: -> `onTimeout: async () => ({ status: 'error', error: { message: 'Lambda timeout.' } })`. - -## 6. External sync units go through a repo - -In v1 the SDK accepted `external_sync_units` in the emit data and internally -uploaded them via a repo. With emit gone, push them to the -`EXTERNAL_SYNC_UNITS` repo yourself: - -```ts -// v1 -await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: externalSyncUnits, -}); -``` - -```ts -// v2 -import { AirSyncDefaultItemTypes, processExtractionTask } from '@devrev/airsync-sdk'; - -processExtractionTask({ - task: async ({ adapter }) => { - const externalSyncUnits = await fetchExternalSyncUnits(); - - adapter.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - // match the batching the v1 SDK used internally for ESUs - overridenOptions: { batchSize: 25000, skipConfirmation: true }, - }, - ]); - await adapter - .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) - ?.push(externalSyncUnits); - - return { status: 'success' }; - }, -}); -``` - -The repo is uploaded automatically before the `Done` event is emitted (same as -all repos). - -## 7. State split: `adapter.state` vs `adapter.sdkState` - -In v1, connector state and SDK bookkeeping lived in one flat object, persisted -as one blob, and `adapter.state` exposed both. In v2: - -- `adapter.state` is **connector state only** — exactly the shape of the - `initialState` you pass to `spawn`. Reads and writes work as before. -- SDK bookkeeping (`lastSyncStarted`, `lastSuccessfulSyncStarted`, - `workersOldest/Newest`, `pendingWorkers*`, `toDevRev`, `fromDevRev`, - `snapInVersionId`) moved to `adapter.sdkState`. It is SDK-internal; do not - write to it. If you were *reading* one of these fields (e.g. - `adapter.state.lastSuccessfulSyncStarted` for incremental sync), prefer - `event.payload.event_context.extract_from` / `extract_to`, which the SDK now - resolves for you. -- On disk, state is persisted as `{ connectorState, sdkState }`. The SDK - **migrates v1 flat blobs automatically on first read** (known SDK keys are - split out; everything else becomes connector state), so in-flight syncs - survive the upgrade. The migration is **not** version-gated — any flat blob is - split, regardless of which SDK version wrote it. (The one thing that is *not* - back-migrated is the legacy `string[]` form of the attachment dedup list — see - §9.) -- Edge case: if your v1 connector state had a top-level key literally named - `connectorState` or `sdkState`, the migration will misread it — rename it - before upgrading. - -```ts -// v1 — SDK fields visible on state (don't carry this over) -if (adapter.state.lastSuccessfulSyncStarted) { ... } - -// v2 — use the resolved extraction window instead -const { extract_from, extract_to } = adapter.event.payload.event_context; -``` - -Pure connector-state usage is unaffected: - -```ts -// works identically in v1 and v2 -adapter.state[itemType].cursor = nextCursor; -adapter.state[itemType].complete = true; -``` - -## 8. `WorkerAdapter` → `ExtractionAdapter` / `LoadingAdapter` - -The `WorkerAdapter` class no longer exists. Helper signatures change to the -mode-specific adapter: - -```ts -// v1 -async function extractList(adapter: WorkerAdapter, ...) { ... } -// v2 -import { ExtractionAdapter } from '@devrev/airsync-sdk'; -async function extractList(adapter: ExtractionAdapter, ...) { ... } -``` - -Extraction surface (`initializeRepos`, `getRepo`, `artifacts`, -`streamAttachments`, `processAttachment`, `shouldExtract`) lives on -`ExtractionAdapter`; loading surface (`loadItemTypes`, `loadAttachments`, -`loadItem`, `loadAttachment`, `mappers`, `reports`, `processedFiles`) lives on -`LoadingAdapter`. Shared: `event`, `state`, `sdkState`, `extractionScope`, -`postState`, `isTimeout`. - -> Only the `WorkerAdapter` **class** was removed. The **types** -> `WorkerAdapterInterface` and `WorkerAdapterOptions` still exist and are still -> exported, unchanged. Don't do a blind global rename of every `WorkerAdapter` -> token — replace the class/`WorkerAdapter` annotations only. - -> Symbols that previously required a deep `dist/` import are now on the root -> barrel: `Mappers` (class), `ItemTypeToLoad`, and `Item` all import from -> `@devrev/airsync-sdk` directly. `ToDevRev` is **not** exported — it is -> SDK-internal now (drop any use of it; see §7). - -## 9. SDK loading/streaming methods return a `TaskResult` - -`loadItemTypes`, `loadAttachments` and `streamAttachments` no longer emit or -exit mid-flight — they return a `TaskResult` you simply pass through: - -```ts -// v1 -processTask({ - task: async ({ adapter }) => { - const { reports, processed_files } = await adapter.loadItemTypes({ itemTypesToLoad }); - await adapter.emit(LoaderEventType.DataLoadingDone, { reports, processed_files }); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(LoaderEventType.DataLoadingProgress, { - reports: adapter.reports, - processed_files: adapter.processedFiles, - }); - }, -}); -``` - -```ts -// v2 -processLoadingTask({ - task: async ({ adapter }) => { - return adapter.loadItemTypes({ itemTypesToLoad }); - }, -}); -``` - -Rate limits (→ `delay`), timeouts (→ `progress`), errors (→ `error`) and -completion (→ `success`) are all encoded in the returned result; `reports` and -`processed_files` are attached to the emitted event automatically. - -Attachment extraction: - -```ts -// v1 -const response = await adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); -if (response?.delay) { - await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { delay: response.delay }); -} else if (response?.error) { - await adapter.emit(ExtractorEventType.AttachmentExtractionError, { error: response.error }); -} else { - await adapter.emit(ExtractorEventType.AttachmentExtractionDone); -} -``` - -```ts -// v2 -return adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); -``` - -Custom attachment `processors` (reducer/iterator) are still supported with the -same signatures. - -> **Edge regression — very old in-flight attachment syncs.** In v1 the SDK -> migrated the legacy `string[]` form of the processed-attachments dedup list -> (`sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList`) to the -> current `{ id, parent_id }[]` form on read. v2 removed that conversion. The -> `string[]` form only exists in state written by SDK **< 1.15.2**. If an -> attachment-extraction phase started on a pre-1.15.2 SDK and is *still mid-flight* -> when the connector upgrades to v2, the v2 dedup check (`it.id === …`) silently -> fails to match the bare-string entries, so attachments already downloaded in -> that sync get re-uploaded once. New syncs (and any sync started on ≥ 1.15.2) are -> unaffected. The duplicates are deduplicated downstream by the platform, so the -> only cost is wasted re-download/upload work on that one continuation. - -## 10. Deleted legacy modules - -Everything under the v1 `deprecated/` tree is gone: `Adapter`, -`createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, and the -legacy `Uploader` export. Replacements: the v2 adapters + `process*Task` for -`Adapter`/`createAdapter`; `axiosClient` for `HTTPClient`; repos for the legacy -uploader. (`formatAxiosError` is still exported.) - -## 11. Migration checklist - -1. `npm uninstall @devrev/ts-adaas && npm install @devrev/airsync-sdk`; replace import specifiers. -2. Rename `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage`. -3. Replace deleted enum members with their new names (§3) — values unchanged. -4. Split workers: extraction files use `processExtractionTask`, loading files use `processLoadingTask`. -5. Convert every `adapter.emit(...)` into a returned `TaskResult` (§4–5); bubble outcomes up from helpers. -6. ESU workers: push external sync units to the `EXTERNAL_SYNC_UNITS` repo (§6). -7. Replace `WorkerAdapter` annotations with `ExtractionAdapter` / `LoadingAdapter`. -8. Remove reads/writes of SDK-owned state fields from `adapter.state`; use the event context's `extract_from`/`extract_to` (§7). -9. Pass through the `TaskResult` from `loadItemTypes`/`loadAttachments`/`streamAttachments` (§9). -10. Decide per worker whether you need an explicit `onTimeout` (recommended for ESU/metadata to control the error message; usually omit elsewhere). -11. Remove deep `dist/**` imports and any usage of deleted legacy modules. -12. Update jest mocks of the SDK module (they hardcode the v1 shape: `processTask`, `WorkerAdapter`, old enum members). diff --git a/README.md b/README.md index e2fc2c9b..84898bf6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,3 @@ It provides features such as: ```bash npm install @devrev/airsync-sdk ``` - -## Reference - -Please refer to the [REFERENCE.md](./REFERENCE.md) file for more information on the types, interfaces and functions used in the library. diff --git a/REFERENCE.md b/REFERENCE.md deleted file mode 100644 index b949a3b4..00000000 --- a/REFERENCE.md +++ /dev/null @@ -1,1337 +0,0 @@ -## Reference - -### `SdkState` interface - -Defines the base state structure used by the Airdrop SDK. - -'SdkState' is an internal member that is not exported. - -#### Properties - -- _lastSyncStarted_ - - Optional. A **string** representing the timestamp when the last sync operation started. **Deprecated** - use `extract_from` and `extract_to` from the event context instead, which are automatically resolved by the SDK from `extraction_start_time` and `extraction_end_time`. - -- _lastSuccessfulSyncStarted_ - - Optional. A **string** representing the timestamp when the last successful sync operation started. **Deprecated** - use `extract_from` and `extract_to` from the event context instead, which are automatically resolved by the SDK from `extraction_start_time` and `extraction_end_time`. - -- _pendingWorkersOldest_ - - Optional. A **string** representing the pending (not yet committed) oldest extraction boundary as an ISO 8601 timestamp. Set on `StartExtractingData`, reused across subsequent phases, cleared on `AttachmentExtractionDone`. - -- _pendingWorkersNewest_ - - Optional. A **string** representing the pending (not yet committed) newest extraction boundary as an ISO 8601 timestamp. Set on `StartExtractingData`, reused across subsequent phases, cleared on `AttachmentExtractionDone`. - -- _workersOldest_ - - Optional. A **string** representing the oldest point of extraction as an ISO 8601 timestamp. - -- _workersNewest_ - - Optional. A **string** representing the newest point of extraction as an ISO 8601 timestamp. - -- _toDevRev_ - - Optional. An object of type **ToDevRev** containing data to be sent to DevRev. - -- _fromDevRev_ - - Optional. An object of type **FromDevRev** containing data received from DevRev. - -- _snapInVersionId_ - - Optional. A **string** representing the snap-in version ID. - -### `AdapterState` type - -A generic type that combines snap-in-specific state with the SDK's base state. - -#### Usage - -```typescript -type AdapterState = ConnectorState & SdkState; -``` - -The `AdapterState` type extends a snap-in's state type with additional fields from `SdkState`, providing a complete state structure to share with Airdrop platform. - -### `ToDevRev` interface - -Provides additional information within the state that is available only during data synchronization to DevRev (extraction). - -#### Properties - -- _attachmentsMetadata_ - - _artifactIds_: An array of **strings** containing artifact IDs - - _lastProcessed_: A **number** which is the index of the last processed attachment from the array - - _lastProcessedAttachmentsIdsList_: Optional. An array of **ProcessedAttachment** objects for deduplication on the SDK side - -### `FromDevRev` interface - -Provides additional information within the state that is available only during data synchronization from DevRev to external system (loading). - -#### Properties - -- _filesToLoad_ - - An array of **FileToLoad** objects representing files that need to be loaded. - -### `StateInterface` interface - -Defines the configuration structure for initializing state of the worker adapter. - -#### Properties - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _initialState_ - - Required. An object of type **ConnectorState** representing the initial state of the snap-in. - -- _initialDomainMapping_ - - Optional. An object of type **InitialDomainMapping** representing the initial domain mapping configuration. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions** for configuring the worker adapter. - -### `NormalizedItem` interface - -Represents the standardized structure of an item after normalization. - -#### Properties - -- _id_ - - Required. A **string** that uniquely identifies the normalized item. - -- _created_date_ - - Required. A **string** representing the timestamp, formatted as RFC3339, when the item was created. - -- _modified_date_ - - Required. A **string** representing the timestamp, formatted as RFC3339, when the item was last modified. - -- _data_ - - Required. An **object** containing the actual data of the normalized item. - -### `NormalizedAttachment` interface - -Represents the standardized structure of an attachment after normalization in the Airdrop platform. This interface defines the essential properties needed to identify and link attachments to their parent items. - -#### Properties - -- _url_ - - Required. A **string** representing the URL where the attachment can be accessed. - -- _id_ - - Required. A **string** that uniquely identifies the normalized attachment. - -- _file_name_ - - Required. A **string** representing the name of the attachment file. - -- _parent_id_ - - Required. A **string** identifying the parent item this attachment belongs to. - -- _author_id_ - - Optional. A **string** identifying the author or creator of the attachment. - -- _inline_ - - Optional. A **boolean** indicating whether the attachment is inline. - -- _content_type_ - - Optional. A **string** specifying the MIME type of the attachment (e.g. `'application/pdf'`, `'image/png'`). When provided, this takes precedence over the content type from the HTTP response header during streaming. Falls back to the HTTP `Content-Type` header, or `'application/octet-stream'` if neither is available. - -- _grand_parent_id_ - - Optional. A **number** or **string** identifying a higher-level parent entity, if applicable. - -#### Example - -```typescript -const normalizedAttachment: NormalizedAttachment = { - url: 'https://example.com/files/document.pdf', - id: 'att_123456', - file_name: 'document.pdf', - parent_id: 'task_789', - author_id: 'user_456', - inline: false, - content_type: 'application/pdf', - grand_parent_id: 1001, -}; -``` - -### `RepoInterface` interface - -Defines the structure of a repo which is used to store and upload extracted data. This interface provides the basic structure for repositories that handle data extraction and normalization. - -#### Properties - -- _itemType_ - - Required. A **string** that specifies the type of items stored in this repository. - -- _normalize_ - - Optional. A **function** that takes an object and returns either a **NormalizedItem** or **NormalizedAttachment**. This function is responsible for transforming raw data into a standardized format. - -- _overridenOptions_ - - Optional. An object of type **WorkerAdapterOptions** that overrides the default options for this specific repo. - -#### Example - -```typescript -const taskRepo: RepoInterface = { - itemType: 'tasks', - normalize: (rawTask) => ({ - id: rawTask.id, - created_date: rawTask.created_at, - modified_date: rawTask.updated_at, - data: rawTask, - }), -}; -``` - -### `ExternalSyncUnit` interface - -Represents an external sync unit (such as repositories, projects, etc.) that can be extracted. This interface defines the structure for organizing and identifying extractable units of data. - -#### Properties - -- _id_ - - Required. A **string** that uniquely identifies the external sync unit. - -- _name_ - - Required. A **string** representing the name of the external sync unit. - -- _description_ - - Required. A **string** providing a description of the external sync unit. - -- _item_count_ - - Optional. A **number** indicating the total count of items in this external sync unit. - -- _item_type_ - - Optional. A **string** specifying the type of items contained in this external sync unit. - -### `EventContext` interface - -Defines the structure of the event context that is sent to the external connector from Airdrop. - -#### Properties - -- _callback_url_ - - Required. A **string** representing the callback URL. - -- _dev_org_ - - Required. A **string** representing the organization ID. **Deprecated** - use `dev_oid` instead. - -- _dev_oid_ - - Required. A **string** representing the organization ID. - -- _dev_org_id_ - - Required. A **string** representing the organization ID. - -- _dev_user_ - - Required. A **string** representing the user ID. **Deprecated** - use `dev_uid` instead. - -- _dev_user_id_ - - Required. A **string** representing the user ID. **Deprecated** - use `dev_uid` instead. - -- _dev_uid_ - - Required. A **string** representing the user ID. - -- _event_type_adaas_ - - Required. A **string** representing the event type in ADaaS. - -- _external_sync_unit_ - - Required. A **string** representing the external sync unit ID. **Deprecated** - use `external_sync_unit_id` instead. - -- _external_sync_unit_id_ - - Required. A **string** representing the external sync unit ID. - -- _external_sync_unit_name_ - - Required. A **string** representing the external sync unit name. - -- _external_system_ - - Required. A **string** representing the external system. **Deprecated** - use `external_system_id` instead. - -- _external_system_id_ - - Required. A **string** representing the external system ID. - -- _external_system_name_ - - Required. A **string** representing the external system name. - -- _external_system_type_ - - Required. A **string** representing the external system type. - -- _extract_from_ - - Optional. A **string** representing the resolved start timestamp of extraction in ISO 8601 format. Automatically computed by the SDK from `extraction_start_time` and worker state. This is the field developers should read to know when to start extracting from. - -- _extract_to_ - - Optional. A **string** representing the resolved end timestamp of extraction in ISO 8601 format. Automatically computed by the SDK from `extraction_end_time` and worker state. This is the field developers should read to know when to stop extracting at. - -- _extraction_start_time_ - - Optional. An object of type **TimeValue** representing the start time value for extraction as sent by the platform. The SDK resolves this into a concrete ISO 8601 timestamp on `extract_from`. - -- _extraction_end_time_ - - Optional. An object of type **TimeValue** representing the end time value for extraction as sent by the platform. The SDK resolves this into a concrete ISO 8601 timestamp on `extract_to`. - -- _import_slug_ - - Required. A **string** representing the import slug. - -- _initial_sync_scope_ - - Optional. An enum **InitialSyncScope** representing the scope of the initial sync (can be 'full-history' or 'time-scoped'). - -- _mode_ - - Required. A **string** representing the mode (can be 'INITIAL', 'INCREMENTAL', or 'LOADING'). - -- _request_id_ - - Required. A **string** representing the request ID. - -- _request_id_adaas_ - - Required. A **string** representing the ADaaS request ID. - -- _reset_extraction_ - - Optional. A **boolean** signifying the incremental sync should start from the given `extract_from` timestamp if true or from `lastSuccessfulSyncStarted` timestamp if false. **Deprecated** - use `reset_extract_from` instead. - -- _reset_extract_from_ - - Optional. A **boolean** signifying the incremental sync should start from the given `extract_from` timestamp if true or from `lastSuccessfulSyncStarted` timestamp if false. **Deprecated** - use `extraction_start_time`/`extraction_end_time` instead, which are automatically resolved into `extract_from` and `extract_to`. - -- _run_id_ - - Required. A **string** representing the run ID. - -- _sequence_version_ - - Required. A **string** representing the sequence version. - -- _snap_in_slug_ - - Required. A **string** representing the snap-in slug. - -- _snap_in_version_id_ - - Required. A **string** representing the snap-in version ID. - -- _sync_run_ - - Required. A **string** representing the sync run ID. **Deprecated** - use `run_id` instead. - -- _sync_run_id_ - - Required. A **string** representing the sync run ID. **Deprecated** - use `run_id` instead. - -- _sync_tier_ - - Required. A **string** representing the sync tier. - -- _sync_unit_ - - Required. A **string** representing the sync unit ID. - -- _sync_unit_id_ - - Required. A **string** representing the sync unit ID. - -- _uuid_ - - Required. A **string** representing the unique identifier. **Deprecated** - use `request_id_adaas` instead. - -- _worker_data_url_ - - Required. A **string** representing the worker data URL. - -### `AirdropEvent` interface - -Defines the structure of events sent to external extractors from Airdrop platform. This interface encapsulates all necessary information for processing Airdrop events, including authentication, context, and payload data. - -#### Properties - -- _context_ - - Required. An object containing: - - _secrets_: An object containing: - - _service_account_token_: A **string** representing the DevRev authentication token for Airdrop platform - - _snap_in_version_id_: A **string** representing the version ID of the snap-in - - _snap_in_id_: A **string** representing the ID of the snap-in - -- _payload_ - - Required. An object of type **AirdropMessage** containing: - - _connection_data_: An object containing: - - _org_id_: A **string** representing the organization ID - - _org_name_: A **string** representing the organization name - - _key_: A **string** representing the key - - _key_type_: A **string** representing the key type - - _event_context_: An object of type [**EventContext**](#EventContext-interface) - - _event_type_: A value from the **EventType** enum (see `EventType` enum documentation below) - - _event_data_: Optional. An object that may contain: - - _external_sync_units_: Optional array of **ExternalSyncUnit** objects - - _progress_: Optional **number** indicating progress - - _error_: Optional error record - - _delay_: Optional **number** indicating delay - - _reports_: Optional array of loader reports - - _processed_files_: Optional array of **strings** representing processed files - - _stats_file_: Optional **string** representing stats file - -- _execution_metadata_ - - Required. An object containing: - - _devrev_endpoint_: A **string** representing the DevRev endpoint URL - -- _input_data_ - - Required. An object containing input data for snap-ins from '@devrev/typescript-sdk' - -### `EventData` interface - -Defines the structure of event data that is sent from the external extractor to Airdrop. This interface encapsulates various types of data that can be included in events, such as progress updates, errors, and processing results. - -#### Properties - -- _external_sync_units_ - - Optional. An array of **ExternalSyncUnit** objects representing external sync units to be processed. **Deprecated** - external sync units should be pushed to the `AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS` repo instead. - -- _progress_ - - Optional. A **number** indicating the progress of the current operation. **Deprecated** - progress is now calculated on the backend. - -- _error_ - - Optional. An object of type **ErrorRecord** containing error information if an error occurred. - -- _delay_ - - Optional. A **number** specifying a delay duration in seconds. - -- _artifacts_ - - Optional. An array of **Artifact** objects. **Deprecated** - should not be used directly. - -- _reports_ - - Optional. An array of **LoaderReport** objects representing loader reports. - -- _processed_files_ - - Optional. An array of **strings** representing processed file IDs. - -- _stats_file_ - - Optional. A **string** representing the stats file artifact ID. - -### `EventType` enum - -Defines the different types of events that can be sent to the external extractor from ADaaS. The external extractor uses these events to know what to do next in the extraction process. - -#### Extraction Events (preferred) - -- `StartExtractingExternalSyncUnits` = `'START_EXTRACTING_EXTERNAL_SYNC_UNITS'` -- `StartExtractingMetadata` = `'START_EXTRACTING_METADATA'` -- `StartExtractingData` = `'START_EXTRACTING_DATA'` -- `ContinueExtractingData` = `'CONTINUE_EXTRACTING_DATA'` -- `StartDeletingExtractorState` = `'START_DELETING_EXTRACTOR_STATE'` -- `StartExtractingAttachments` = `'START_EXTRACTING_ATTACHMENTS'` -- `ContinueExtractingAttachments` = `'CONTINUE_EXTRACTING_ATTACHMENTS'` -- `StartDeletingExtractorAttachmentsState` = `'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE'` - -#### Extraction Events (deprecated) - -- `ExtractionExternalSyncUnitsStart` **Deprecated** - use `StartExtractingExternalSyncUnits` -- `ExtractionMetadataStart` **Deprecated** - use `StartExtractingMetadata` -- `ExtractionDataStart` **Deprecated** - use `StartExtractingData` -- `ExtractionDataContinue` **Deprecated** - use `ContinueExtractingData` -- `ExtractionDataDelete` **Deprecated** - use `StartDeletingExtractorState` -- `ExtractionAttachmentsStart` **Deprecated** - use `StartExtractingAttachments` -- `ExtractionAttachmentsContinue` **Deprecated** - use `ContinueExtractingAttachments` -- `ExtractionAttachmentsDelete` **Deprecated** - use `StartDeletingExtractorAttachmentsState` - -#### Loading Events - -- `StartLoadingData` = `'START_LOADING_DATA'` -- `ContinueLoadingData` = `'CONTINUE_LOADING_DATA'` -- `StartLoadingAttachments` = `'START_LOADING_ATTACHMENTS'` -- `ContinueLoadingAttachments` = `'CONTINUE_LOADING_ATTACHMENTS'` -- `StartDeletingLoaderState` = `'START_DELETING_LOADER_STATE'` -- `StartDeletingLoaderAttachmentState` = `'START_DELETING_LOADER_ATTACHMENT_STATE'` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `ExtractorEventType` enum - -Defines the different types of events that can be sent from the external extractor to ADaaS. The external extractor uses these events to inform ADaaS about the progress of the extraction process. - -#### Extraction Events (preferred) - -- `ExternalSyncUnitExtractionDone` = `'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE'` -- `ExternalSyncUnitExtractionError` = `'EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR'` -- `MetadataExtractionDone` = `'METADATA_EXTRACTION_DONE'` -- `MetadataExtractionError` = `'METADATA_EXTRACTION_ERROR'` -- `DataExtractionProgress` = `'DATA_EXTRACTION_PROGRESS'` -- `DataExtractionDelayed` = `'DATA_EXTRACTION_DELAYED'` -- `DataExtractionDone` = `'DATA_EXTRACTION_DONE'` -- `DataExtractionError` = `'DATA_EXTRACTION_ERROR'` -- `ExtractorStateDeletionDone` = `'EXTRACTOR_STATE_DELETION_DONE'` -- `ExtractorStateDeletionError` = `'EXTRACTOR_STATE_DELETION_ERROR'` -- `AttachmentExtractionProgress` = `'ATTACHMENT_EXTRACTION_PROGRESS'` -- `AttachmentExtractionDelayed` = `'ATTACHMENT_EXTRACTION_DELAYED'` -- `AttachmentExtractionDone` = `'ATTACHMENT_EXTRACTION_DONE'` -- `AttachmentExtractionError` = `'ATTACHMENT_EXTRACTION_ERROR'` -- `ExtractorAttachmentsStateDeletionDone` = `'EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE'` -- `ExtractorAttachmentsStateDeletionError` = `'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR'` - -#### Extraction Events (deprecated) - -- `ExtractionExternalSyncUnitsDone` **Deprecated** - use `ExternalSyncUnitExtractionDone` -- `ExtractionExternalSyncUnitsError` **Deprecated** - use `ExternalSyncUnitExtractionError` -- `ExtractionMetadataDone` **Deprecated** - use `MetadataExtractionDone` -- `ExtractionMetadataError` **Deprecated** - use `MetadataExtractionError` -- `ExtractionDataProgress` **Deprecated** - use `DataExtractionProgress` -- `ExtractionDataDelay` **Deprecated** - use `DataExtractionDelayed` -- `ExtractionDataDone` **Deprecated** - use `DataExtractionDone` -- `ExtractionDataError` **Deprecated** - use `DataExtractionError` -- `ExtractionDataDeleteDone` **Deprecated** - use `ExtractorStateDeletionDone` -- `ExtractionDataDeleteError` **Deprecated** - use `ExtractorStateDeletionError` -- `ExtractionAttachmentsProgress` **Deprecated** - use `AttachmentExtractionProgress` -- `ExtractionAttachmentsDelay` **Deprecated** - use `AttachmentExtractionDelayed` -- `ExtractionAttachmentsDone` **Deprecated** - use `AttachmentExtractionDone` -- `ExtractionAttachmentsError` **Deprecated** - use `AttachmentExtractionError` -- `ExtractionAttachmentsDeleteDone` **Deprecated** - use `ExtractorAttachmentsStateDeletionDone` -- `ExtractionAttachmentsDeleteError` **Deprecated** - use `ExtractorAttachmentsStateDeletionError` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `LoaderEventType` enum - -Defines the different types of events that can be sent from the loader to ADaaS. - -#### Data Loading Events - -- `DataLoadingProgress` = `'DATA_LOADING_PROGRESS'` -- `DataLoadingDelayed` = `'DATA_LOADING_DELAYED'` -- `DataLoadingDone` = `'DATA_LOADING_DONE'` -- `DataLoadingError` = `'DATA_LOADING_ERROR'` -- `DataLoadingDelay` **Deprecated** - this was a typo, use `DataLoadingDelayed` instead - -#### Attachment Loading Events - -- `AttachmentLoadingProgress` = `'ATTACHMENT_LOADING_PROGRESS'` -- `AttachmentLoadingDelayed` = `'ATTACHMENT_LOADING_DELAYED'` -- `AttachmentLoadingDone` = `'ATTACHMENT_LOADING_DONE'` -- `AttachmentLoadingError` = `'ATTACHMENT_LOADING_ERROR'` - -#### Attachment Loading Events (deprecated aliases) - -- `AttachmentsLoadingProgress` **Deprecated** - use `AttachmentLoadingProgress` -- `AttachmentsLoadingDelayed` **Deprecated** - use `AttachmentLoadingDelayed` -- `AttachmentsLoadingDone` **Deprecated** - use `AttachmentLoadingDone` -- `AttachmentsLoadingError` **Deprecated** - use `AttachmentLoadingError` - -#### State Deletion Events - -- `LoaderStateDeletionDone` = `'LOADER_STATE_DELETION_DONE'` -- `LoaderStateDeletionError` = `'LOADER_STATE_DELETION_ERROR'` -- `LoaderAttachmentStateDeletionDone` = `'LOADER_ATTACHMENT_STATE_DELETION_DONE'` -- `LoaderAttachmentStateDeletionError` = `'LOADER_ATTACHMENT_STATE_DELETION_ERROR'` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `SyncMode` enum - -Defines the different modes of sync that can be used by the external extractor. It can be either INITIAL, INCREMENTAL or LOADING. - -#### Values - -- `INITIAL` = `'INITIAL'` - Used for the first/initial import -- `INCREMENTAL` = `'INCREMENTAL'` - Used for doing syncs -- `LOADING` = `'LOADING'` - Used for loading data from DevRev to the external system - -### `ExtractionMode` enum **Deprecated** - -Defines the different modes of extraction. Use `SyncMode` instead. - -#### Values - -- `INITIAL` = `'INITIAL'` -- `INCREMENTAL` = `'INCREMENTAL'` - -### `InitialSyncScope` enum - -Defines the different scopes of initial sync that can be used by the external extractor. - -#### Values - -- `FULL_HISTORY` = `'full-history'` -- `TIME_SCOPED` = `'time-scoped'` - -### `TimeUnit` enum - -Defines the supported Go duration units for time window calculations. These correspond directly to Go's `time.ParseDuration` units. - -#### Values - -- `NANOSECONDS` = `'ns'` -- `MICROSECONDS` = `'us'` -- `MICROSECONDS_MU` = `'µs'` -- `MILLISECONDS` = `'ms'` -- `SECONDS` = `'s'` -- `MINUTES` = `'m'` -- `HOURS` = `'h'` - -### `TimeValueType` enum - -Defines the type of a time value used in extraction start/end times. The platform sends these types to indicate how the extraction time should be resolved by the SDK. - -#### Values - -- `WORKERS_OLDEST` = `'workers_oldest'` - Oldest timestamp from worker state -- `WORKERS_OLDEST_MINUS_WINDOW` = `'workers_oldest_minus_window'` - Oldest timestamp from worker state minus a duration window -- `WORKERS_NEWEST` = `'workers_newest'` - Newest timestamp from worker state -- `WORKERS_NEWEST_PLUS_WINDOW` = `'workers_newest_plus_window'` - Newest timestamp from worker state plus a duration window -- `CURRENT_TIME` = `'current_time'` - Current time -- `ABSOLUTE_TIME` = `'absolute_time'` - User-specified absolute timestamp -- `UNBOUNDED` = `'unbounded'` - No bound, extract all available data - -### `TimeValue` interface - -Represents a time value used in extraction start/end times. - -#### Properties - -- _type_ - - Required. A **TimeValueType** enum value which denotes how the value should be resolved. - -- _value_ - - Optional. A **string** whose meaning depends on the type: - - For `ABSOLUTE_TIME`: an ISO 8601 timestamp - - For `*_WINDOW` types: a Go duration string (e.g. `'500ms'`, `'30s'`, `'5m'`, `'2h'`) - - For other types: not used - -### `ExtractionScope` type - -Represents the parsed extraction scope from the platform. Each key is an item type name, and the value indicates whether it should be extracted. - -#### Usage - -```typescript -type ExtractionScope = Record; -``` - -### `ExtractionCommonError` const enum - -Provides predefined error codes for common extraction errors. - -#### Values - -- `EXTERNAL_SYNC_UNIT_DELETED` = `'ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED'` -- `EXTERNAL_SYNC_UNIT_DEACTIVATED` = `'ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED'` -- `USER_DELETED` = `'ERROR_CODE=USER_DELETED'` - -### `AirSyncDefaultItemTypes` enum - -Defines the default item types used by the SDK. - -#### Values - -- `EXTERNAL_DOMAIN_METADATA` = `'external_domain_metadata'` -- `ATTACHMENTS` = `'attachments'` -- `EXTERNAL_SYNC_UNITS` = `'external_sync_units'` - -### `UNBOUNDED_DATE_TIME_VALUE` constant - -Sentinel value representing an unbounded (no limit) extraction timestamp. Used as the resolved value for `TimeValueType.UNBOUNDED`. Its value is `'1970-01-01T00:00:00.000Z'`. - -### `spawn` function - -This function initializes a new worker thread and oversees its lifecycle. It should be invoked when the snap-in receives a message from the Airdrop platform. The worker script provided then handles the event accordingly. - -#### Usage - -```typescript -spawn({ event, initialState, options, baseWorkerPath }); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _initialState_ - - Required. Object of **any** type that represents the initial state of the snap-in. - -- _workerPath_ - - Optional. A **string** that represents the path to the worker file. **Deprecated** - use `baseWorkerPath` instead. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions**, which will be passed to the newly created worker. This worker will then initialize a `WorkerAdapter` by invoking the `processTask` function. The options include: - - `isLocalDevelopment` - - A **boolean** flag. If set to `true`, intermediary files containing extracted data will be stored on the local machine, which is useful during development. The default value is `false`. - - - `timeout` - - A **number** that specifies the timeout duration for the lambda function, in milliseconds. The default is 10 minutes (10 \* 60 \* 1000 milliseconds). - - - `batchSize` - - A **number** that determines the maximum number of items to be processed and saved to an intermediary file before being sent to the Airdrop platform. The default batch size is 2,000. - - - `workerPathOverrides` - - Optional. A partial map of **EventType** to **string** paths, allowing you to override the default worker path for specific event types. - - - `skipConfirmation` - - Optional. A **boolean** flag. If set to `true`, skips artifact upload confirmation. - -- _initialDomainMapping_ - - Optional. An object of type **InitialDomainMapping** representing the initial domain mapping configuration. - -- _baseWorkerPath_ - - Optional. A **string** that represents the base path for the worker files, usually `__dirname`. When provided, the SDK automatically resolves the worker script based on the event type. - -#### Return value - -A **promise** that resolves once the worker has completed processing. - -#### Example - -```typescript -const run = async (events: AirdropEvent[]) => { - for (const event of events) { - await spawn({ - event, - initialState, - baseWorkerPath: __dirname, - }); - } -}; -``` - -### `processTask` function - -The `processTask` function retrieves the current state from the Airdrop platform and initializes a new `WorkerAdapter`. It executes the code specified in the `task` parameter, which contains the worker's functionality. If a timeout occurs, the function handles it by executing the `onTimeout` callback, ensuring the worker exits gracefully. Both functions receive an `adapter` parameter, representing the initialized `WorkerAdapter` object. - -#### Usage - -```typescript -processTask({ task, onTimeout }); -``` - -#### Parameters - -- _task_ - - Required. A **function** that defines the logic associated with the given event type. - -- _onTimeout_ - - Required. A **function** managing the timeout of the lambda invocation, including saving any necessary progress at the time of timeout. - -#### Example - -```typescript -// External sync units extraction -processTask({ - task: async ({ adapter }) => { - const httpClient = new HttpClient(adapter.event); - - const todoLists = await httpClient.getTodoLists(); - - const externalSyncUnits: ExternalSyncUnit[] = todoLists.map((todoList) => - normalizeTodoList(todoList) - ); - - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: externalSyncUnits, - }); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { - error: { - message: 'Failed to extract external sync units. Lambda timeout.', - }, - }); - }, -}); -``` - -### `Spawn` class - -`Spawn` class is responsible for spawning a new worker thread and managing the lifecycle of the worker. Provides utilities to emit control events to the platform and exit the worker gracefully. In case of lambda timeout, the class emits a lambda timeout event to the platform. - -#### Usage - -```typescript -new Spawn({ - event, - worker, - options, - resolve, - originalConsole, -}); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _worker_ - - Required. A Node worker of the **Worker** class, created with the createWorker function, which represents an independent JavaScript execution thread. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions**, which defines the options to create a new instance of Spawn class. - -- _resolve_ - - Required. A resolve **function** for the promise inside which the Spawn class was created. - -- _originalConsole_ - - Optional. A **Console** object representing the original console before the SDK logger replaces it. - -#### Example - -```typescript -new Promise((resolve) => { - new Spawn({ - event, - worker, - options, - resolve, - }); -}); -``` - -### `WorkerAdapter` class - -Used to interact with Airdrop platform. Provides utilities to emit events to the Airdrop platform, update the state of the snap-in and upload artifacts (files with data) to the platform. - -### Usage - -```typescript -new WorkerAdapter({ - event, - adapterState, - options, -}); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _adapterState_ - - Required. An object of type **State**, which represents the initial state of the adapter. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions** that specifies additional configuration options for the `WorkerAdapter`. This object is passed via the `spawn` function. - -#### Example - -```typescript -const adapter = new WorkerAdapter({ - event, - adapterState, - options, -}); -``` - -### `WorkerAdapter.state` property - -Getter and setter methods for working with the adapter state. - -### Usage - -```typescript -// get state -const adapterState = adapter.state; - -// set state -adapter.state = newAdapterState; -``` - -#### Example - -```typescript -export const initialState: ExtractorState = { - users: { completed: false }, - tasks: { completed: false }, - attachments: { completed: false }, -}; - -adapter.state = initialState; -``` - -### `WorkerAdapter.extractionScope` property - -Getter for the parsed extraction scope from the platform. Returns an `ExtractionScope` object. - -### Usage - -```typescript -const scope = adapter.extractionScope; -``` - -### `WorkerAdapter.shouldExtract` method - -Returns whether the given item type should be extracted. Defaults to `true` if the scope is empty or the item type is not listed. - -### Usage - -```typescript -adapter.shouldExtract(itemType); -``` - -#### Parameters - -- _itemType_ - - Required. A **string** representing the item type to check. - -#### Return value - -A **boolean** indicating whether the item type should be extracted. - -#### Example - -```typescript -if (adapter.shouldExtract('tasks')) { - // Extract tasks -} -``` - -### `WorkerAdapter.initializeRepos` method - -Initializes a `Repo` object for each item provided. - -### Usage - -```typescript -adapter.initializeRepos(repos); -``` - -#### Parameters - -- _repos_ - - Required. An array of objects of type `RepoInterface`. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function in the data extraction phase. - -```typescript -const repos = [ - { - itemType: 'tasks', - normalize: normalizeTask, - }, -]; - -adapter.initializeRepos(repos); -``` - -### `WorkerAdapter.getRepo` method - -Finds a Repo from the initialized repos. - -### Usage - -```typescript -adapter.getRepo(itemType); -``` - -#### Parameters - -- _itemType_ - - Required. A **string** that represents the itemType property for the searched repo. - -#### Return value - -An object of type **Repo** if the repo is found, otherwise **undefined**. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function. - -```typescript -// Push users to the repository designated for 'users' data. -await adapter.getRepo('users')?.push(users); -``` - -### `WorkerAdapter.emit` method - -Emits an event to the Airdrop platform. - -### Usage - -```typescript -adapter.emit( newEventType, data ): -``` - -#### Parameters - -- _newEventType_ - - Required. The event type to be emitted, of type **ExtractorEventType** or **LoaderEventType**. - -- _data_ - - Optional. An object of type **EventData** which represents the data to be sent with the event. - -#### Return value - -A **promise**, which resolves to undefined after the emit function completes its execution or rejects with an error. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function. - -```typescript -// Emitting successfully finished data extraction. -await adapter.emit(ExtractorEventType.DataExtractionDone); - -// Emitting a delay in attachments extraction phase. -await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { - delay: 10, -}); -``` - -### `WorkerAdapter.postState` method - -Saves the current adapter state to the Airdrop platform. - -### Usage - -```typescript -await adapter.postState(); -``` - -#### Return value - -A **promise** that resolves once the state has been posted. - -### `WorkerAdapter.mappers` property - -Provides access to the `Mappers` helper within the worker during loading. Use it to look up, create, or update sync mapper records that link external system items to DevRev items. - -#### Usage - -```typescript -// inside processTask({ task }) -await adapter.mappers.getByTargetId({ - sync_unit: adapter.event.payload.event_context.sync_unit, - target: devrevId, -}); -``` - -### `WorkerAdapter.reports` property - -Getter for the accumulated loader reports. Returns an array of **LoaderReport** objects. - -### `WorkerAdapter.processedFiles` property - -Getter for the list of processed file IDs. Returns an array of **strings**. - -### `WorkerAdapter.loadItemTypes` method - -Loads item types from DevRev to the external system during the loading phase. - -#### Usage - -```typescript -const response = await adapter.loadItemTypes({ itemTypesToLoad }); -``` - -#### Parameters - -- _itemTypesToLoad_ - - Required. An array of **ItemTypeToLoad** objects, each containing an `itemType` string, a `create` function, and an `update` function. - -#### Return value - -A **promise** resolving to a **LoadItemTypesResponse** containing `reports` and `processed_files`. - -### `WorkerAdapter.loadAttachments` method - -Loads attachments from DevRev to the external system during the loading phase. - -#### Usage - -```typescript -const response = await adapter.loadAttachments({ create }); -``` - -#### Parameters - -- _create_ - - Required. A function of type **ExternalSystemLoadingFunction\** that creates the attachment in the external system. - -#### Return value - -A **promise** resolving to a **LoadItemTypesResponse** containing `reports` and `processed_files`. - -### `WorkerAdapter.streamAttachments` method - -Streams attachments to the DevRev platform during the attachment extraction phase. Handles batching, deduplication, and progress tracking. - -#### Usage - -```typescript -await adapter.streamAttachments({ stream, processors, batchSize }); -``` - -#### Parameters - -- _stream_ - - Required. A function of type **ExternalSystemAttachmentStreamingFunction** that opens an HTTP stream for a given attachment. - -- _processors_ - - Optional. An object of type **ExternalSystemAttachmentProcessors** for custom attachment processing with `reducer` and `iterator` functions. - -- _batchSize_ - - Optional. A **number** specifying how many attachments to stream concurrently. Default is `1`, maximum is `50`. - -#### Return value - -A **promise** that resolves to a **StreamAttachmentsReturnType** (may contain `delay` or `error`), or `undefined` on success. - -### `WorkerAdapter.processAttachment` method - -Processes a single attachment: streams it from the external system, uploads it to DevRev, and records the SSOR attachment mapping. - -#### Usage - -```typescript -const result = await adapter.processAttachment(attachment, stream); -``` - -#### Parameters - -- _attachment_ - - Required. A **NormalizedAttachment** object representing the attachment to process. - -- _stream_ - - Required. A function of type **ExternalSystemAttachmentStreamingFunction** that returns the HTTP stream for the attachment. - -#### Return value - -A **promise** resolving to a **ProcessAttachmentReturnType** (may contain `error` or `delay`), or `undefined` on success. - ---- - -### `Mappers` class - -Manages sync mapper records that link external system items to DevRev items during loading. Access it via `adapter.mappers` inside your worker code. - -#### Methods - -- `getByTargetId(params)` - - **params**: `MappersGetByTargetIdParams` - - **returns**: `Promise>` - - Use when you know the DevRev ID and want the corresponding mapping. - -- `getByExternalId(params)` - - **params**: `MappersGetByExternalIdParams` - - **returns**: `Promise>` - - Use when you know an external ID and need the DevRev mapping. - -- `create(params)` - - **params**: `MappersCreateParams` - - **returns**: `Promise>` - - Call after creating an item in the external system to persist the mapping. - -- `update(params)` - - **params**: `MappersUpdateParams` - - **returns**: `Promise>` - - Call after updating an item in the external system to add IDs, targets, or version markers. - -### `SyncMapperRecordStatus` enum - -Status of a sync mapper record indicating its operational state. - -#### Values - -- `OPERATIONAL` = `'operational'` - The mapping is active and operational (default) -- `FILTERED` = `'filtered'` - The mapping was filtered out by user filter settings -- `IGNORED` = `'ignored'` - The external object should be ignored in sync operations - -### `SyncMapperRecordTargetType` enum - -Types of DevRev entities that can be targets in sync mapper records. - -#### Values - -- `ACCESS_CONTROL_ENTRY` = `'access_control_entry'` -- `ACCOUNT` = `'account'` -- `AIRDROP_AUTHORIZATION_POLICY` = `'airdrop_authorization_policy'` -- `AIRDROP_FIELD_AUTHORIZATION_POLICY` = `'airdrop_field_authorization_policy'` -- `AIRDROP_PLATFORM_GROUP` = `'airdrop_platform_group'` -- `ARTICLE` = `'article'` -- `ARTIFACT` = `'artifact'` -- `CHAT` = `'chat'` -- `CONVERSATION` = `'conversation'` -- `CUSTOM_OBJECT` = `'custom_object'` -- `DIRECTORY` = `'directory'` -- `GROUP` = `'group'` -- `INCIDENT` = `'incident'` -- `LINK` = `'link'` -- `MEETING` = `'meeting'` -- `OBJECT_MEMBER` = `'object_member'` -- `PART` = `'part'` -- `REV_ORG` = `'rev_org'` -- `ROLE` = `'role'` -- `ROLE_SET` = `'role_set'` -- `TAG` = `'tag'` -- `TIMELINE_COMMENT` = `'timeline_comment'` -- `USER` = `'user'` -- `WORK` = `'work'` - -### `installInitialDomainMapping` function - -Installs the initial domain mapping for a snap-in. This creates recipe blueprints and installs domain mappings via the DevRev API. - -#### Usage - -```typescript -await installInitialDomainMapping(event, initialDomainMappingJson); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent**. - -- _initialDomainMappingJson_ - - Required. An object of type **InitialDomainMapping** containing: - - `starting_recipe_blueprint`: Optional object with the recipe blueprint configuration - - `additional_mappings`: Optional object with additional mapping configuration - -### `MockServer` class - -A lightweight HTTP mock server for local development and testing of connectors. Allows you to define routes with static responses or custom handlers. - -#### Exported types - -- **RequestInfo** - Information about a request received by the mock server (method, url, body) -- **RetryConfig** - Configuration for retry simulation behavior (failureCount, errorStatus, errorBody, headers, delay) -- **RouteConfig** - Configuration object for setting up a route response (path, method, status, body, headers, retry, delay) - -### `formatAxiosError` function - -Formats an Axios error into a structured object for logging. - -#### Usage - -```typescript -const formatted = formatAxiosError(error); -``` - -#### Parameters - -- _error_ - - Required. An **AxiosError** object. - -#### Return value - -An **object** with structured error information. - -### `serializeAxiosError` function - -Serializes an Axios error into a structured response object. - -#### Usage - -```typescript -const serialized = serializeAxiosError(error); -``` - -#### Parameters - -- _error_ - - Required. An **AxiosError** object. - -#### Return value - -An **AxiosErrorResponse** object with structured error details. diff --git a/V2_PROGRESS.md b/V2_PROGRESS.md deleted file mode 100644 index 5094c948..00000000 --- a/V2_PROGRESS.md +++ /dev/null @@ -1,309 +0,0 @@ -# ADaaS SDK v2 Refactor — Progress & Plan (source of truth) - -> This file is the **single source of truth** for the v2 refactor. It is self-contained: -> any session (or subagent) can read ONLY this file + the named git oracles and have -> everything needed. Do NOT rely on chat history. Update the **Status** table after every commit. - -## TL;DR -Rebuild the v2 branch cleanly from `main` as a sequence of small, single-purpose, reviewable -commits. Mechanical/structural transforms first (Phase 1), polish + surface-defining work last -(Phase 2). `npm run build` stays green every commit; the test suite + api report are intentionally -**left broken** until the final steps. No npm publishing during the work. - -## Git facts -- **Working branch:** `v2` (already hard-reset to `origin/main`). -- **Base commit:** `origin/main` = `5b81ef2` (feat: Add new common error enums #204). -- **Oracle (target shape):** tag `v2-old-backup` = `9202e47`. This is the PREVIOUS - v2 attempt — it already implemented the rename, deletions, adapter split, state split+envelope, - and emit-from-return, but bundled into huge unreviewable commits built on a stale base. **Use it - as a structural reference / oracle only. Never copy wholesale. Re-author cleanly.** - Read oracle files with `git show v2-old-backup:src/...`. -- **IMPORTANT:** `origin/v2` is NOW our working branch (post-reset), NOT the oracle. The oracle is the - TAG `v2-old-backup`. Don't confuse them. -- **Safety:** old work preserved at tag `v2-old-backup`. Force-push of `v2` is approved by Rado. - -## Hard rules (apply to EVERY Phase-1 commit) -1. **`npm run build` must stay green.** Achieved by commit 1 adding a build tsconfig that excludes - `**/*.test.ts`. (ts-jest still transpiles tests independently, so tests still *run* — and will - fail on old names — that is expected and accepted.) -2. **Never touch `*.test.ts` files or any api-extractor report** (`*.api.md`, `*.api.json`, - `latest.json`, backwards-compatibility fixtures) until Phase 2. Reviewer rejects any commit that does. -3. **Do NOT rename DevRev backend API route strings.** Only SDK-owned identifiers/types/classes are - renamed Airdrop→AirSync. Route strings like `airdrop.sync-mapper-record.get-by-target` and any - `/internal/airdrop.*` endpoints stay verbatim (they are platform API, not SDK naming). -4. **Every deletion must be grep-justified** (zero live references in SDK `src/` non-test + the 3 - inspectable connectors). Record the justification in the commit body. -5. Each commit is **single-purpose**. If a change belongs to a later commit, defer it. -6. Keep `multithreading/` directory name. No logging/console changes. Both out of scope. - -## Commit sequence - -### Phase 1 — structural (review commit-by-commit) -- **C0 — Package rename** `@devrev/ts-adaas` → `@devrev/airsync-sdk` (scoped, stays under @devrev). - Touch: `package.json` `name`; README references; api-extractor config (entry point / package name); - rename the report file `*/ts-adaas.api.md` → `airsync-sdk.api.md` IF trivial, else defer report to Phase 2. - Do NOT publish. Version → `2.0.0-beta.0` placeholder. -- **C1 — Delete deprecated dir + add build tsconfig.** - - Delete `src/deprecated/**` (see list below) and its 4 `export * from './deprecated/...'` lines from `src/index.ts`. - - Add `tsconfig.build.json` (`include: ["src"]`, `exclude: ["**/*.test.ts","node_modules","dist"]`) - and point `build` script at it. This is the "build stays green" enabler. - - NOTE: `event-type-translation.ts` deletion MOVED to C3 — it is NOT dead; production code - (`process-task.ts`, `spawn.ts`, `worker-adapter.ts`, `control-protocol.ts`) imports it, so it - can only be removed alongside the old enum members it translates (C3). Deleting it here would - break the build. -- **C2 — Airdrop→AirSync identifier rename.** DECIDED with Rado: - - HARD rename, NO back-compat alias: `AirdropEvent`→`AirSyncEvent`, `AirdropMessage`→`AirSyncMessage`. - (v2-old-backup (oracle tag) did NOT do this rename — oracle unreliable here; do it properly.) - - Update stale branding in comments/prose: bare "Airdrop" + "ADaaS" → "AirSync". - - MUST NOT touch: `/internal/airdrop.*` API routes; the `AIRDROP_*` mapping enum members AND their - `'airdrop_*'` string values (mappers.interface.ts); the `external_system_type: 'ADaaS'` string LITERAL - (platform contract — keep quotes-protected). - - PRODUCTION scope only: exclude `**/*.test.ts` AND `src/tests/**` (test scaffolding renamed later in C10). - - Connectors all import AirdropEvent → breaking; captured for migrate-v2 skill. -- **C3 — Remove old event-type compatibility layer** (NOT a rename — main carries old+new side by side; drop old). - - Delete deprecated enum members, leaving only the new ones (tables below). Files: `src/types/extraction.ts`, - `src/types/loading.ts`. - - Delete `src/common/event-type-translation.ts` + `.test.ts`. - - Rewire its 4 production callers to use event types directly (no translation; backend now sends/accepts - only new types): `src/multithreading/process-task.ts` (translateIncomingEventType), - `src/multithreading/spawn/spawn.ts` (translateIncomingEventType), - `src/multithreading/worker-adapter/worker-adapter.ts` (translateOutgoingEventType), - `src/common/control-protocol.ts` (translateOutgoingEventType). - - Plus any `case`/reference cleanups in `spawn.helpers.ts`, adapters. - - Exported translation fns are NOT in index.ts (internal) — safe to delete. -- **C4a — State split (structural only).** Introduce `BaseState` + `ExtractionState` + `LoadingState`. - KEEP the flat `AdapterState = ConnectorState & SdkState` shape (behavior identical). - Author fresh; oracle `src/state/base-state.ts` etc. guide STRUCTURE only (the oracle is already - envelope-based = C4b; do NOT copy its envelope split here). - DESIGN (decided after reading current state.ts + oracle): - - `base-state.ts`: `abstract class BaseState` holds flat `_state: AdapterState`, - `_extractionScope`, shared fields, `state` get/set, `extractionScope` get, `init()`, `fetchState()`, - `postState()`, and `installInitialDomainMappingIfNeeded()` (extracted from current factory). Ctor takes - `initialSdkState`. - - `extraction-state.ts`: `ExtractionState extends BaseState` (ctor passes `extractionSdkState`) + - `resolveExtractionWindow()` + lastSyncStarted logic; factory `createExtractionState()`. - - `loading-state.ts`: `LoadingState extends BaseState` (ctor passes `loadingSdkState`); factory - `createLoadingState()`. - - `state.ts`: keep a thin `createAdapterState()` DISPATCHER picking Extraction/Loading by - `event_context.mode === SyncMode.LOADING` (process-task stays unchanged until C6). Re-export classes. - - `state.interfaces.ts`: UNCHANGED (flat shape kept; type reshaping is C4b). - - Consumers: `types/workers.ts` + `worker-adapter.ts` change `State` type ref → `BaseState`. -- **C4b — State envelope + migration.** Change on-disk shape to `{ connectorState, sdkState }`. - Add migration shim: read legacy flat v1 blob → split SDK-owned keys into `sdkState` → persist envelope. - (oracle `base-state.ts` has the reference impl incl. `V1_SDK_STATE_KEYS`.) - DESIGN DECISION: keep a SINGLE combined `SdkState` for C4b (do NOT narrow into Base/Extraction/Loading - SdkState — oracle narrows but that's coupled to the adapter split; defer to C5). `BaseState` - keeps ONE type param; gains `_connectorState` + `_sdkState: SdkState`. Keep deprecated lastSyncStarted/ - lastSuccessfulSyncStarted in SdkState (still live: stamped in extraction-state, promoted in worker-adapter, - read by time-value-resolver). Blast radius (SDK-field access moves off `.state` → `.sdkState`): - - state.interfaces.ts: add AdapterStateEnvelope + V1_SDK_STATE_KEYS; keep SdkState/initials/AdapterState. - - base-state.ts: `_connectorState`+`_sdkState`; `state` getter→connector, add `sdkState` getter; - init/postState use envelope; private normalizeFetchedState() (v2 as-is; v1 flat split by V1_SDK_STATE_KEYS; - malformed→throw). installIDM uses this.sdkState.snapInVersionId. - - extraction-state.ts: resolveExtractionWindow this.state.X→this.sdkState.X; pass this.sdkState to resolver. - - worker-adapter.ts: get state()→ConnectorState (KEY public breaking change); add get sdkState(); internal - this.state.→this.sdkState; this.adapterState.state.fromDevRev→this.adapterState.sdkState.fromDevRev. - - attachments-streaming-pool.ts: this.adapter.state.toDevRev→this.adapter.sdkState.toDevRev. - - time-value-resolver.ts: signature unchanged. - BREAKING: connectors reading SDK fields via adapter.state break; on-disk state auto-migrates v1→v2 on read. -- **C5 — Adapter split (structural only).** `BaseAdapter` + `ExtractionAdapter` + `LoadingAdapter`. - KEEP existing `emit(eventType, data)` contract (behavior identical). - CORRECTION: the oracle's adapter split IS C5-compatible — its `emit()` still has the - `emit(eventType, data)` signature (emit-from-return is genuinely deferred to C6 in the oracle too). - So follow the oracle's adapter design closely (it's a faithful reference), re-authored cleanly. - DESIGN (decided after reading current worker-adapter + oracle adapters): - - `adapters/base-adapter.ts`: `abstract class BaseAdapter` holds event/options/isTimeout/ - hasWorkerEmitted, `protected adapterState: BaseState`, `protected uploader`, - `state` get/set, `sdkState` get, `extractionScope` get, `postState()`, and the TEMPLATE-METHOD `emit()` - skeleton (hasWorkerEmitted guard → beforeEmit hook → state save (non-stateless) → emit() w/ buildEmitPayload - → afterEmit hook → WorkerMessageEmitted). Three abstract hooks: `beforeEmit(type)`, `buildEmitPayload(type)`, - `afterEmit(type)`. - - `adapters/extraction-adapter.ts`: `ExtractionAdapter extends BaseAdapter`. Owns _artifacts, repos, - currentEventDataLength, mappers? NO (mappers is loader). Owns: shouldExtract, initializeRepos, getRepo, - artifacts get/set, uploadAllRepos, processAttachment, destroyHttpStream, streamAttachments. - beforeEmit = ESU-repo-upload (the TODO block) + uploadAllRepos + the AttachmentExtractionDone boundary - update (lastSuccessfulSyncStarted/workers* on sdkState). buildEmitPayload = { artifacts }. afterEmit = - clear artifacts. ctor seeds mappers? no. (mappers stays loader-only per oracle.) - - `adapters/loading-adapter.ts`: `LoadingAdapter extends BaseAdapter`. Owns loaderReports, _processedFiles, - _mappers, reports/processedFiles/mappers getters, loadItemTypes, getLoaderBatches, loadAttachments, - loadItem, loadAttachment. beforeEmit = noop. buildEmitPayload = { reports, processed_files }. afterEmit = noop. - - NOTE the current emit() computes isExtraction/isLoader to decide payload extras; the template split makes - that implicit (each subclass's buildEmitPayload returns its own extras). Behavior-equivalent. - - `worker-adapter.ts`: REPLACE with a thin re-export module OR keep file but it just re-exports. The - `WorkerAdapter` public type becomes a UNION alias in types/workers.ts (see below). `createWorkerAdapter` - factory: keep but split or drop (process-task builds directly). Check consumers. - - `types/workers.ts`: `export type WorkerAdapter = ExtractionAdapter | LoadingAdapter`. - WorkerAdapterInterface stays (adapterState: BaseState). - - `process-task.ts`: dispatch by `event_context.mode === SyncMode.LOADING` → new LoadingAdapter, else - new ExtractionAdapter; pass concrete adapter to task/onTimeout. (process-task SPLIT into two entry points - is C6, NOT here — keep single processTask dispatching.) - - `attachments-streaming-pool.ts` + `.interfaces.ts`: retype `adapter: WorkerAdapter` → `ExtractionAdapter` - (pool is extraction-only; uses sdkState.toDevRev, processAttachment, isTimeout, emit). - - `types/extraction.ts`: imports WorkerAdapter for processor fn types — keep importing the union alias. - - DECIDED: union alias is throwaway, removed in C6 when processTask splits into typed entry points. - - Helpers `worker-adapter.helpers.ts` (getFilesToLoad, addReportToLoaderReport): move to loading-adapter's - dir or keep; used only by loading now. Keep path stable to minimize churn (import from new loading-adapter). -- **C6 — Emit-from-return contract.** NET-NEW DESIGN: emit-from-return was NEVER implemented in any - branch (oracle process-task.ts is still emit-based). Only spec is the TaskResult surface recovered from - `src/tests/backwards-compatibility/temp/airsync-sdk.api.md` (untracked leftover). DESIGN: - - types: add to types/workers.ts: - `export type TaskResult = {status:'success'} | {status:'progress'} | {status:'delay';delaySeconds:number} - | {status:'error';error:ErrorRecord};` and `export type TaskStatus = TaskResult['status'];` - Change ProcessTaskInterface: `task: (p)=>Promise`, `onTimeout?: (p)=>Promise` (optional). - - Status→event mapping (SDK-owned, per phase). Resumable phases (data/attachment extraction, data/attachment - loading): success→*_DONE, progress→*_PROGRESS, delay→*_DELAYED, error→*_ERROR. Non-resumable (ESU, metadata): - success→*_DONE, error→*_ERROR, progress/delay→*_ERROR (illegal, descriptive msg). Map keyed off - event.payload.event_type (which incoming EventType → which Extractor/Loader event per status). - - base-adapter: emit() becomes SDK-INTERNAL (drop from public surface / index). Add a driver-facing method - e.g. `async emitFromResult(result: TaskResult)` that computes the event type from event_type+status via the - mapping and calls the internal emit with the right payload (delaySeconds for delay, error for error). - - LOADER/STREAMING methods RETURN TaskResult (Rado decided): loadItemTypes, loadAttachments, streamAttachments - stop calling emit mid-flight; instead they RETURN a TaskResult (success/progress/delay/error). The reports/ - processed_files still flow via buildEmitPayload from adapter state (already wired in C5), so the methods just - return status. Their old in-method emit() calls (rate-limit→delay, timeout→progress, error→error) become - `return {status:...}`. They no longer call process.exit; the driver handles emit+exit. - - process-task.ts: split into `processExtractionTask` + `processLoadingTask` (drop single processTask + the - WorkerAdapter union alias). Shared `runWorkerTask(buildAdapter, {task,onTimeout})` driver (oracle has the - skeleton): run task→get TaskResult; if isTimeout && !hasWorkerEmitted run onTimeout (or SDK default if - onTimeout omitted: progress for resumable, error for ESU/metadata)→get TaskResult; then - adapter.emitFromResult(result); exit. Build the typed adapter via createExtractionState/createLoadingState. - - index.ts: export TaskResult, TaskStatus, processExtractionTask, processLoadingTask. REMOVE processTask. - - DECIDED (Rado): emit is SDK-INTERNAL — NOT on the public ExtractionAdapter/LoadingAdapter surface. - The spec confirms this (its adapter surfaces show no emit()). Implementation: the template-method skeleton - + beforeEmit/buildEmitPayload/afterEmit hooks become `protected` (already are for hooks); the actual emit - entry the DRIVER calls is `emitFromResult(result: TaskResult)` — make it a method the driver can call but - not documented as connector surface. Connectors calling adapter.emit() get a compile error (good). - Keep the old `emit(eventType,data)` as a protected/internal helper that emitFromResult delegates to (the - loader/streaming methods no longer call it — they return TaskResult). - - test connectors (src/tests/**): these CALL adapter.emit today → they'd break. They are test files (deferred - to C10), and build excludes them, so C6 build stays green. Migration of test workers to return-style = C10. - - Reference: oracle process-task.ts (the runWorkerTask skeleton + typed entry points) — but oracle is still - emit-based, so the TaskResult return + emitFromResult mapping is authored fresh per the spec above. - -### Phase 2 — closing / interactive (batched, done at the end) -- **C7 — JSDoc pass (DEFERRED to the very end).** Bar = `src/mappers/mappers.ts` style (class block: - what+when; method block: one-line what, "Used to/for…" usage, `@param` w/ type, `@returns`). Public - surface + non-obvious internals (state migration, emit-from-return mapping, attachment streaming pool). - FIRST attempt (d05434b, fanned-out subagents) was REVERTED (4fea755): too heavy (902 insertions), some - blocks just restated the obvious, made structural review harder. REDO as the LAST step of the whole - refactor — done once, by hand, against settled code, with an explicit **"skip-the-obvious" rule**: - NO docs on trivial getters/setters, NO restating the method name; document only class-level "what+when", - non-obvious params, and genuinely tricky internals. The high-value blocks (normalizeFetchedState - migration shim, getEventTypeForResult/EVENT_PHASE_MAP, zero-doc types/loading.ts) still get covered. -- **C8 — Regenerate api report** (`airsync-sdk.api.md`). -- **C9 — Exposure audit (INTERACTIVE with Rado).** Read the regenerated report; decide per-symbol what - to keep public vs hide. Empirical floor = anything imported by the 3 connectors (table below). -- **C10 — Fix tests + bw-compat baseline.** Update test files to new names/contract; decide re-baseline - vs remove the backwards-compatibility gate (v2 is an intentional break, so a v1-comparison gate is wrong). -- **C11 — Migration deliverable.** Scan full `main..v2` diff → derive v1→v2 change catalog → build a - **dedicated `migrate-v2` skill in `adaas-sdk`** (`.claude/skills/migrate-v2/`), later ported to the - `connectors-codegen` repo (owns the `connector-dev` plugin). Mechanical changes auto-applied; semantic - (emit-from-return, state access) flagged for review; ambiguous → `MIGRATION_TODO.md`. Validate against - the 3 inspectable connectors. Skill philosophy mirrors existing `update-sdk` (autonomous + defer-on-ambiguity). - -## Orchestration model -- Per Phase-1 commit, in the main session across multiple sittings: - 1. **Implementer subagent** — does the one commit's work; obeys all Hard rules; build stays green. - 2. **Reviewer subagent** (read-only) — verifies diff against that commit's contract + Hard rules - (esp. "no test/report files touched", "deletions grep-justified", "structure-only vs behavior-only"). - 3. Rado eyeballs → commit → next. -- Mini-workflows for parallel sub-steps: deletion grep-verification (C1), JSDoc-by-module (C7), - exposure-by-symbol (C9). - ---- - -## Reference data (baked in so future sessions don't re-derive) - -### `src/deprecated/**` files to delete (C1) -``` -src/deprecated/adapter/index.ts -src/deprecated/common/helpers.ts -src/deprecated/demo-extractor/external_domain_metadata.json -src/deprecated/demo-extractor/index.ts -src/deprecated/http/client.ts -src/deprecated/uploader/index.ts -``` -Also delete `src/common/event-type-translation.ts` (+ `.test.ts`). -`src/index.ts` on main exports these deprecated barrels — remove those export lines: -`./deprecated/adapter`, `./deprecated/demo-extractor`, `./deprecated/http/client`, `./deprecated/uploader`, -and the `formatAxiosError` export (v2-old-backup (oracle tag) dropped it — confirm against connector usage; azure-boards imports it, so this is a migration note). - -### C3 — EventType (incoming): DELETE these deprecated members, keep the new ones -| DELETE (old member = old VALUE) | KEEP (new member = new VALUE) | -|--------------------------------------------------------------|--------------------------------------------------------------------------| -| ExtractionExternalSyncUnitsStart = EXTRACTION_EXTERNAL_SYNC_UNITS_START | StartExtractingExternalSyncUnits = START_EXTRACTING_EXTERNAL_SYNC_UNITS | -| ExtractionMetadataStart = EXTRACTION_METADATA_START | StartExtractingMetadata = START_EXTRACTING_METADATA | -| ExtractionDataStart = EXTRACTION_DATA_START | StartExtractingData = START_EXTRACTING_DATA | -| ExtractionDataContinue = EXTRACTION_DATA_CONTINUE | ContinueExtractingData = CONTINUE_EXTRACTING_DATA | -| ExtractionDataDelete = EXTRACTION_DATA_DELETE | StartDeletingExtractorState = START_DELETING_EXTRACTOR_STATE | -| ExtractionAttachmentsStart = EXTRACTION_ATTACHMENTS_START | StartExtractingAttachments = START_EXTRACTING_ATTACHMENTS | -| ExtractionAttachmentsContinue = EXTRACTION_ATTACHMENTS_CONTINUE | ContinueExtractingAttachments = CONTINUE_EXTRACTING_ATTACHMENTS | -| ExtractionAttachmentsDelete = EXTRACTION_ATTACHMENTS_DELETE | StartDeletingExtractorAttachmentsState = START_DELETING_EXTRACTOR_ATTACHMENTS_STATE | -Loading members (StartLoadingData…StartDeletingLoaderAttachmentState) + UnknownEventType are unchanged. - -### C3 — ExtractorEventType (outgoing): DELETE deprecated, keep new -| DELETE (old) | KEEP (new) | -|---------------------------------------|-----------------------------------------| -| ExtractionExternalSyncUnitsDone | ExternalSyncUnitExtractionDone | -| ExtractionExternalSyncUnitsError | ExternalSyncUnitExtractionError | -| ExtractionMetadataDone | MetadataExtractionDone | -| ExtractionMetadataError | MetadataExtractionError | -| ExtractionDataProgress | DataExtractionProgress | -| ExtractionDataDelay | DataExtractionDelayed | -| ExtractionDataDone | DataExtractionDone | -| ExtractionDataError | DataExtractionError | -| ExtractionDataDeleteDone | ExtractorStateDeletionDone | -| ExtractionDataDeleteError | ExtractorStateDeletionError | -| ExtractionAttachmentsProgress | AttachmentExtractionProgress | -| ExtractionAttachmentsDelay | AttachmentExtractionDelayed | -| ExtractionAttachmentsDone | AttachmentExtractionDone | -| ExtractionAttachmentsError | AttachmentExtractionError | -| ExtractionAttachmentsDeleteDone | ExtractorAttachmentsStateDeletionDone | -| ExtractionAttachmentsDeleteError | ExtractorAttachmentsStateDeletionError | -(values for new members are the *_EXTRACTION_* / *_DELETION_* strings — see v2-old-backup (oracle tag) extraction.ts.) - -### C3 — LoaderEventType: DELETE deprecated typo/plural members -DELETE: `DataLoadingDelay` (typo), `AttachmentsLoadingProgress/Delayed/Done/Error` (the plural-typo dupes). -KEEP: `DataLoadingProgress, DataLoadingDelayed, DataLoadingDone, DataLoadingError, -AttachmentLoadingProgress/Delayed/Done/Error, LoaderStateDeletionDone/Error, -LoaderAttachmentStateDeletionDone/Error, UnknownEventType`. - -### Connector import surface (empirical floor for C9 exposure audit + C11 migration) -Symbols imported from `@devrev/ts-adaas` by the 3 inspectable connectors: -- **asana-internal:** AirSyncDefaultItemTypes, AirdropEvent, ErrorRecord, EventType, ExternalDomainMetadata, - ExternalSyncUnit, ExtractorEventType, LoaderEventType, NormalizedAttachment, NormalizedItem, RepoInterface, - SyncMode, WorkerAdapter, axios, processTask, spawn -- **azure-boards:** AirSyncDefaultItemTypes, AirdropEvent, AirdropMessage, EventType, ExternalSyncUnit, - ExternalSystemAttachment, ExternalSystemItem, ExternalSystemItemLoadingParams, ExtractorEventType, - LoaderEventType, NormalizedAttachment, NormalizedItem, SyncMode, WorkerAdapter, formatAxiosError, - installInitialDomainMapping, processTask, spawn -- **google-drive:** AirdropEvent, EventType, ExternalSystemAttachmentStreamingParams, ExtractorEventType, - NormalizedAttachment, NormalizedItem, SyncMode, WorkerAdapter, processTask, spawn, axios, axiosClient - -**Migration-relevant removals these connectors will hit:** `WorkerAdapter` (removed → use processExtraction/LoadingTask -+ return-based contract), `processTask` (split), `formatAxiosError` (dropped from index), `AirdropEvent`/`AirdropMessage` -(renamed AirSync*), all old `EXTRACTION_*` enum members (deleted). - -**v2 on-disk-state baseline assumption (for C11 migrate-v2 note):** v2 assumes persisted state was written by -SDK **>= v1.15.2**. The pre-1.15.2 `string[]` form of `lastProcessedAttachmentsIdsList` is NO LONGER migrated -(shim removed in CLEANUP/9b2b975). The v1→v2 *envelope* migration (`normalizeFetchedState`, C4b) is KEPT — that's -the real v1-vs-v2 boundary (every v1 version, incl. 1.19, writes flat state). Only the ancient intra-v1 string[] -processed-attachments format was dropped. - -## Status -| Commit | State | Notes | -|--------|-------|-------| -| C0 package rename | ☑ done | 8ddeb87. @devrev/ts-adaas→@devrev/airsync-sdk, v2.0.0-beta.0. Report filename rename deferred to C8. | -| C1 delete + tsconfig | ☑ done | d573cb6. Deleted src/deprecated/ (6 files) + 4 index exports; added tsconfig.build.json (excludes tests), build script points to it. Reviewer-approved. event-type-translation deletion moved to C3. | -| C2 AirSync rename | ☑ done | 1fa9afc. AirdropEvent→AirSyncEvent, AirdropMessage→AirSyncMessage (hard, no alias) + prose ADaaS/Airdrop→AirSync. Protected: airdrop.* routes, AIRDROP_* enum, 'ADaaS' literal. Reviewer-approved. | -| C3 enum cleanup | ☑ done | cc05f41. Deleted deprecated enum members (EventType, ExtractorEventType, LoaderEventType) + event-type-translation.ts/.test; rewired 4 callers (process-task, spawn, control-protocol, worker-adapter) + spawn.helpers cases. Behavior-equivalent. Reviewer-approved. | -| C4a state split | ☑ done | b63f3ab. BaseState + ExtractionState + LoadingState, flat shape preserved; state.ts is dispatcher by mode. Reviewer-confirmed behavior-equivalent (loading only loses inert logs). | -| C4b state envelope | ☑ done | 30ba1b3. { connectorState, sdkState } envelope + v1->v2 migration shim (normalizeFetchedState). adapter.state→connector-only, new adapter.sdkState; ~28 SDK-field access sites moved. SdkState kept combined (narrowing deferred to C5). Reviewer-approved (migration cases verified). | -| C5 adapter split | ☑ done | a7a877f. BaseAdapter (template emit + hooks) + ExtractionAdapter + LoadingAdapter; WorkerAdapter→union alias; processTask dispatches by mode (still single entry). worker-adapter.ts deleted; helpers→loading-adapter.helpers. Reviewer-approved (emit equivalence verified). SdkState kept combined (narrowing dropped from scope). | -| C6 emit-from-return | ☑ done | 0fb6116. task/onTimeout return TaskResult; SDK maps status→event via getEventTypeForResult and emits once (emitFromResult); emit now protected/internal; processTask→processExtractionTask+processLoadingTask; loader/stream methods return TaskResult. Reviewer-approved (mapping+state-save+no-double-emit verified). NET-NEW design (no oracle). | -| CLEANUP (post-review) | ☑ done | 9b2b975. (1) Removed dead pre-1.15.2 `migrateProcessedAttachments` shim (v2 assumes on-disk state >= v1.15.2); (2) moved emit primitive `common/control-protocol.ts`→`multithreading/emit.ts` (co-located w/ its only consumers base-adapter+spawn); (3) deleted dead `createAdapterState` dispatcher (zero callers post-C6), `state.ts` now a thin barrel. Build green, lint clean. Preceded by 4fea755 (revert of C7). | -| C7 JSDoc | ☐ todo | DEFERRED to very end. First attempt d05434b REVERTED (4fea755) — too heavy/redundant, hurt review. Redo by hand, once, against settled code, with "skip-the-obvious" rule (see C7 plan above). | -| C8 api report | ☐ todo | Phase 2 | -| C9 exposure audit | ☐ todo | Phase 2, interactive | -| C10 tests + baseline | ☐ todo | Phase 2 | -| C11 migrate-v2 skill | ☐ todo | Phase 2 | From a9126dea1a0ed815900f39123c93215f0067e4d8 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Fri, 19 Jun 2026 11:23:04 +0200 Subject: [PATCH 28/30] refactor(v2)!: remove deprecated ESU/state fields, types, enums & workerPath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the remaining deprecated public surface ahead of v2.0 GA, and align MIGRATION.md. - EventData: remove external_sync_units (ESUs ride out as repo artifacts) and the dead progress field; fix artifacts' incorrect @deprecated tag — it is load-bearing (buildEmitPayload emits it). - SdkState: remove lastSyncStarted / lastSuccessfulSyncStarted. The time-value resolver now degrades WORKERS_NEWEST(_PLUS_WINDOW) to UNBOUNDED when workersNewest is unset (was a lastSuccessfulSyncStarted fallback). Both legacy keys are retained in V1_SDK_STATE_KEYS so old flat blobs are still stripped on migration. - Remove orphaned deprecated types/enums: ExtractionMode, EventContextIn, EventContextOut, DomainObjectState, ErrorLevel, LogRecord, AdapterUpdateParams, AdapterState. - Remove duplicate EventType.UnknownEventType / ExtractorEventType .UnknownEventType (LoaderEventType.UnknownEventType retained). - Remove deprecated spawn() workerPath option (use baseWorkerPath). - Prune the public barrel accordingly. Cross-checked against 9 connector repos: none of the removed symbols are imported by any v2 connector. Build typechecks clean. The pre-existing broken test suite is repaired in a follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- MIGRATION.md | 785 ++++++++++++++++++ src/index.ts | 16 +- .../adapters/extraction-adapter.ts | 10 +- src/multithreading/spawn/spawn.ts | 6 +- src/state/extraction-state.ts | 10 - src/state/state.interfaces.ts | 29 +- src/state/time-value-resolver.ts | 25 +- src/types/common.ts | 29 - src/types/extraction.ts | 84 +- src/types/workers.ts | 4 - 10 files changed, 806 insertions(+), 192 deletions(-) create mode 100644 MIGRATION.md diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..a1183527 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,785 @@ +# Migrating connectors from v1 (`@devrev/ts-adaas`) to v2 (`@devrev/airsync-sdk`) + +This guide covers every connector-facing breaking change between +`@devrev/ts-adaas` v1.x and `@devrev/airsync-sdk` v2.0.0, with before/after +examples drawn from real connectors. + +**The wire protocol is unchanged.** Event payloads sent to the platform, API +routes, headers, the artifact/upload flow, and every surviving event-type +**string value** are byte-for-byte identical to v1. Only the connector-facing +**TypeScript API** changed. A correctly migrated connector talks to the +platform exactly like a v1 connector did. + +> **Why a hard break with no deprecation shims?** v2 is a single bundled-pain +> major release: the package was renamed, so there are no existing +> `@devrev/airsync-sdk` consumers to keep backward-compatible. Everything that +> needed to change changed at once, in one guide, rather than across a long +> deprecation cycle. + +--- + +## TL;DR — what breaks + +| # | Change | Who it affects | +|---|--------|----------------| +| 1 | Package renamed `@devrev/ts-adaas` → `@devrev/airsync-sdk` | every import | +| 2 | `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage` | type annotations (all connectors) | +| 3 | `processTask` → `processExtractionTask` / `processLoadingTask` | every worker file | +| 4 | `adapter.emit(...)` is gone — tasks **return** a `TaskResult` instead | every worker file | +| 5 | `WorkerAdapter` **class** removed → `ExtractionAdapter` / `LoadingAdapter` | helper signatures | +| 6 | `loadItemTypes` / `loadAttachments` / `streamAttachments` now **return** a `TaskResult` | loading + attachment workers | +| 7 | `EventData.external_sync_units` **removed** — external sync units must be pushed to a repo | ESU workers | +| 8 | `adapter.state` is connector-state only; SDK fields readable via `adapter.sdkState`. `lastSyncStarted` / `lastSuccessfulSyncStarted` **removed** | connectors reading/writing SDK bookkeeping | +| 9 | `axios`, `axiosClient`, `formatAxiosError`, `serializeAxiosError`, `HTTPResponse` no longer exported | anyone importing the SDK's HTTP/axios surface | +| 10 | `Mappers` methods return the unwrapped body (`Promise`), not `Promise>` | loaders calling `adapter.mappers.*` | +| 11 | Deprecated v1 modules deleted (`Adapter`, `createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, deprecated `Uploader`) | only legacy code | +| 12 | Deprecated event-type enum **members** deleted; deprecated types/enums (`ExtractionMode`, `EventContextIn`/`Out`, `DomainObjectState`, `ErrorLevel`, `LogRecord`, `AdapterUpdateParams`, `AdapterState`) **removed** | only if you used the old members/types | +| 13 | Several deep `dist/**` import paths moved/removed | deep-importers | +| 14 | Legacy `string[]` attachment-dedup migration dropped | in-flight attachment syncs started on SDK **< 1.15.2** | +| 15 | `spawn`'s deprecated `workerPath` option **removed** — use `baseWorkerPath: __dirname` | connectors still passing `workerPath` | +| 16 | `EventData.progress` field removed (no-op since v1; backend computes progress) | connectors that set `progress` in emit data | + +What does **not** change: `spawn(...)` and its surviving options (`initialState`, +`initialDomainMapping`, `workerPathOverrides`, `baseWorkerPath`, +`options.batchSize`, `timeout`, `isLocalDevelopment`, …) — the one exception is +the long-deprecated `workerPath`, which is removed (§15); default worker paths +(`/workers/data-extraction`, etc.); repos (`initializeRepos`, `getRepo`, +`push`); the normalization +interfaces (`NormalizedItem`, `NormalizedAttachment`, `RepoInterface`, +`ExternalSyncUnit`); `installInitialDomainMapping`; `createMockEvent` / +`MockServer`; HTTP retry behavior; `event_context.extract_from` / +`extract_to`; and every surviving event-type string value on the wire. + +--- + +## 1. Package rename + +```bash +npm uninstall @devrev/ts-adaas +npm install @devrev/airsync-sdk +``` + +Then global-replace the import specifier: + +```ts +// v1 +import { spawn, EventType } from '@devrev/ts-adaas'; +// v2 +import { spawn, EventType } from '@devrev/airsync-sdk'; +``` + +> **Deep imports** like `@devrev/ts-adaas/dist/...` are fragile — several of +> these paths moved or were removed in v2 (see §13). Prefer root imports; the +> v2 barrel now exports several symbols that previously required a deep import +> (`Mappers`, `Item`, `ItemTypeToLoad`). + +## 2. Type renames: `AirdropEvent` → `AirSyncEvent` + +Hard rename, no compatibility alias. The payload **shape** is identical — only +the type name changed. + +| v1 | v2 | +|----|----| +| `AirdropEvent` | `AirSyncEvent` | +| `AirdropMessage` | `AirSyncMessage` | + +```ts +// v1 +const run = async (events: AirdropEvent[]) => { ... }; +// v2 +const run = async (events: AirSyncEvent[]) => { ... }; +``` + +No other public type was renamed — `ConnectionData`, `EventContext`, +`EventData`, `ExtractorEvent`, and `ExternalSyncUnit` keep their v1 names. +Platform-owned strings (`/internal/airdrop.*` routes, the `'ADaaS'` external +system type, the `adaas_library_version` metadata key, `airdrop_*` mapping enum +values) are intentionally unchanged. + +### `AirSyncEvent.context` gained identity fields + +`AirSyncEvent.context` now declares the identity fields the platform already +sends, in addition to the existing `secrets`, `snap_in_id`, and +`snap_in_version_id`: + +```ts +// v2 — AirSyncEvent.context +context: { + secrets: { service_account_token: string }; + snap_in_version_id: string; + snap_in_id: string; + user_id: string; // new + dev_oid: string; // new + source_id: string; // new + service_account_id: string; // new +}; +``` + +If you previously extended the event to read these (e.g. a hand-rolled +`CustomAirdropEvent` that added `user_id`), you can drop the extension and read +`adapter.event.context.user_id` directly. (`snap_in_id` was already present in +v1; only the four fields above are new.) + +> Note: these four fields live on the **top-level** `AirSyncEvent.context`, not +> on the `EventContext` inside `payload.event_context`. The latter is a +> different object and is unchanged. + +## 3 + 4. The new worker contract: **return a `TaskResult`** instead of emitting + +This is the core change of v2. In v1 the connector decided *which event* to +emit and called `adapter.emit(...)`. In v2 the connector only reports *how the +phase ended* by **returning** a `TaskResult`; the SDK maps it to the correct +platform event for the current phase and emits it exactly once. + +```ts +// the exact union (exported from @devrev/airsync-sdk) +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } // note: delaySeconds, not delay + | { status: 'error'; error: ErrorRecord }; // ErrorRecord = { message: string } +``` + +`processTask` is split into two typed entry points — pick the one matching the +worker's phase: + +### Before (v1) + +```ts +import { processTask, ExtractorEventType } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + // ... extract ... + await adapter.emit(ExtractorEventType.DataExtractionDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(ExtractorEventType.DataExtractionProgress, { progress: 50 }); + }, +}); +``` + +### After (v2) + +```ts +import { processExtractionTask } from '@devrev/airsync-sdk'; + +processExtractionTask({ + task: async ({ adapter }) => { + // ... extract ... + return { status: 'success' }; + }, + // onTimeout can be omitted entirely for resumable phases: + // the SDK emits a progress (continuation) result by default. +}); +``` + +Loading workers use `processLoadingTask` the same way. + +> `adapter.emit(...)` is now **`protected`** — calling `adapter.emit(...)` or +> `this.adapter.emit(...)` is a hard **compile error** in v2, not a +> deprecation. Every emit site must be converted. + +### `emit()` → `return` translation table + +| v1 emit call | v2 return | +|--------------|-----------| +| `await adapter.emit(XxxDone)` | `return { status: 'success' }` | +| `await adapter.emit(XxxProgress, { progress })` | `return { status: 'progress' }` | +| `await adapter.emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` | +| `await adapter.emit(XxxError, { error })` | `return { status: 'error', error }` | +| `await adapter.emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to repo + `return { status: 'success' }` — see §7 | + +`progress` (`{ progress: n }`) carried no semantic value in v1 beyond "not done +yet" and is dropped; the platform tracks progress itself. + +### Status → emitted event, per phase + +The SDK picks the platform event from the **current phase** and the returned +status: + +| status | Resumable phases — data/attachment **extraction**, data/attachment **loading** | Non-resumable — external sync units, metadata, state deletion | +|--------|------------------------------------------------------------------------------|---------------------------------------------------------------| +| `'success'` | `*Done` | `*Done` | +| `'progress'` | `*Progress` (continuation) | **`*Error`** (illegal for these phases; emitted with a generated message) | +| `'delay'` | `*Delayed` (with `delaySeconds`) | **`*Error`** (illegal) | +| `'error'` | `*Error` (with the error record) | `*Error` | + +### Emits buried inside helper functions + +A common v1 pattern is emitting deep inside a helper and returning a boolean to +tell the caller to stop. Since only the **task's return value** reaches the SDK +in v2, the helper must **bubble the outcome up** instead. + +A clean way (used by the migrated google-drive connector) is to store the +terminal result on a field and return it once the loop unwinds: + +```ts +// v1 — helper emits, returns false to abort +private async handlePermissionDeniedError(error: unknown) { + await this.adapter.emit(ExtractorEventType.DataExtractionError, { + error: { message: '...' }, + }); +} +``` + +```ts +// v2 — helper records the result; the task returns it +private result: TaskResult | undefined; + +private handlePermissionDeniedError(error: unknown) { + this.result = { status: 'error', error: { message: '...' } }; +} + +// ... and where the loop ends: +private finalResult(): TaskResult { + // a stored error/delay set by a helper, else progress (continuation) + return this.result ?? { status: 'progress' }; +} +``` + +For a simple "stop iterating" signal, return the `TaskResult` directly up the +call chain: + +```ts +// v2 +async function extractList(adapter: ExtractionAdapter): Promise { + if (rateLimited) return { status: 'delay', delaySeconds: retryAfter }; + // ... + return null; // keep going +} + +processExtractionTask({ + task: async ({ adapter }) => { + for (const itemType of itemTypes) { + const stop = await extractList(adapter); + if (stop) return stop; + } + return { status: 'success' }; + }, +}); +``` + +### Timeout handling + +Checking `adapter.isTimeout` in your extraction loop still works as in v1 — but +instead of emitting progress and exiting, **return** progress: + +```ts +// v2 +if (adapter.isTimeout) { + return { status: 'progress' }; // platform sends CONTINUE_* next +} +``` + +Two important behaviors: + +- **The timeout outcome always wins.** Once the soft timeout has fired, the SDK + emits the `onTimeout` result (or the default `progress`) and **ignores + whatever the task returned**, by design: a phase that ran out of time must + hand off for continuation, not report itself complete. Put any cleanup that + must survive a timeout in `onTimeout`, not in the task body. +- **`onTimeout` is optional**, but its default is `{ status: 'progress' }`. + That is correct for resumable phases. For **non-resumable** phases (external + sync units, metadata, state deletion), `progress` is illegal and is emitted + as an **error** — so provide an explicit `onTimeout` there to control the + message: + + ```ts + onTimeout: async () => ({ + status: 'error', + error: { message: 'Failed to extract metadata. Lambda timeout.' }, + }), + ``` + +> Do **not** call `process.exit()` yourself in v2. v1 helpers sometimes called +> `process.exit(0/1)` after emitting; the v2 SDK owns the single worker exit +> after it emits your `TaskResult`. + +## 5. `WorkerAdapter` → `ExtractionAdapter` / `LoadingAdapter` + +The `WorkerAdapter` **class** is gone. Replace the type annotation on your +helpers with the mode-specific adapter: + +```ts +// v1 +async function extractList(adapter: WorkerAdapter) { ... } +// v2 +import { ExtractionAdapter } from '@devrev/airsync-sdk'; +async function extractList(adapter: ExtractionAdapter) { ... } +``` + +| Surface | Lives on | +|---------|----------| +| `initializeRepos`, `getRepo`, `streamAttachments`, `shouldExtract`, `artifacts` | `ExtractionAdapter` | +| `loadItemTypes`, `loadAttachments`, `mappers`, `reports`, `processedFiles` | `LoadingAdapter` | +| `event`, `state`, `sdkState`, `postState`, `isTimeout`, `extractionScope` | both (shared `BaseAdapter`) | + +> Only the **class** was removed. The **types** `WorkerAdapterInterface` and +> `WorkerAdapterOptions` still exist and are still exported — don't blindly +> rename every `WorkerAdapter` token; replace the `WorkerAdapter` +> annotations only. + +> **`mappers` / `reports` / `processedFiles` moved to `LoadingAdapter` only.** +> In v1 they were on the single `WorkerAdapter` and technically reachable in any +> phase. In v2 they are not on `ExtractionAdapter`. Code that touched +> `adapter.mappers` (etc.) during an extraction phase must move to the loading +> path. + +## 6. Loading & attachment methods return a `TaskResult` + +`loadItemTypes`, `loadAttachments`, and `streamAttachments` no longer emit or +exit mid-flight — they **return** a `TaskResult` you pass straight through. +Rate limits (→ `delay`), timeouts (→ `progress`), errors (→ `error`), and +completion (→ `success`) are all encoded in the result; `reports` / +`processed_files` (loading) and artifacts (extraction) are attached to the +emitted event automatically. + +### Loading — before (v1) + +```ts +import { LoaderEventType, processTask } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + await adapter.loadItemTypes({ itemTypesToLoad }); + await adapter.emit(LoaderEventType.DataLoadingDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(LoaderEventType.DataLoadingProgress); + }, +}); +``` + +### Loading — after (v2) + +```ts +import { processLoadingTask } from '@devrev/airsync-sdk'; + +processLoadingTask({ + task: async ({ adapter }) => { + return adapter.loadItemTypes({ itemTypesToLoad }); + }, +}); +``` + +### Attachment streaming — before (v1) + +```ts +const response = await adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +if (response?.delay) { + await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { delay: response.delay }); +} else if (response?.error) { + await adapter.emit(ExtractorEventType.AttachmentExtractionError, { error: response.error }); +} else { + await adapter.emit(ExtractorEventType.AttachmentExtractionDone); +} +``` + +### Attachment streaming — after (v2) + +```ts +return adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +``` + +Custom attachment processors (reducer/iterator) are still supported with the +same call signatures; only their `adapter` parameter type changes from +`WorkerAdapter` to `ExtractionAdapter` (§5). The `getAttachmentStream` +function you implement still returns `{ httpStream }` / `{ delay }` / +`{ error }` — but see §9 for the `httpStream` type change. + +## 7. External sync units go through a repo + +In v1 the SDK accepted `external_sync_units` in the emit data and uploaded them +internally. With emit gone, push them to the `EXTERNAL_SYNC_UNITS` repo +yourself. The `EventData.external_sync_units` field — deprecated in v1 — has +been **removed entirely** in v2: there is no inline ESU path at all, and any +code still referencing `external_sync_units` in emit data is now a compile +error. External sync units leave the worker only as repo artifacts. + +### Before (v1) + +```ts +await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { + external_sync_units: externalSyncUnits, +}); +``` + +### After (v2) + +```ts +import { AirSyncDefaultItemTypes, processExtractionTask } from '@devrev/airsync-sdk'; + +adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + // mirror the batching the v1 SDK used internally for ESUs + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, +]); +await adapter.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push(externalSyncUnits); + +return { status: 'success' }; +``` + +The repo is uploaded automatically before the `Done` event is emitted. + +> If your connector already pushed ESUs to a repo in v1 (some do), this section +> is a no-op for you — just convert the trailing emit to `return`. + +## 8. State split: `adapter.state` vs `adapter.sdkState` + +In v1, connector state and SDK bookkeeping lived in one flat object, persisted +as one blob, and `adapter.state` exposed both. In v2: + +- **`adapter.state` is connector state only** — exactly the shape of the + `initialState` you pass to `spawn`. The getter and setter both survive, so + in-place reads/writes work unchanged: + + ```ts + // works identically in v1 and v2 + adapter.state[itemType].cursor = nextCursor; + adapter.state[itemType].complete = true; + ``` + +- **SDK bookkeeping** (`workersOldest`/`workersNewest`, `pendingWorkers*`, + `toDevRev`, `fromDevRev`, `snapInVersionId`) moved to **`adapter.sdkState`** + (read-only getter). +- On disk, state is persisted as a `{ connectorState, sdkState }` envelope. The + SDK **migrates a v1 flat blob automatically on first read** (recognized SDK + keys split into `sdkState`, the rest into `connectorState`), so in-flight + syncs survive the upgrade. + +> **`lastSyncStarted` / `lastSuccessfulSyncStarted` are gone.** Both fields were +> `@deprecated` in v1 and are **removed from `SdkState` in v2** — they are no +> longer set, persisted, or readable via `adapter.sdkState`. They were +> wall-clock sync-start timestamps; the SDK now resolves the incremental window +> from the extraction-data boundaries instead (`workersOldest`/`workersNewest` +> and the resolved `extract_from`/`extract_to`). The two old key names are still +> recognized by the v1-blob migration, so they are cleanly dropped from any old +> state rather than leaking into your connector state. + +### If your v1 connector mixed SDK fields into its own `State` interface + +Some connectors declared SDK-owned fields (e.g. `toDevRev`, +`lastSuccessfulSyncStarted`) on their hand-written `State` type and seeded them +in `getInitialState()`. Remove those from your interface and initial state — +`toDevRev`/`fromDevRev`/`workers*` are SDK-managed now, and +`lastSyncStarted`/`lastSuccessfulSyncStarted` no longer exist at all: + +```ts +// v1 — State interface mixing SDK fields (remove these) +export interface State { + // ... your fields ... + lastSyncStarted?: string; // removed in v2 — no replacement field + lastSuccessfulSyncStarted?: string; // removed in v2 — read workersNewest / extract_to instead + toDevRev?: ToDevRev; // also drop the dist/state/state.interfaces import +} +``` + +### Reading the incremental-sync window + +In v1 you read `lastSuccessfulSyncStarted` to decide an incremental cursor. That +field is gone (above). Read the window the SDK resolves for you instead: + +```ts +// preferred — the resolved window for this invocation, on the event context +const { extract_from, extract_to } = adapter.event.payload.event_context; + +// the persisted cross-sync high-water mark (committed at end of each cycle) +const lastExtractedTo = adapter.sdkState.workersNewest; +``` + +`extract_from`/`extract_to` are the per-invocation window the SDK computes from +the platform's `extraction_start_time`/`extraction_end_time`; +`workersOldest`/`workersNewest` are the persisted boundaries the SDK commits at +the end of a completed cycle and the true replacement for the old +"last successful sync" resume point. + +> **`AdapterState` is removed.** The deprecated flat +> `ConnectorState & SdkState` alias no longer exists. If you annotated anything +> `AdapterState`, drop the import — use your own connector `State` type for +> `adapter.state`, and read SDK fields from `adapter.sdkState` (or the event +> context). The v2 on-disk shape is the `AdapterStateEnvelope` +> (`{ connectorState, sdkState }`), which the SDK manages for you. + +> **Edge regression (old in-flight incremental syncs):** the SDK used to fall +> back to `lastSuccessfulSyncStarted` when resolving a `WORKERS_NEWEST` window +> on state that predated `workersNewest` (SDK **< 1.17.1**). With the field +> removed, that fallback is gone: such a window now resolves to the unbounded +> epoch, so an incremental sync still mid-flight across the upgrade re-extracts +> from the beginning **once**. The next completed cycle commits `workersNewest` +> and the sync self-heals; the platform deduplicates downstream. New syncs and +> any sync whose first full cycle completed on ≥ 1.17.1 are unaffected. Drain +> in-flight incremental syncs before upgrading if the one-time re-extract +> matters for your deployment. + +> **Edge case:** if your v1 connector state had a top-level key literally named +> `connectorState` or `sdkState`, the auto-migration mis-reads it (the envelope +> is detected by those key names). Rename such a key **before** upgrading. + +## 9. HTTP / axios surface removed from the SDK + +> This is the area the previously-published guide got **wrong** — it claimed +> the axios surface was "still exported." It is not. This was verified by +> typechecking a real connector against the v2 build. + +The SDK no longer re-exports any axios surface from its public entry point: + +| v1 export | v2 | +|-----------|----| +| `axios` (raw instance) | **removed** — import `axios` in your connector | +| `axiosClient` (retry-wrapped instance) | **removed from the public API** (still exists internally, but not exported and not at the old deep path) | +| `formatAxiosError` | **removed** (deleted from source) | +| `serializeAxiosError` | **removed from the public API** — use `serializeError` | +| `HTTPResponse` | **removed** | + +`axios` and `axios-retry` are still runtime **dependencies** of the SDK (so +they're available transitively), but you must construct your own client: + +### Before (v1) + +```ts +import { axios, axiosClient } from '@devrev/ts-adaas'; + +const res = await axiosClient.get(url, { responseType: 'stream' }); +``` + +### After (v2) + +```ts +import axios from 'axios'; +import axiosRetry from 'axios-retry'; + +const axiosClient = axios.create(); +axiosRetry(axiosClient, { retries: 5, retryDelay: axiosRetry.exponentialDelay }); + +const res = await axiosClient.get(url, { responseType: 'stream' }); +``` + +> If you previously deep-imported `@devrev/ts-adaas/dist/http/axios-client`, +> that path **no longer exists** (the file was renamed to `http/client` and is +> internal). Bring your own axios instance as above. + +For error logging, replace `formatAxiosError` / `serializeAxiosError` with the +exported `serializeError`: + +```ts +import { serializeError } from '@devrev/airsync-sdk'; +console.error(serializeError(error)); +``` + +### `httpStream` type changed + +The connector-implemented attachment-stream function returns +`ExternalSystemAttachmentStreamingResponse`, whose `httpStream` field changed +from axios's `AxiosResponse` to the new public `HttpStreamResponse` +(`{ data: any; headers: Record }`). An axios stream response still +satisfies it structurally, but if you annotated the stream with `AxiosResponse` +imported *from the SDK*, switch the annotation to `HttpStreamResponse` (or +import `AxiosResponse` from `axios` directly). + +## 10. `Mappers` methods return the unwrapped body + +`Mappers.getByTargetId` / `getByExternalId` / `create` / `update` changed their +return type from `Promise>` to `Promise` — they now return +the response **body** directly. Drop the `.data` access: + +### Before (v1) + +```ts +const mapperResponse = await this.mappers.getByTargetId({ sync_unit, target }); +const resolvedId = mapperResponse.data.sync_mapper_record.external_ids[0]; +``` + +### After (v2) + +```ts +const mapperResponse = await this.mappers.getByTargetId({ sync_unit, target }); +const resolvedId = mapperResponse.sync_mapper_record.external_ids[0]; +``` + +This is a **silent** change for code that reads `.data` (it will fail to +type-check, or read `undefined` if loosely typed). The `Mappers` class is also +now exported from the package root, so you no longer need to deep-import it from +`dist/mappers/mappers`. + +## 11. Deleted legacy modules + +Everything under the v1 `deprecated/` tree is gone: + +| Removed | Replacement | +|---------|-------------| +| `Adapter`, `createAdapter` | `ExtractionAdapter` / `LoadingAdapter` + `processExtractionTask` / `processLoadingTask` | +| `DemoExtractor` | — (reference implementation only) | +| `HTTPClient`, `defaultResponse` | your own axios client (§9) | +| deprecated `Uploader` | repos (`initializeRepos` / `getRepo` / `push`) | + +(The SDK has an internal `Uploader` class, but it was never part of the public +API in either version — the *public* v1 `Uploader` was the deprecated one.) + +## 12. Deleted deprecated enum members, types & enums + +The old/new duplicate enum members were collapsed; only the modern names +remain. **The string values of the surviving members are byte-identical to +v1**, so nothing changes on the wire — only the TypeScript member names. + +> ⚠️ The **deleted** `EventType` / `ExtractorEventType` members carried +> *different, older* string values than their replacements (e.g. v1 +> `ExtractionDataStart = 'EXTRACTION_DATA_START'` vs the surviving +> `StartExtractingData = 'START_EXTRACTING_DATA'`). The modern members already +> existed in v1 with the modern values, so survivors are wire-compatible — but +> a deleted member and its replacement did **not** share a string value. + +**`EventType` (incoming):** + +| Deleted (v1 deprecated) | Use instead | +|--------------------------|-------------| +| `ExtractionExternalSyncUnitsStart` | `StartExtractingExternalSyncUnits` | +| `ExtractionMetadataStart` | `StartExtractingMetadata` | +| `ExtractionDataStart` | `StartExtractingData` | +| `ExtractionDataContinue` | `ContinueExtractingData` | +| `ExtractionDataDelete` | `StartDeletingExtractorState` | +| `ExtractionAttachmentsStart` | `StartExtractingAttachments` | +| `ExtractionAttachmentsContinue` | `ContinueExtractingAttachments` | +| `ExtractionAttachmentsDelete` | `StartDeletingExtractorAttachmentsState` | + +**`ExtractorEventType` (outgoing):** the `Extraction*`-prefixed members +(`ExtractionDataDone`, `ExtractionDataDelay`, `ExtractionAttachmentsProgress`, +…) are deleted; use the `*Extraction*` members (`DataExtractionDone`, +`DataExtractionDelayed`, `AttachmentExtractionProgress`, …). In practice you'll +rarely reference `ExtractorEventType` at all in v2 — see §4. + +**`LoaderEventType`:** the duplicate members `DataLoadingDelay` and +`AttachmentsLoading*` (plural) are deleted; use `DataLoadingDelayed` and +`AttachmentLoading*` (singular). These duplicates shared the same string value +as their survivors, so removing them is a pure source-name change. + +> **No more incoming-event-type translation.** v1 shipped an +> `event-type-translation` module that mapped legacy platform strings onto the +> modern enum members (and translated outgoing types). v2 removed it entirely +> and passes `event_context.event_type` through untouched. Any connector +> importing `translateIncomingEventType` / `translateOutgoingEventType` / +> `translateExtractorEventType` / `translateLoaderEventType` from the SDK will +> fail to compile — drop them; the platform sends modern strings. + +### Removed duplicate `UnknownEventType` members + +`UnknownEventType = 'UNKNOWN_EVENT_TYPE'` was declared on three enums in v1. +The copies on `EventType` and `ExtractorEventType` are **removed**; the +`LoaderEventType.UnknownEventType` member (the one the SDK actually uses as its +"unrecognized event" sentinel) **stays**. If you matched on +`EventType.UnknownEventType` or `ExtractorEventType.UnknownEventType`, switch to +`LoaderEventType.UnknownEventType` or compare against the raw +`'UNKNOWN_EVENT_TYPE'` string (the value is unchanged). + +### Removed deprecated types & enums + +These were `@deprecated` in v1 and are **deleted from the public API** in v2. +None are referenced by the modern worker contract; each row gives the +replacement (or "no replacement" where the concept is gone): + +| Removed | Replacement | +|---------|-------------| +| `ExtractionMode` (enum) | `SyncMode` (adds `LOADING` alongside `INITIAL`/`INCREMENTAL`) | +| `EventContextIn` (interface) | `EventContext` (the single, current event-context type) | +| `EventContextOut` (interface) | `EventContext` | +| `DomainObjectState` (interface) | — (no replacement; was an unused per-object state shape) | +| `ErrorLevel` (enum) | — (logger uses its own internal log level) | +| `LogRecord` (interface) | — (unused) | +| `AdapterUpdateParams` (interface) | — (unused) | +| `AdapterState` (type alias) | your connector `State` for `adapter.state`; `adapter.sdkState` for SDK fields (§8) | + +Also removed from `EventData`: the deprecated `external_sync_units` field (§7) +and the deprecated `progress` field (a no-op since v1 — the backend computes +progress). The `artifacts` field on `EventData` is **kept** — it is how the SDK +attaches uploaded repo artifacts (including external sync units) to the emitted +event. + +## 13. Deep-import paths that moved or broke + +The compiled `dist/` mirrors `src/` 1:1, so a deep import keeps working only if +the underlying source file still lives at the same relative path. Status of the +paths real connectors used: + +| Deep import | Status in v2 | +|-------------|--------------| +| `dist/http/axios-client` (`axiosClient`) | ❌ **broken** — file renamed to `http/client` and made internal. Bring your own axios (§9). | +| `dist/state/state.interfaces` (`ToDevRev`) | ⚠️ still resolves, but `ToDevRev` is SDK-internal now (§8) — **drop** the import, don't repoint it | +| `dist/repo/repo.interfaces` (`Item`) | ✅ resolves — but `Item` is now on the root barrel, prefer the root import | +| `dist/types/loading` (`ItemTypeToLoad`) | ✅ resolves — also now on the root barrel | +| `dist/mappers/mappers` (`Mappers`) | ✅ resolves — also now on the root barrel (and note the return-type change, §10) | +| `dist/types/extraction` (`InitialSyncScope`) | ✅ resolves — also on the root barrel | + +Also: `mappers/mappers.interface` was renamed to `mappers.interfaces` +(singular → plural), so a deep import of `dist/mappers/mappers.interface` +breaks even though every symbol inside is unchanged. + +**Recommended:** replace all deep `dist/**` imports with root imports. If a +symbol you need isn't on the root barrel, request it rather than deep-importing. + +## 14. Edge regression: very old in-flight attachment syncs + +In v1 the SDK migrated the legacy `string[]` form of the processed-attachments +dedup list (`lastProcessedAttachmentsIdsList`) to the current +`{ id, parent_id }[]` form on read. v2 removed that conversion. + +The `string[]` form only exists in state written by SDK **< 1.15.2**. If an +attachment-extraction phase started on a pre-1.15.2 SDK and is **still +mid-flight** when the connector upgrades to v2, the v2 dedup check +(`it.id === …`) won't match the bare-string entries, so attachments already +downloaded in that sync get re-uploaded once. New syncs — and any sync started +on ≥ 1.15.2 — are unaffected. The platform deduplicates downstream, so the only +cost is the wasted re-download/upload on that one continuation. + +If this matters for your deployment, drain in-flight attachment syncs before +upgrading. + +## 15. `spawn`'s deprecated `workerPath` option removed + +`SpawnFactoryInterface.workerPath` was `@deprecated` in v1 and is **removed** in +v2. Point `spawn` at your worker directory with `baseWorkerPath: __dirname` +instead — the SDK resolves the per-event worker file from there +(`workerPathOverrides` still works for custom paths). + +### Before (v1) + +```ts +spawn({ event, initialState, workerPath: __dirname + '/workers/data-extraction' }); +``` + +### After (v2) + +```ts +spawn({ event, initialState, baseWorkerPath: __dirname }); +``` + +--- + +## Migration checklist + +1. `npm uninstall @devrev/ts-adaas && npm install @devrev/airsync-sdk`; replace the import specifier everywhere. +2. Rename `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage`. Drop any `CustomAirdropEvent` cast that only added `user_id`/`dev_oid`/`source_id`/`service_account_id` (§2). +3. Split workers: extraction files use `processExtractionTask`, loading files use `processLoadingTask` (§3). +4. Convert **every** `adapter.emit(...)` into a returned `TaskResult`; bubble outcomes up from helpers (§4). +5. Replace `WorkerAdapter` annotations with `ExtractionAdapter` / `LoadingAdapter` (§5). Move any `mappers`/`reports`/`processedFiles` access into the loading path. +6. Pass the `TaskResult` straight through from `loadItemTypes` / `loadAttachments` / `streamAttachments` (§6). +7. ESU workers: push external sync units to the `EXTERNAL_SYNC_UNITS` repo; remove any `external_sync_units` (and `progress`) from emit data — those fields are gone from `EventData` (§7, §12). +8. Remove SDK-owned fields from your `State` interface and `getInitialState`; `lastSyncStarted`/`lastSuccessfulSyncStarted` are gone — read incremental cursors from `event_context.extract_from`/`extract_to` or `adapter.sdkState.workersNewest` (§8). +9. Replace `axios` / `axiosClient` / `formatAxiosError` / `serializeAxiosError` SDK imports with your own axios client + `serializeError` (§9). +10. Drop `.data` from `adapter.mappers.*` result reads (§10). +11. Remove usage of deleted legacy modules and event-type-translation helpers (§11, §12). +12. Replace deleted enum members with their modern names — values unchanged; drop any use of the removed deprecated types (`ExtractionMode`, `EventContextIn`/`Out`, `DomainObjectState`, `ErrorLevel`, `LogRecord`, `AdapterUpdateParams`, `AdapterState`) (§12). +13. Replace deep `dist/**` imports with root imports; drop the now-internal `ToDevRev` import (§13). +14. Decide per worker whether to keep an explicit `onTimeout` (recommended for ESU / metadata / state-deletion phases to control the error message; can be omitted for resumable phases) (§4). +15. Replace `spawn({ workerPath })` with `spawn({ baseWorkerPath: __dirname })` (§15). +16. Update jest mocks of the SDK module — they hardcode the v1 shape (`processTask`, `WorkerAdapter`, old enum members, `axiosClient`). + +## A note on `2.0.0-beta.0` + +An early beta (`2.0.0-beta.0`) still re-exported `axios` / `axiosClient`. The +GA release removed them (§9). If you migrated a connector against the beta and +imported either symbol from `@devrev/airsync-sdk`, it will fail to compile +against GA — apply §9. diff --git a/src/index.ts b/src/index.ts index aa4878ee..3ce9465c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,18 +42,12 @@ export { installInitialDomainMapping } from './state/install-initial-domain-mapp export { serializeError } from './logger/logger'; // ── Common types ── -export { ErrorLevel, SyncMode } from './types/common'; -export type { - AdapterUpdateParams, - ErrorRecord, - InitialDomainMapping, - LogRecord, -} from './types/common'; +export { SyncMode } from './types/common'; +export type { ErrorRecord, InitialDomainMapping } from './types/common'; // ── Extraction types ── export { EventType, - ExtractionMode, ExtractorEventType, InitialSyncScope, TimeUnit, @@ -63,10 +57,7 @@ export type { AirSyncEvent, AirSyncMessage, ConnectionData, - DomainObjectState, EventContext, - EventContextIn, - EventContextOut, EventData, ExternalProcessAttachmentFunction, ExternalSyncUnit, @@ -112,9 +103,6 @@ export type { MappersUpdateParams, } from './mappers/mappers.interfaces'; -// ── State types ── -export type { AdapterState } from './state/state.interfaces'; - // ── Uploader types ── export type { Artifact, diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts index 7ae5a062..5d01e443 100644 --- a/src/multithreading/adapters/extraction-adapter.ts +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -135,17 +135,11 @@ export class ExtractionAdapter< ); await this.uploadAllRepos(); - // If the extraction is done, we want to save the timestamp of the last successful sync + // When the full extraction cycle completes, commit the extraction window + // boundaries so subsequent incremental syncs can resume from them. if (newEventType === ExtractorEventType.AttachmentExtractionDone) { const sdkState = this.sdkState; - console.log( - `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${sdkState.lastSyncStarted}).` - ); - - sdkState.lastSuccessfulSyncStarted = sdkState.lastSyncStarted; - sdkState.lastSyncStarted = ''; - // Clear pending extraction boundaries now that the cycle is complete sdkState.pendingWorkersOldest = ''; sdkState.pendingWorkersNewest = ''; diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 064d82c5..9473df01 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -66,14 +66,12 @@ function getWorkerPath({ * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class * @param {AirSyncEvent} options.event - The event object received from the platform * @param {object} options.initialState - The initial state of the adapter - * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname` * @returns {Promise} - A new instance of Spawn class */ export async function spawn({ event, initialState, - workerPath, initialDomainMapping, options, baseWorkerPath, @@ -96,9 +94,7 @@ export async function spawn({ } let script = null; - if (workerPath != null) { - script = workerPath; - } else if ( + if ( baseWorkerPath != null && options?.workerPathOverrides != null && options.workerPathOverrides[event.payload.event_type as EventType] != null diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts index 24d51083..5548dc78 100644 --- a/src/state/extraction-state.ts +++ b/src/state/extraction-state.ts @@ -23,7 +23,6 @@ export class ExtractionState extends BaseState { /** * Resolves the extraction window onto the event context. * - * On StartExtractingData: stamp `lastSyncStarted` if not already set. * On StartExtractingMetadata: resolve fresh from the TimeValue objects in the * event context and cache them as pending boundaries (always overwrite). * On all other events: reuse the pending boundaries cached during @@ -32,15 +31,6 @@ export class ExtractionState extends BaseState { resolveExtractionWindow(): void { const sdkState = this.sdkState; - // Set lastSyncStarted if the event type is StartExtractingData - if ( - this.event.payload.event_type === EventType.StartExtractingData && - !sdkState.lastSyncStarted - ) { - sdkState.lastSyncStarted = new Date().toISOString(); - console.log(`Setting lastSyncStarted to ${sdkState.lastSyncStarted}.`); - } - const eventContext = this.event.payload.event_context; if (this.event.payload.event_type === EventType.StartExtractingMetadata) { diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index 9e105772..d6b77bf4 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -4,16 +4,6 @@ import { FileToLoad } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; export interface SdkState { - /** - * @deprecated Use extract_from and extract_to from the event context instead, - * which are automatically resolved by the SDK from extraction_start_time and extraction_end_time. - */ - lastSyncStarted?: string; - /** - * @deprecated Use extract_from and extract_to from the event context instead, - * which are automatically resolved by the SDK from extraction_start_time and extraction_end_time. - */ - lastSuccessfulSyncStarted?: string; /** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp). * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */ pendingWorkersOldest?: string; @@ -29,16 +19,6 @@ export interface SdkState { snapInVersionId?: string; } -/** - * AdapterState is the legacy (v1) flat adapter state: connector state merged - * with SDK bookkeeping in a single object. - * - * @deprecated v2 persists the `{ connectorState, sdkState }` envelope - * (see {@link AdapterStateEnvelope}). Connector state is now exposed via - * `adapter.state` and SDK state is kept internal. - */ -export type AdapterState = ConnectorState & SdkState; - /** * AdapterStateEnvelope is the v2 on-disk state shape: connector state and SDK * bookkeeping are stored as disjoint sub-objects so SDK internals stay @@ -77,8 +57,6 @@ export interface StateInterface { } export const extractionSdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', pendingWorkersOldest: '', pendingWorkersNewest: '', workersOldest: '', @@ -106,8 +84,15 @@ export const loadingSdkState = { * Used by the migration shim to split a flat v1 state blob into the * `{ connectorState, sdkState }` envelope: keys in this set go to `sdkState`, * everything else is connector state. + * + * `lastSyncStarted` / `lastSuccessfulSyncStarted` are retained here even though + * they are no longer fields on `SdkState`: a flat v1 blob may still carry them, + * and they must be recognized as SDK-owned so they are stripped out rather than + * leaking into connector state during migration. */ export const V1_SDK_STATE_KEYS: ReadonlySet = new Set([ ...Object.keys(extractionSdkState), ...Object.keys(loadingSdkState), + 'lastSyncStarted', + 'lastSuccessfulSyncStarted', ]); diff --git a/src/state/time-value-resolver.ts b/src/state/time-value-resolver.ts index 9a8f9cfe..a10721c3 100644 --- a/src/state/time-value-resolver.ts +++ b/src/state/time-value-resolver.ts @@ -83,16 +83,15 @@ export function applyDuration( * - ABSOLUTE: Returns the value directly (must be an ISO 8601 timestamp) * - NOW: Returns the current time as ISO 8601 * - UNBOUNDED: Returns UNBOUNDED_DATE_TIME_VALUE ('1970-01-01T00:00:00.000Z') - * - WORKERS_OLDEST: Returns workersOldest from state, or throws if not set - * - WORKERS_NEWEST: Returns workersNewest from state, or throws if not set - * - WORKERS_OLDEST_MINUS_WINDOW: Subtracts duration from workersOldest, or throws if not set - * - WORKERS_NEWEST_PLUS_WINDOW: Adds duration to workersNewest, or throws if not set + * - WORKERS_OLDEST: Returns workersOldest from state, or UNBOUNDED if not set + * - WORKERS_NEWEST: Returns workersNewest from state, or UNBOUNDED if not set + * - WORKERS_OLDEST_MINUS_WINDOW: Subtracts duration from workersOldest, or UNBOUNDED if not set + * - WORKERS_NEWEST_PLUS_WINDOW: Adds duration to workersNewest, or UNBOUNDED if not set * * @param timeValue - The TimeValue to resolve * @param state - The current SDK state containing workersOldest and workersNewest * @returns Resolved ISO 8601 timestamp string * @throws Error if required TimeValue.value is missing for ABSOLUTE or *_WINDOW types - * @throws Error if workersOldest/workersNewest is not set in state for WORKERS_* types */ export function resolveTimeValue( timeValue: TimeValue, @@ -135,13 +134,7 @@ export function resolveTimeValue( case TimeValueType.WORKERS_NEWEST: { if (!state.workersNewest) { // To support backwards-compatibility for the old state - if (state.lastSuccessfulSyncStarted) { - return state.lastSuccessfulSyncStarted; - } - - throw new Error( - 'Field workersNewest is not set in state. Cannot resolve TimeValue of type WORKERS_NEWEST without a prior extraction boundary.' - ); + return UNBOUNDED_DATE_TIME_VALUE; } return state.workersNewest; } @@ -167,13 +160,7 @@ export function resolveTimeValue( } if (!state.workersNewest) { // To support backwards-compatibility for the old state - if (state.lastSuccessfulSyncStarted) { - return state.lastSuccessfulSyncStarted; - } - - throw new Error( - 'Field workersNewest is not set in state. Cannot resolve TimeValue of type WORKERS_NEWEST_PLUS_WINDOW without a prior extraction boundary.' - ); + return UNBOUNDED_DATE_TIME_VALUE; } return applyDuration(state.workersNewest, timeValue.value, 'add'); } diff --git a/src/types/common.ts b/src/types/common.ts index f0428be9..557f8da7 100644 --- a/src/types/common.ts +++ b/src/types/common.ts @@ -1,15 +1,3 @@ -import { Artifact } from '../uploader/uploader.interfaces'; - -/** - * ErrorLevel is an enum that represents the level of an error. - * @deprecated - */ -export enum ErrorLevel { - Warning = 'WARNING', - Error = 'ERROR', - Info = 'INFO', -} - /** * ErrorRecord is an interface that defines the structure of an error record. */ @@ -17,23 +5,6 @@ export interface ErrorRecord { message: string; } -/** - * LogRecord is an interface that defines the structure of a log record. - * @deprecated - */ -export interface LogRecord { - level: ErrorLevel; - message: string; -} - -/** - * AdapterUpdateParams is an interface that defines the structure of the parameters that can be passed to the update adapter. - * @deprecated - */ -export interface AdapterUpdateParams { - artifact?: Artifact; -} - /** * InitialDomainMapping is an interface that defines the structure of the initial domain mapping. */ diff --git a/src/types/extraction.ts b/src/types/extraction.ts index fba35a1b..600437f9 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -30,9 +30,6 @@ export enum EventType { ContinueLoadingAttachments = 'CONTINUE_LOADING_ATTACHMENTS', StartDeletingLoaderState = 'START_DELETING_LOADER_STATE', StartDeletingLoaderAttachmentState = 'START_DELETING_LOADER_ATTACHMENT_STATE', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', } /** @@ -57,19 +54,6 @@ export enum ExtractorEventType { AttachmentExtractionError = 'ATTACHMENT_EXTRACTION_ERROR', ExtractorAttachmentsStateDeletionDone = 'EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE', ExtractorAttachmentsStateDeletionError = 'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', -} - -/** - * @deprecated - * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor. - * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs. - */ -export enum ExtractionMode { - INITIAL = 'INITIAL', - INCREMENTAL = 'INCREMENTAL', } /** @@ -146,44 +130,6 @@ export interface TimeValue { value?: string; } -/** - * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from AirSync. - * @deprecated - */ -export interface EventContextIn { - callback_url: string; - dev_org: string; - dev_org_id: string; - dev_user: string; - dev_user_id: string; - external_sync_unit: string; - external_sync_unit_id: string; - external_sync_unit_name: string; - external_system: string; - external_system_type: string; - import_slug: string; - mode: string; - request_id: string; - snap_in_slug: string; - sync_run: string; - sync_run_id: string; - sync_tier: string; - sync_unit: DonV2; - sync_unit_id: string; - uuid: string; - worker_data_url: string; -} - -/** - * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to AirSync. - * @deprecated - */ -export interface EventContextOut { - uuid: string; - sync_run: string; - sync_unit?: string; -} - /** * EventContext is an interface that defines the structure of the event context that is sent to the external connector from AirSync. */ @@ -291,20 +237,12 @@ export interface ConnectionData { * EventData is an interface that defines the structure of the event data that is sent from the external extractor to AirSync. */ export interface EventData { - /** - * @deprecated This field is deprecated and should not be used. External sync units should be pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo. - * - */ - external_sync_units?: ExternalSyncUnit[]; - /** - * @deprecated This field is deprecated and should not be used. Progress is - * now calculated on the backend. - */ - progress?: number; error?: ErrorRecord; delay?: number; /** - * @deprecated This field is deprecated and should not be used. + * Artifacts produced by the worker's repos, attached to the emitted event by + * the SDK. This includes external sync units, which are pushed to the + * AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo and uploaded as artifacts. */ artifacts?: Artifact[]; @@ -321,22 +259,6 @@ export interface WorkerMetadata { adaas_library_version: string; } -/** - * DomainObject is an interface that defines the structure of a domain object that can be extracted. - * It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count. - * @deprecated - */ -export interface DomainObjectState { - name: string; - nextChunkId: number; - pages?: { - pages: number[]; - }; - lastModified: string; - isDone: boolean; - count: number; -} - /** * AirSyncEvent is an interface that defines the structure of the event that is sent to the external extractor from AirSync. * It contains the context, payload, execution metadata, and input data as common snap-ins. diff --git a/src/types/workers.ts b/src/types/workers.ts index dc617d11..2d328c8d 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -70,7 +70,6 @@ export interface SpawnInterface { * @constructor * @param {AirSyncEvent} event - The event object received from the platform * @param {object=} initialState - The initial state of the adapter - * @param {string} workerPath - The path to the worker file * @param {string} initialDomainMapping - The initial domain mapping * @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class * @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname` @@ -78,9 +77,6 @@ export interface SpawnInterface { export interface SpawnFactoryInterface { event: AirSyncEvent; initialState: ConnectorState; - - /** @deprecated Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath */ - workerPath?: string; options?: WorkerAdapterOptions; initialDomainMapping?: InitialDomainMapping; baseWorkerPath?: string; From f14b68b3e2404633b0c7a72c89975a95bc239b28 Mon Sep 17 00:00:00 2001 From: radovanjorgic Date: Mon, 22 Jun 2026 07:27:51 +0200 Subject: [PATCH 29/30] Test fixes --- .../attachments-streaming-pool.test.ts | 48 ++++++---- src/common/constants.ts | 1 + src/logger/logger.test.ts | 4 +- src/multithreading/create-worker.test.ts | 10 +- src/multithreading/process-task.test.ts | 96 +++++-------------- .../spawn/spawn.helpers.test.ts | 47 ++++----- src/multithreading/spawn/spawn.helpers.ts | 10 +- src/multithreading/spawn/spawn.test.ts | 17 +++- src/repo/repo.test.ts | 12 +-- .../install-initial-domain-mapping.test.ts | 4 +- src/state/state.extract-window.test.ts | 14 +-- src/state/state.pending-boundaries.test.ts | 34 ++++--- src/state/state.post-state.test.ts | 23 +++-- src/state/state.test.ts | 83 +++++----------- src/state/state.time-value-resolution.test.ts | 29 +++--- src/state/time-value-resolver.test.ts | 36 +++---- .../dummy-connector/data-extraction.test.ts | 4 +- src/tests/dummy-connector/data-extraction.ts | 12 +-- .../external-sync-units-extraction.ts | 31 ++++-- src/tests/dummy-connector/extraction.ts | 9 +- .../metadata-extraction.test.ts | 4 +- .../dummy-connector/metadata-extraction.ts | 14 +-- src/tests/event-data-size-limit/extraction.ts | 9 +- .../event-data-size-limit/size-limit-1.ts | 19 ++-- src/tests/spawn-worker/extraction.ts | 9 +- src/tests/spawn-worker/some-cleanup-worker.ts | 17 ++-- .../spawn-worker/unknown-event-type.test.ts | 5 +- src/tests/spawn-worker/unknown-event-type.ts | 14 +-- src/tests/test-helpers.interfaces.ts | 6 +- src/tests/timeout-handling/extraction.ts | 9 +- src/tests/timeout-handling/no-timeout.test.ts | 4 +- src/tests/timeout-handling/no-timeout.ts | 14 +-- .../timeout-handling/timeout-blocked.test.ts | 4 +- src/tests/timeout-handling/timeout-blocked.ts | 14 +-- .../timeout-handling/timeout-graceful.test.ts | 4 +- .../timeout-handling/timeout-graceful.ts | 13 +-- .../timeout-unblocked.test.ts | 4 +- .../timeout-handling/timeout-unblocked.ts | 13 +-- src/types/loading.ts | 2 - 39 files changed, 343 insertions(+), 359 deletions(-) diff --git a/src/attachments-streaming/attachments-streaming-pool.test.ts b/src/attachments-streaming/attachments-streaming-pool.test.ts index 604ba1d4..3c97881b 100644 --- a/src/attachments-streaming/attachments-streaming-pool.test.ts +++ b/src/attachments-streaming/attachments-streaming-pool.test.ts @@ -1,9 +1,9 @@ -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, ProcessAttachmentReturnType, -} from '../types'; +} from '../types/extraction'; import { AttachmentsStreamingPool } from './attachments-streaming-pool'; interface TestState { @@ -13,7 +13,7 @@ interface TestState { /* eslint-disable @typescript-eslint/no-explicit-any */ describe(AttachmentsStreamingPool.name, () => { - let mockAdapter: jest.Mocked>; + let mockAdapter: jest.Mocked>; let mockStream: jest.MockedFunction; let mockAttachments: NormalizedAttachment[]; @@ -22,6 +22,8 @@ describe(AttachmentsStreamingPool.name, () => { mockAdapter = { state: { attachments: { completed: false }, + }, + sdkState: { toDevRev: { attachmentsMetadata: { lastProcessedAttachmentsIdsList: [], @@ -103,7 +105,7 @@ describe(AttachmentsStreamingPool.name, () => { describe(AttachmentsStreamingPool.prototype.streamAll.name, () => { it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined as any; mockAdapter.processAttachment.mockResolvedValue({}); @@ -121,7 +123,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); }); @@ -169,11 +171,15 @@ describe(AttachmentsStreamingPool.name, () => { expect(result).toEqual({ delay: 5000 }); }); - it('should resume attachment extraction if it encounters old ids', async () => { - // Test migration from old string[] format to new ProcessedAttachment[] format - // Using 'as any' because we're intentionally testing legacy data format - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = - ['attachment-1', 'attachment-2'] as any; + it('should resume attachment extraction, skipping already-processed ids', async () => { + // Resume case: the state already records attachment-1 and attachment-2 as + // processed (in the v2 ProcessedAttachment {id, parent_id} format), so a + // re-run must skip them and only process the remaining attachment-3. + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + [ + { id: 'attachment-1', parent_id: 'parent-1' }, + { id: 'attachment-2', parent_id: 'parent-2' }, + ]; const pool = new AttachmentsStreamingPool({ adapter: mockAdapter, @@ -183,12 +189,11 @@ describe(AttachmentsStreamingPool.name, () => { const result = await pool.streamAll(); + // attachment-1/2 are skipped (already processed); attachment-3 is appended. expect( - mockAdapter.state.toDevRev?.attachmentsMetadata + mockAdapter.sdkState.toDevRev?.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ - { id: 'attachment-1', parent_id: '' }, - { id: 'attachment-2', parent_id: '' }, { id: 'attachment-1', parent_id: 'parent-1' }, { id: 'attachment-2', parent_id: 'parent-2' }, { id: 'attachment-3', parent_id: 'parent-3' }, @@ -243,7 +248,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(warnSpy).toHaveBeenCalledTimes(3); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); }); @@ -251,7 +256,7 @@ describe(AttachmentsStreamingPool.name, () => { describe(AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => { it('should skip already processed attachments', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = [{ id: 'attachment-1', parent_id: 'parent-1' }]; mockAdapter.processAttachment.mockResolvedValue({}); @@ -278,7 +283,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { id: 'attachment-1', parent_id: 'parent-1' }, @@ -308,7 +313,7 @@ describe(AttachmentsStreamingPool.name, () => { error ); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { @@ -338,7 +343,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { id: 'attachment-1', parent_id: 'parent-1' }, @@ -661,7 +666,10 @@ describe(AttachmentsStreamingPool.name, () => { }); mockAdapter.processAttachment.mockImplementation( - async (attachment, stream) => { + async ( + attachment: NormalizedAttachment, + stream: ExternalSystemAttachmentStreamingFunction + ) => { // processAttachment should be called with the user's stream function const result = await stream({ item: attachment, diff --git a/src/common/constants.ts b/src/common/constants.ts index e697144c..8ac7bf31 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -67,6 +67,7 @@ export const EVENT_SIZE_THRESHOLD_BYTES = Math.floor( ); export const SSOR_ATTACHMENT: string = 'ssor_attachment'; +export const UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE'; export enum AirSyncDefaultItemTypes { EXTERNAL_DOMAIN_METADATA = 'external_domain_metadata', diff --git a/src/logger/logger.test.ts b/src/logger/logger.test.ts index be30fcf6..1e2781ce 100644 --- a/src/logger/logger.test.ts +++ b/src/logger/logger.test.ts @@ -3,7 +3,7 @@ import { inspect } from 'node:util'; import { LIBRARY_VERSION } from '../common/constants'; import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; -import { AirdropEvent, EventType } from '../types/extraction'; +import { AirSyncEvent, EventType } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; import { getPrintableState, @@ -30,7 +30,7 @@ jest.mock('node:worker_threads', () => { }); describe(Logger.name, () => { - let mockEvent: AirdropEvent; + let mockEvent: AirSyncEvent; let mockOptions: WorkerAdapterOptions; beforeEach(() => { diff --git a/src/multithreading/create-worker.test.ts b/src/multithreading/create-worker.test.ts index b1b069f6..6aa23bbd 100644 --- a/src/multithreading/create-worker.test.ts +++ b/src/multithreading/create-worker.test.ts @@ -10,7 +10,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); // Act @@ -38,7 +38,7 @@ describe(createWorker.name, () => { (isMainThread as any) = false; const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); // Act & Assert @@ -58,7 +58,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); if (isMainThread) { @@ -85,7 +85,7 @@ describe(createWorker.name, () => { }, }; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }); if (isMainThread) { @@ -106,7 +106,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionMetadataStart }, + payload: { event_type: EventType.StartExtractingMetadata }, }); if (isMainThread) { diff --git a/src/multithreading/process-task.test.ts b/src/multithreading/process-task.test.ts index 04dcce38..9c868ce3 100644 --- a/src/multithreading/process-task.test.ts +++ b/src/multithreading/process-task.test.ts @@ -3,9 +3,6 @@ import { WorkerEvent, WorkerMessageSubject } from '../types/workers'; // These tests cover logic that is NOT exercised by the end-to-end integration // tests under src/tests/timeout-handling/: -// - translation of legacy wire event types into the new enum (mutates event in place) -// - the hasWorkerEmitted guard that prevents onTimeout from firing after a -// successful emit (integration tests only exercise the positive case) // - the error branch that posts WorkerMessageFailed and exits(1) // - the WorkerMessage handler's guard that only flips isTimeout on // WorkerMessageExit (integration tests can't cleanly target non-Exit messages) @@ -34,10 +31,6 @@ jest.mock('node:worker_threads', () => ({ }, })); -jest.mock('../common/event-type-translation', () => ({ - translateIncomingEventType: jest.fn((t: string) => t), -})); - jest.mock('../logger/logger', () => ({ Logger: jest.fn().mockImplementation(() => ({ log: jest.fn(), @@ -51,21 +44,20 @@ jest.mock('../logger/logger', () => ({ runWithUserLogContext: jest.fn((fn: () => unknown) => fn()), })); -jest.mock('../state/state', () => ({ - createAdapterState: jest.fn(), +jest.mock('../state/extraction-state', () => ({ + createExtractionState: jest.fn(), })); -jest.mock('./worker-adapter/worker-adapter', () => ({ - WorkerAdapter: jest.fn().mockImplementation(() => ({ +jest.mock('./adapters/extraction-adapter', () => ({ + ExtractionAdapter: jest.fn().mockImplementation(() => ({ isTimeout: false, - hasWorkerEmitted: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), })), })); -import { processTask } from './process-task'; -import { translateIncomingEventType } from '../common/event-type-translation'; -import { createAdapterState } from '../state/state'; -import { WorkerAdapter } from './worker-adapter/worker-adapter'; +import { processExtractionTask } from './process-task'; +import { createExtractionState } from '../state/extraction-state'; +import { ExtractionAdapter } from './adapters/extraction-adapter'; import { createMockEvent } from '../testing/mock-event'; function setWorkerData(data: Record) { @@ -81,7 +73,7 @@ function makeEvent(eventType = EventType.StartExtractingData) { // Flush the microtask queue enough to let the async IIFE inside processTask run. const flush = async () => new Promise((r) => setTimeout(r, 0)); -describe(processTask.name, () => { +describe(processExtractionTask.name, () => { let processExitSpy: jest.SpyInstance; beforeEach(() => { @@ -92,68 +84,27 @@ describe(processTask.name, () => { .spyOn(process, 'exit') .mockImplementation((() => {}) as () => never); - (createAdapterState as jest.Mock).mockResolvedValue({}); + (createExtractionState as jest.Mock).mockResolvedValue({}); }); afterEach(() => { processExitSpy.mockRestore(); }); - it('should translate incoming event type before passing to task', async () => { - // Arrange - const event = makeEvent(EventType.StartExtractingData); - setWorkerData({ event, initialState: {}, options: {} }); - (translateIncomingEventType as jest.Mock).mockReturnValue( - EventType.StartExtractingMetadata - ); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); - - // Act - processTask({ task, onTimeout }); - await flush(); - - // Assert - expect(translateIncomingEventType).toHaveBeenCalledWith( - EventType.StartExtractingData - ); - // The event is mutated in place — downstream code (including task) sees the - // translated type, not the original wire type. - expect(event.payload.event_type).toBe(EventType.StartExtractingMetadata); - }); - - it('should NOT call onTimeout when the worker already emitted before timeout check', async () => { - // Arrange - const event = makeEvent(); - setWorkerData({ event, initialState: {}, options: {} }); - // Both flags true: a timeout arrived but the worker had already emitted — - // onTimeout must be skipped. This is the guard the integration suite cannot - // target cleanly because it requires a precise race between emit and timeout. - const mockAdapter = { isTimeout: true, hasWorkerEmitted: true }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); - - // Act - processTask({ task, onTimeout }); - await flush(); - - // Assert - expect(onTimeout).not.toHaveBeenCalled(); - expect(processExitSpy).toHaveBeenCalledWith(0); - }); - it('should NOT flip adapter.isTimeout when a non-Exit WorkerMessage arrives', async () => { // Arrange const event = makeEvent(); setWorkerData({ event, initialState: {}, options: {} }); - const mockAdapter = { isTimeout: false, hasWorkerEmitted: false }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const mockAdapter = { + isTimeout: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const task = jest.fn().mockResolvedValue({ status: 'success' }); + const onTimeout = jest.fn().mockResolvedValue({ status: 'progress' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task, onTimeout }); await flush(); // Grab the handler registered for WorkerMessage events and invoke it with @@ -175,14 +126,17 @@ describe(processTask.name, () => { // Arrange const event = makeEvent(); setWorkerData({ event, initialState: {}, options: {} }); - const mockAdapter = { isTimeout: false, hasWorkerEmitted: false }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const mockAdapter = { + isTimeout: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); const taskError = new Error('task boom'); const task = jest.fn().mockRejectedValue(taskError); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const onTimeout = jest.fn().mockResolvedValue({ status: 'progress' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task, onTimeout }); await flush(); // Assert diff --git a/src/multithreading/spawn/spawn.helpers.test.ts b/src/multithreading/spawn/spawn.helpers.test.ts index 6645ac80..668fdb10 100644 --- a/src/multithreading/spawn/spawn.helpers.test.ts +++ b/src/multithreading/spawn/spawn.helpers.test.ts @@ -1,3 +1,4 @@ +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; @@ -19,9 +20,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.MetadataExtractionError); }); - it('should return MetadataExtractionError for deprecated ExtractionMetadataStart', () => { + it('should return MetadataExtractionError for StartExtractingMetadata (renamed from ExtractionMetadataStart)', () => { // Arrange - const eventType = EventType.ExtractionMetadataStart; + const eventType = EventType.StartExtractingMetadata; // Act const result = getTimeoutErrorEventType(eventType); @@ -54,9 +55,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.DataExtractionError); }); - it('should return DataExtractionError for deprecated ExtractionDataStart', () => { + it('should return DataExtractionError for StartExtractingData (renamed from ExtractionDataStart)', () => { // Arrange - const eventType = EventType.ExtractionDataStart; + const eventType = EventType.StartExtractingData; // Act const result = getTimeoutErrorEventType(eventType); @@ -65,9 +66,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.DataExtractionError); }); - it('should return DataExtractionError for deprecated ExtractionDataContinue', () => { + it('should return DataExtractionError for ContinueExtractingData (renamed from ExtractionDataContinue)', () => { // Arrange - const eventType = EventType.ExtractionDataContinue; + const eventType = EventType.ContinueExtractingData; // Act const result = getTimeoutErrorEventType(eventType); @@ -91,9 +92,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExtractorStateDeletionError for deprecated ExtractionDataDelete', () => { + it('should return ExtractorStateDeletionError for StartDeletingExtractorState (renamed from ExtractionDataDelete)', () => { // Arrange - const eventType = EventType.ExtractionDataDelete; + const eventType = EventType.StartDeletingExtractorState; // Act const result = getTimeoutErrorEventType(eventType); @@ -132,9 +133,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsStart', () => { + it('should return AttachmentExtractionError for StartExtractingAttachments (renamed from ExtractionAttachmentsStart)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsStart; + const eventType = EventType.StartExtractingAttachments; // Act const result = getTimeoutErrorEventType(eventType); @@ -145,9 +146,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsContinue', () => { + it('should return AttachmentExtractionError for ContinueExtractingAttachments (renamed from ExtractionAttachmentsContinue)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsContinue; + const eventType = EventType.ContinueExtractingAttachments; // Act const result = getTimeoutErrorEventType(eventType); @@ -173,9 +174,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExtractorAttachmentsStateDeletionError for deprecated ExtractionAttachmentsDelete', () => { + it('should return ExtractorAttachmentsStateDeletionError for StartDeletingExtractorAttachmentsState (renamed from ExtractionAttachmentsDelete)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsDelete; + const eventType = EventType.StartDeletingExtractorAttachmentsState; // Act const result = getTimeoutErrorEventType(eventType); @@ -201,9 +202,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExternalSyncUnitExtractionError for deprecated ExtractionExternalSyncUnitsStart', () => { + it('should return ExternalSyncUnitExtractionError for StartExtractingExternalSyncUnits (renamed from ExtractionExternalSyncUnitsStart)', () => { // Arrange - const eventType = EventType.ExtractionExternalSyncUnitsStart; + const eventType = EventType.StartExtractingExternalSyncUnits; // Act const result = getTimeoutErrorEventType(eventType); @@ -292,9 +293,9 @@ describe(getTimeoutErrorEventType.name, () => { }); describe('unknown event types', () => { - it('[edge] should return UnknownEventType and log error for unrecognized event type', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE and log error for unrecognized event type', () => { // Arrange - const eventType = EventType.UnknownEventType; + const eventType = 'TOTALLY_UNKNOWN' as EventType; const consoleErrorSpy = jest .spyOn(console, 'error') .mockImplementation(() => {}); @@ -303,7 +304,7 @@ describe(getTimeoutErrorEventType.name, () => { const result = getTimeoutErrorEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Event type not recognized in getTimeoutErrorEventType function: ' + eventType @@ -375,7 +376,7 @@ describe(getNoScriptEventType.name, () => { }); describe('unknown event types', () => { - it('[edge] should return UnknownEventType and log error for unrecognized event type', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE and log error for unrecognized event type', () => { // Arrange const eventType = EventType.StartExtractingData; const consoleErrorSpy = jest @@ -386,7 +387,7 @@ describe(getNoScriptEventType.name, () => { const result = getNoScriptEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Event type not recognized in getNoScriptEventType function: ' + eventType @@ -396,7 +397,7 @@ describe(getNoScriptEventType.name, () => { consoleErrorSpy.mockRestore(); }); - it('[edge] should return UnknownEventType for StartLoadingData', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE for StartLoadingData', () => { // Arrange const eventType = EventType.StartLoadingData; const consoleErrorSpy = jest @@ -407,7 +408,7 @@ describe(getNoScriptEventType.name, () => { const result = getNoScriptEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalled(); // Cleanup diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index 1f52764c..e0db6c89 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -1,3 +1,4 @@ +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; import { TaskStatus } from '../../types/workers'; @@ -33,7 +34,10 @@ export function getEventTypeForResult( 'Event type not recognized in getEventTypeForResult function: ' + eventType ); - return { eventType: LoaderEventType.UnknownEventType, illegal: true }; + return { + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, + illegal: true, + }; } // Non-resumable phases only define done/error events. @@ -250,7 +254,7 @@ export function getTimeoutErrorEventType(eventType: EventType): { eventType ); return { - eventType: LoaderEventType.UnknownEventType, + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, }; } } @@ -284,7 +288,7 @@ export function getNoScriptEventType(eventType: EventType) { eventType ); return { - eventType: LoaderEventType.UnknownEventType, + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, }; } } diff --git a/src/multithreading/spawn/spawn.test.ts b/src/multithreading/spawn/spawn.test.ts index 320725be..543135cb 100644 --- a/src/multithreading/spawn/spawn.test.ts +++ b/src/multithreading/spawn/spawn.test.ts @@ -12,7 +12,7 @@ jest.mock('../create-worker', () => ({ createWorker: jest.fn(), })); -jest.mock('../../common/control-protocol', () => ({ +jest.mock('../emit', () => ({ emit: jest.fn().mockResolvedValue({}), })); @@ -45,7 +45,7 @@ jest.mock('../../common/helpers', () => ({ // --------------------------------------------------------------------------- import { spawn, Spawn } from './spawn'; import { createWorker } from '../create-worker'; -import { emit } from '../../common/control-protocol'; +import { emit } from '../emit'; import { getMemoryUsage } from '../../common/helpers'; // --------------------------------------------------------------------------- @@ -111,7 +111,7 @@ describe('spawn() factory', () => { it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => { // Arrange const event = createMockEvent('http://localhost:0', { - payload: { event_type: EventType.UnknownEventType }, + payload: { event_type: 'TOTALLY_UNKNOWN' as EventType }, }); // Act @@ -135,7 +135,16 @@ describe('spawn() factory', () => { // Act & Assert await expect( - spawn({ event, initialState: {}, workerPath: '/fake/path.js' }) + spawn({ + event, + initialState: {}, + baseWorkerPath: '', + options: { + workerPathOverrides: { + [event.payload.event_type]: '/fake/path.js', + }, + }, + }) ).rejects.toThrow('worker boom'); }); }); diff --git a/src/repo/repo.test.ts b/src/repo/repo.test.ts index 148bb58a..648f1bb3 100644 --- a/src/repo/repo.test.ts +++ b/src/repo/repo.test.ts @@ -2,7 +2,7 @@ import { AirSyncDefaultItemTypes, SSOR_ATTACHMENT } from '../common/constants'; import { createItems, normalizeItem } from '../tests/test-helpers'; import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; -import { EventType } from '../types'; +import { EventType } from '../types/extraction'; import { Repo } from './repo'; jest.mock('../tests/test-helpers', () => ({ @@ -18,7 +18,7 @@ describe(Repo.name, () => { normalize = jest.fn(); repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', normalize, @@ -47,7 +47,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', onUpload: jest.fn(), @@ -74,7 +74,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA, normalize, @@ -94,7 +94,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: SSOR_ATTACHMENT, normalize, @@ -160,7 +160,7 @@ describe(Repo.name, () => { beforeEach(() => { repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', normalize, diff --git a/src/state/install-initial-domain-mapping.test.ts b/src/state/install-initial-domain-mapping.test.ts index bbed61bb..4e953d54 100644 --- a/src/state/install-initial-domain-mapping.test.ts +++ b/src/state/install-initial-domain-mapping.test.ts @@ -2,7 +2,7 @@ import axios from 'axios'; import { axiosClient } from '../http/client'; import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; -import { InitialDomainMapping } from '../types'; +import { InitialDomainMapping } from '../types/common'; import { EventType } from '../types/extraction'; import { installInitialDomainMapping } from './install-initial-domain-mapping'; @@ -29,7 +29,7 @@ const mockIsAxiosError = axios.isAxiosError as unknown as jest.Mock; describe(installInitialDomainMapping.name, () => { // Create mock objects const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }); const mockInitialDomainMapping: InitialDomainMapping = { diff --git a/src/state/state.extract-window.test.ts b/src/state/state.extract-window.test.ts index c0cd0f68..51841262 100644 --- a/src/state/state.extract-window.test.ts +++ b/src/state/state.extract-window.test.ts @@ -1,7 +1,7 @@ import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; describe('State — extraction window validation', () => { let fetchStateSpy: jest.SpyInstance; @@ -11,7 +11,7 @@ describe('State — extraction window validation', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -43,7 +43,7 @@ describe('State — extraction window validation', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -77,7 +77,7 @@ describe('State — extraction window validation', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -110,7 +110,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -140,7 +140,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -173,7 +173,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.pending-boundaries.test.ts b/src/state/state.pending-boundaries.test.ts index 4576e73b..01b36685 100644 --- a/src/state/state.pending-boundaries.test.ts +++ b/src/state/state.pending-boundaries.test.ts @@ -1,7 +1,7 @@ import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; /* eslint-disable @typescript-eslint/no-require-imports */ @@ -16,8 +16,8 @@ describe('State — pending extraction boundaries', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( require('./install-initial-domain-mapping'), 'installInitialDomainMapping' @@ -61,15 +61,17 @@ describe('State — pending extraction boundaries', () => { postStateSpy.mockResolvedValue({ success: true }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Assert - expect(state.state.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z'); - expect(state.state.pendingWorkersNewest).toBe(FIXED_NOW); + expect(state.sdkState.pendingWorkersOldest).toBe( + '1970-01-01T00:00:00.000Z' + ); + expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW); expect(event.payload.event_context.extract_from).toBe( '1970-01-01T00:00:00.000Z' ); @@ -106,16 +108,18 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Assert: pending values are overwritten with fresh resolution, not stale values - expect(state.state.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z'); - expect(state.state.pendingWorkersNewest).toBe(FIXED_NOW); - expect(state.state.pendingWorkersNewest).not.toBe(staleNewest); + expect(state.sdkState.pendingWorkersOldest).toBe( + '1970-01-01T00:00:00.000Z' + ); + expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW); + expect(state.sdkState.pendingWorkersNewest).not.toBe(staleNewest); }); it('should reuse pending values from state on ContinueExtractingData instead of re-resolving', async () => { @@ -149,7 +153,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -159,8 +163,8 @@ describe('State — pending extraction boundaries', () => { expect(event.payload.event_context.extract_from).toBe(pendingOldest); expect(event.payload.event_context.extract_to).toBe(pendingNewest); // Pending values in state remain unchanged - expect(state.state.pendingWorkersOldest).toBe(pendingOldest); - expect(state.state.pendingWorkersNewest).toBe(pendingNewest); + expect(state.sdkState.pendingWorkersOldest).toBe(pendingOldest); + expect(state.sdkState.pendingWorkersNewest).toBe(pendingNewest); }); it('should not set extract_from/extract_to on ContinueExtractingData if no pending values exist', async () => { @@ -178,7 +182,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -209,7 +213,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.post-state.test.ts b/src/state/state.post-state.test.ts index dc2a75d0..b8cac279 100644 --- a/src/state/state.post-state.test.ts +++ b/src/state/state.post-state.test.ts @@ -1,11 +1,11 @@ import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; /* eslint-disable @typescript-eslint/no-require-imports */ -describe('State.postState', () => { +describe('ExtractionState.postState', () => { let postStateSpy: jest.SpyInstance; let fetchStateSpy: jest.SpyInstance; let processExitSpy: jest.SpyInstance; @@ -14,8 +14,8 @@ describe('State.postState', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -37,7 +37,7 @@ describe('State.postState', () => { postStateSpy.mockRestore(); - const adapterState = await createAdapterState({ + const adapterState = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -51,11 +51,14 @@ describe('State.postState', () => { expect(requests).toHaveLength(1); const body = requests[0].body as { state: string }; - // Body must contain the stringified state, preserving the original fields + // Body must contain the stringified v2 envelope, with the posted state + // preserved under connectorState. expect(typeof body.state).toBe('string'); - const parsed = JSON.parse(body.state) as Record; - expect(parsed.foo).toBe('bar'); - expect(parsed.snapInVersionId).toBe('1.0.0'); + const parsed = JSON.parse(body.state) as { + connectorState: Record; + }; + expect(parsed.connectorState.foo).toBe('bar'); + expect(parsed.connectorState.snapInVersionId).toBe('1.0.0'); }); it('should exit(1) when postState HTTP request fails', async () => { @@ -70,7 +73,7 @@ describe('State.postState', () => { postStateSpy.mockRestore(); - const adapterState = await createAdapterState({ + const adapterState = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.test.ts b/src/state/state.test.ts index 2c0b33e9..e9223da2 100644 --- a/src/state/state.test.ts +++ b/src/state/state.test.ts @@ -5,12 +5,12 @@ import { import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; import { EventType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { ExtractionState, createExtractionState } from './extraction-state'; import { extractionSdkState } from './state.interfaces'; /* eslint-disable @typescript-eslint/no-require-imports */ -describe(State.name, () => { +describe(ExtractionState.name, () => { let initSpy: jest.SpyInstance; let postStateSpy: jest.SpyInstance; let fetchStateSpy: jest.SpyInstance; @@ -21,9 +21,9 @@ describe(State.name, () => { jest.clearAllMocks(); jest.restoreAllMocks(); - initSpy = jest.spyOn(State.prototype, 'init'); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + initSpy = jest.spyOn(ExtractionState.prototype, 'init'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( require('./install-initial-domain-mapping'), 'installInitialDomainMapping' @@ -42,7 +42,7 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -70,7 +70,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -91,7 +91,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -112,7 +112,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -151,59 +151,20 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + const result = await createExtractionState({ event, initialState, initialDomainMapping: {}, }); - const expectedState = { - ...initialState, - ...extractionSdkState, - }; - expect(postStateSpy).toHaveBeenCalledWith(expectedState); + // Assert: on 404 the SDK persists the full adapter state — the initial + // connector state and the seeded SDK state — via postState. + expect(postStateSpy).toHaveBeenCalled(); + expect(result.state).toEqual(initialState); + expect(result.sdkState).toEqual(extractionSdkState); } ); - it(EventType.StartExtractingData, async () => { - // Arrange - const initialState = { - test: 'test', - }; - const event = createMockEvent(mockServer.baseUrl, { - context: { - snap_in_version_id: '', - }, - payload: { event_type: EventType.StartExtractingData }, - }); - fetchStateSpy.mockRejectedValue({ - isAxiosError: true, - response: { status: 404 }, - }); - installInitialDomainMappingSpy.mockResolvedValue({ - success: true, - }); - postStateSpy.mockResolvedValue({ - success: true, - }); - - // Act - await createAdapterState({ - event, - initialState, - initialDomainMapping: {}, - }); - - // Assert - // Verify that post state is called with object that contains - // lastSyncStarted which is not empty string - expect(postStateSpy).toHaveBeenCalledWith( - expect.objectContaining({ - lastSyncStarted: expect.not.stringMatching(/^$/), - }) - ); - }); - it.each(STATEFUL_EVENT_TYPES)( 'should exit the process if initialDomainMapping is not provided for event type %s', async (eventType) => { @@ -220,7 +181,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: undefined, @@ -248,7 +209,7 @@ describe(State.name, () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act & Assert - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -280,7 +241,7 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -308,7 +269,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -337,7 +298,7 @@ describe(State.name, () => { postStateSpy.mockResolvedValue({ success: true }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -354,7 +315,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -376,7 +337,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.time-value-resolution.test.ts b/src/state/state.time-value-resolution.test.ts index 9c1cb814..9c4e8285 100644 --- a/src/state/state.time-value-resolution.test.ts +++ b/src/state/state.time-value-resolution.test.ts @@ -1,7 +1,8 @@ import { mockServer } from '../tests/jest.setup'; import { createMockEvent } from '../testing/mock-event'; import { EventType, TimeValue, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; +import { BaseState } from './base-state'; +import { createExtractionState } from './extraction-state'; describe('State — TimeValue resolution', () => { let fetchStateSpy: jest.SpyInstance; @@ -11,7 +12,7 @@ describe('State — TimeValue resolution', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + fetchStateSpy = jest.spyOn(BaseState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -19,13 +20,14 @@ describe('State — TimeValue resolution', () => { describe('Enhanced Control Protocol - TimeValue resolution failures', () => { it('should exit the process if extraction_start_time resolution fails', async () => { - // Arrange: WORKERS_NEWEST type but state has no workersNewest + // Arrange: ABSOLUTE_TIME with an invalid ISO timestamp (still rejected in v2) const event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata, event_context: { extraction_start_time: { - type: TimeValueType.WORKERS_NEWEST, + type: TimeValueType.ABSOLUTE_TIME, + value: 'not-a-date', }, }, }, @@ -40,7 +42,7 @@ describe('State — TimeValue resolution', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -50,7 +52,7 @@ describe('State — TimeValue resolution', () => { }); it('should exit the process if extraction_end_time resolution fails', async () => { - // Arrange: WORKERS_NEWEST type but state has no workersNewest + // Arrange: ABSOLUTE_TIME end_time with an invalid ISO timestamp (still rejected in v2) const event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata, @@ -59,7 +61,8 @@ describe('State — TimeValue resolution', () => { type: TimeValueType.UNBOUNDED, }, extraction_end_time: { - type: TimeValueType.WORKERS_NEWEST, + type: TimeValueType.ABSOLUTE_TIME, + value: 'not-a-date', }, }, }, @@ -74,7 +77,7 @@ describe('State — TimeValue resolution', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -109,7 +112,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -121,7 +124,9 @@ describe('State — TimeValue resolution', () => { expect(event.payload.event_context.extract_to).toBe( '2025-06-01T00:00:00.000Z' ); - expect(state.state.pendingWorkersNewest).toBe('2025-06-01T00:00:00.000Z'); + expect(state.sdkState.pendingWorkersNewest).toBe( + '2025-06-01T00:00:00.000Z' + ); }); it('should skip resolution when extraction_end_time has no type', async () => { @@ -148,7 +153,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -187,7 +192,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/time-value-resolver.test.ts b/src/state/time-value-resolver.test.ts index 43ccf7ef..6cc82a98 100644 --- a/src/state/time-value-resolver.test.ts +++ b/src/state/time-value-resolver.test.ts @@ -154,8 +154,6 @@ describe('time-value-resolver', () => { describe('resolveTimeValue', () => { const baseState: SdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', workersOldest: '2024-01-01T00:00:00.000Z', workersNewest: '2024-06-01T00:00:00.000Z', }; @@ -255,13 +253,12 @@ describe('time-value-resolver', () => { expect(result).toBe('2024-06-01T00:00:00.000Z'); }); - it('should throw if workersNewest is not set', () => { - expect(() => - resolveTimeValue( - { type: TimeValueType.WORKERS_NEWEST }, - { workersNewest: '' } - ) - ).toThrow('workersNewest is not set in state'); + it('should return UNBOUNDED_DATE_TIME_VALUE if workersNewest is not set', () => { + const result = resolveTimeValue( + { type: TimeValueType.WORKERS_NEWEST }, + { workersNewest: '' } + ); + expect(result).toBe(UNBOUNDED_DATE_TIME_VALUE); }); }); @@ -332,16 +329,15 @@ describe('time-value-resolver', () => { expect(result).toBe('2024-06-01T00:30:00.000Z'); }); - it('should throw if workersNewest is not set', () => { - expect(() => - resolveTimeValue( - { - type: TimeValueType.WORKERS_NEWEST_PLUS_WINDOW, - value: '2h', - }, - { workersNewest: '' } - ) - ).toThrow('workersNewest is not set in state'); + it('should return UNBOUNDED_DATE_TIME_VALUE if workersNewest is not set', () => { + const result = resolveTimeValue( + { + type: TimeValueType.WORKERS_NEWEST_PLUS_WINDOW, + value: '30m', + }, + { workersNewest: '' } + ); + expect(result).toBe(UNBOUNDED_DATE_TIME_VALUE); }); it('should throw if value (duration) is missing', () => { @@ -367,8 +363,6 @@ describe('time-value-resolver', () => { const FIXED_NOW = '2026-02-26T15:30:00.000Z'; const scenarioState: SdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', workersOldest: '2024-01-01T00:00:00.000Z', workersNewest: '2024-06-01T00:00:00.000Z', }; diff --git a/src/tests/dummy-connector/data-extraction.test.ts b/src/tests/dummy-connector/data-extraction.test.ts index 09a4d331..89a48ad6 100644 --- a/src/tests/dummy-connector/data-extraction.test.ts +++ b/src/tests/dummy-connector/data-extraction.test.ts @@ -1,5 +1,5 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; @@ -9,7 +9,7 @@ import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; describe('Dummy Connector - Data Extraction', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/dummy-connector/data-extraction.ts b/src/tests/dummy-connector/data-extraction.ts index 8f1f7a6f..57311cbe 100644 --- a/src/tests/dummy-connector/data-extraction.ts +++ b/src/tests/dummy-connector/data-extraction.ts @@ -1,7 +1,6 @@ import { - ExtractorEventType, NormalizedItem, - processTask, + processExtractionTask, RepoInterface, } from '../../index'; @@ -25,7 +24,7 @@ const repos: RepoInterface[] = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -44,9 +43,10 @@ processTask({ } await adapter.getRepo('tasks')?.push(tasks); - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/dummy-connector/external-sync-units-extraction.ts b/src/tests/dummy-connector/external-sync-units-extraction.ts index 0cb6d2f6..c091f1a1 100644 --- a/src/tests/dummy-connector/external-sync-units-extraction.ts +++ b/src/tests/dummy-connector/external-sync-units-extraction.ts @@ -1,6 +1,10 @@ -import { ExternalSyncUnit, ExtractorEventType, processTask } from '../../index'; +import { + AirSyncDefaultItemTypes, + ExternalSyncUnit, + processExtractionTask, +} from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { const dummyExternalSyncUnits: ExternalSyncUnit[] = [ { @@ -12,15 +16,26 @@ processTask({ }, ]; - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: dummyExternalSyncUnits, - }); + adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, + ]); + + await adapter + .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) + ?.push(dummyExternalSyncUnits); + + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to extract external sync units. Lambda timeout.', }, - }); + }; }, }); diff --git a/src/tests/dummy-connector/extraction.ts b/src/tests/dummy-connector/extraction.ts index 4889fb6a..5ff952b8 100644 --- a/src/tests/dummy-connector/extraction.ts +++ b/src/tests/dummy-connector/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,15 +7,18 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/dummy-connector/metadata-extraction.test.ts b/src/tests/dummy-connector/metadata-extraction.test.ts index 02d19d6d..6a35d3b1 100644 --- a/src/tests/dummy-connector/metadata-extraction.test.ts +++ b/src/tests/dummy-connector/metadata-extraction.test.ts @@ -1,5 +1,5 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; @@ -11,7 +11,7 @@ import run from './extraction'; jest.setTimeout(60000); describe('Dummy Connector - Metadata Extraction', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata }, diff --git a/src/tests/dummy-connector/metadata-extraction.ts b/src/tests/dummy-connector/metadata-extraction.ts index 3427a559..89b383cf 100644 --- a/src/tests/dummy-connector/metadata-extraction.ts +++ b/src/tests/dummy-connector/metadata-extraction.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; const repos = [ { @@ -6,7 +6,7 @@ const repos = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -16,11 +16,13 @@ processTask({ .getRepo('external_domain_metadata') ?.push([externalDomainMetadata]); - await adapter.emit(ExtractorEventType.MetadataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.MetadataExtractionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to extract metadata. Lambda timeout.' }, - }); + }; }, }); diff --git a/src/tests/event-data-size-limit/extraction.ts b/src/tests/event-data-size-limit/extraction.ts index 9fa4a366..1c082448 100644 --- a/src/tests/event-data-size-limit/extraction.ts +++ b/src/tests/event-data-size-limit/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -13,16 +13,19 @@ const initialDomainMapping = {}; * With 3000 items and batch size 1, we get 3000 artifacts. * Each artifact metadata is ~55 bytes, so 3000 * 55 = 165KB > 160KB threshold. */ -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { batchSize: 1, // Batch size of 1 to generate many artifacts isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/event-data-size-limit/size-limit-1.ts b/src/tests/event-data-size-limit/size-limit-1.ts index aff16624..54f7aa90 100644 --- a/src/tests/event-data-size-limit/size-limit-1.ts +++ b/src/tests/event-data-size-limit/size-limit-1.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; /** * Test worker that generates items to trigger the SQS size limit. @@ -8,7 +8,7 @@ import { ExtractorEventType, processTask } from '../../index'; * Each artifact metadata is ~55 bytes (id, item_type, item_count). * We need ~2857 artifacts to reach 160KB, so generating 3000 items. */ -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // Using external_domain_metadata itemType which doesn't require normalize adapter.initializeRepos([ @@ -20,10 +20,10 @@ processTask({ const repo = adapter.getRepo('external_domain_metadata'); if (!repo) { console.error('Repo not found after init'); - await adapter.emit(ExtractorEventType.DataExtractionError, { + return { + status: 'error', error: { message: 'Repo not found after init!' }, - }); - return; + }; } // Generate 3000 items with batch size 1 = 3000 artifacts @@ -41,15 +41,16 @@ processTask({ ]); if (adapter.isTimeout) { - return; + return { status: 'progress' }; } } console.log('Size limit was NOT triggered, emitting done'); - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { console.log('onTimeout called - emitting progress'); - await adapter.emit(ExtractorEventType.DataExtractionProgress); + return { status: 'progress' }; }, }); diff --git a/src/tests/spawn-worker/extraction.ts b/src/tests/spawn-worker/extraction.ts index ed0023b7..8965667a 100644 --- a/src/tests/spawn-worker/extraction.ts +++ b/src/tests/spawn-worker/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,15 +7,18 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath?: string) => { +const run = async (events: AirSyncEvent[], workerPath?: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/spawn-worker/some-cleanup-worker.ts b/src/tests/spawn-worker/some-cleanup-worker.ts index 268c5f07..52934d85 100644 --- a/src/tests/spawn-worker/some-cleanup-worker.ts +++ b/src/tests/spawn-worker/some-cleanup-worker.ts @@ -1,15 +1,18 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { console.log('Some cleanup logic executed.'); - await adapter.emit(ExtractorEventType.ExtractorStateDeletionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExtractorStateDeletionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to execute cleanup logic. Lambda timeout.', }, - }); + }; }, }); diff --git a/src/tests/spawn-worker/unknown-event-type.test.ts b/src/tests/spawn-worker/unknown-event-type.test.ts index 7d04f8e0..58f5204d 100644 --- a/src/tests/spawn-worker/unknown-event-type.test.ts +++ b/src/tests/spawn-worker/unknown-event-type.test.ts @@ -1,4 +1,5 @@ -import { EventType, ExtractorEventType } from '../../types/extraction'; +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; +import { EventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; import { createMockEvent } from '../../testing/mock-event'; @@ -18,7 +19,7 @@ describe('Unknown event type', () => { expect(lastRequest?.url).toContain('/callback_url'); expect(lastRequest?.method).toBe('POST'); expect((lastRequest?.body as { event_type: string }).event_type).toBe( - ExtractorEventType.UnknownEventType + UNKNOWN_EVENT_TYPE ); }); }); diff --git a/src/tests/spawn-worker/unknown-event-type.ts b/src/tests/spawn-worker/unknown-event-type.ts index 05284e4e..02c8e7cc 100644 --- a/src/tests/spawn-worker/unknown-event-type.ts +++ b/src/tests/spawn-worker/unknown-event-type.ts @@ -1,12 +1,14 @@ -import { processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { console.log('task should not be called.'); + return { status: 'success' }; }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - onTimeout: async ({ adapter }) => { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { console.log('onTimeout should not be called.'); + return { status: 'success' }; }, }); diff --git a/src/tests/test-helpers.interfaces.ts b/src/tests/test-helpers.interfaces.ts index 987c0827..e782ab7a 100644 --- a/src/tests/test-helpers.interfaces.ts +++ b/src/tests/test-helpers.interfaces.ts @@ -1,11 +1,11 @@ import { DeepPartial } from '../testing/mock-event'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; /** * Internal variant of the createMockEvent overrides — a deep partial of - * {@link AirdropEvent}. The shared test wrapper injects defaults automatically. + * {@link AirSyncEvent}. The shared test wrapper injects defaults automatically. */ -export type CreateMockEventOverrides = DeepPartial; +export type CreateMockEventOverrides = DeepPartial; /** * Options for creating a file stream response. diff --git a/src/tests/timeout-handling/extraction.ts b/src/tests/timeout-handling/extraction.ts index f24232aa..bc9d00d6 100644 --- a/src/tests/timeout-handling/extraction.ts +++ b/src/tests/timeout-handling/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,17 +7,20 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { batchSize: 1000, timeout: 5 * 1000, // 5 seconds isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/timeout-handling/no-timeout.test.ts b/src/tests/timeout-handling/no-timeout.test.ts index 22582eed..06f60cad 100644 --- a/src/tests/timeout-handling/no-timeout.test.ts +++ b/src/tests/timeout-handling/no-timeout.test.ts @@ -1,5 +1,5 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; @@ -10,7 +10,7 @@ import { createMockEvent } from '../../testing/mock-event'; import run from './extraction'; describe('No timeout', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/no-timeout.ts b/src/tests/timeout-handling/no-timeout.ts index 8eb47e39..2a87475b 100644 --- a/src/tests/timeout-handling/no-timeout.ts +++ b/src/tests/timeout-handling/no-timeout.ts @@ -1,14 +1,16 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { for (let i = 0; i < 10; i++) { console.log('no-timeout iteration', i); } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-blocked.test.ts b/src/tests/timeout-handling/timeout-blocked.test.ts index daecc18c..72faab8c 100644 --- a/src/tests/timeout-handling/timeout-blocked.test.ts +++ b/src/tests/timeout-handling/timeout-blocked.test.ts @@ -1,5 +1,5 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; @@ -11,7 +11,7 @@ import run from './extraction'; jest.setTimeout(10000); describe('Timeout blocked', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-blocked.ts b/src/tests/timeout-handling/timeout-blocked.ts index f7c4ee6d..55f9b99e 100644 --- a/src/tests/timeout-handling/timeout-blocked.ts +++ b/src/tests/timeout-handling/timeout-blocked.ts @@ -1,7 +1,8 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { // Simple CPU-intensive nested loops that block the event loop let result = 0; for (let i = 0; i < 100000; i++) { @@ -16,9 +17,10 @@ processTask({ } } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-graceful.test.ts b/src/tests/timeout-handling/timeout-graceful.test.ts index 1a5071da..12c6caa7 100644 --- a/src/tests/timeout-handling/timeout-graceful.test.ts +++ b/src/tests/timeout-handling/timeout-graceful.test.ts @@ -1,5 +1,5 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; @@ -11,7 +11,7 @@ import run from './extraction'; jest.setTimeout(10000); describe('Timeout graceful', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-graceful.ts b/src/tests/timeout-handling/timeout-graceful.ts index 56ccb396..5293aaef 100644 --- a/src/tests/timeout-handling/timeout-graceful.ts +++ b/src/tests/timeout-handling/timeout-graceful.ts @@ -1,21 +1,22 @@ import { sleep } from '../../common/helpers'; -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // Use async delays that allow the event loop to process timeout messages for (let i = 0; i < 10; i++) { if (adapter.isTimeout) { - return; + return { status: 'progress' }; } console.log('timeout-graceful iteration', i); await sleep(1000); } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-unblocked.test.ts b/src/tests/timeout-handling/timeout-unblocked.test.ts index 3747fe24..0924c6ea 100644 --- a/src/tests/timeout-handling/timeout-unblocked.test.ts +++ b/src/tests/timeout-handling/timeout-unblocked.test.ts @@ -1,5 +1,5 @@ import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; @@ -11,7 +11,7 @@ import run from './extraction'; jest.setTimeout(10000); // 10 seconds describe('Timeout unblocked', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-unblocked.ts b/src/tests/timeout-handling/timeout-unblocked.ts index 0ac268c1..d4132ddd 100644 --- a/src/tests/timeout-handling/timeout-unblocked.ts +++ b/src/tests/timeout-handling/timeout-unblocked.ts @@ -1,7 +1,7 @@ import { sleep } from '../../common/helpers'; -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // CPU-intensive nested loops that yield control after logging // This allows the event loop to process timeout messages @@ -9,7 +9,7 @@ processTask({ for (let i = 0; i < 100000; i++) { for (let j = 0; j < 10000; j++) { if (adapter.isTimeout) { - return; + return { status: 'progress' }; } result += Math.sqrt(i * j) * Math.sin(i + j); @@ -23,9 +23,10 @@ processTask({ } } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/types/loading.ts b/src/types/loading.ts index 37d2d305..834b02a9 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -151,6 +151,4 @@ export enum LoaderEventType { LoaderAttachmentStateDeletionDone = 'LOADER_ATTACHMENT_STATE_DELETION_DONE', LoaderAttachmentStateDeletionError = 'LOADER_ATTACHMENT_STATE_DELETION_ERROR', - - UnknownEventType = 'UNKNOWN_EVENT_TYPE', } From 75ed4924ab18d25f677a7786f28e5d74fd387c5b Mon Sep 17 00:00:00 2001 From: svc-devrev-sdk Date: Mon, 22 Jun 2026 05:49:55 +0000 Subject: [PATCH 30/30] chore: release v2.0.0-beta.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index d0038c23..254cd1a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@devrev/airsync-sdk", - "version": "2.0.0-beta.0", + "version": "2.0.0-beta.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@devrev/airsync-sdk", - "version": "2.0.0-beta.0", + "version": "2.0.0-beta.4", "license": "ISC", "dependencies": { "@devrev/typescript-sdk": "^1.1.74", diff --git a/package.json b/package.json index 6858c359..b8f8bf33 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/airsync-sdk", - "version": "2.0.0-beta.0", + "version": "2.0.0-beta.4", "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.", "type": "commonjs", "main": "./dist/index.js",