From b70a58bda11bb84b5eadd19e137ca23fb5ff9c8a Mon Sep 17 00:00:00 2001 From: Navin Agarwal Date: Wed, 15 Oct 2025 10:18:48 -0700 Subject: [PATCH 1/6] (compat) Removed deprecated properties from IContainerStorageService and IRuntimeStorageService --- .changeset/heavy-bugs-thank.md | 26 + .../container-definitions.legacy.beta.api.md | 6 - .../common/container-definitions/package.json | 9 +- .../container-definitions/src/runtime.ts | 29 - ...eContainerDefinitionsPrevious.generated.ts | 2 + packages/framework/aqueduct/package.json | 6 +- .../validateAqueductPrevious.generated.ts | 1 + .../loader/container-loader/src/container.ts | 4 +- .../src/containerStorageAdapter.ts | 12 +- .../package.json | 9 +- ...nerRuntimeDefinitionsPrevious.generated.ts | 2 + .../runtime/container-runtime/package.json | 6 +- .../src/storageServiceWithAttachBlobs.ts | 92 -- .../src/test/blobManager.spec.ts | 1278 +++++++++++++++++ .../src/test/blobs/blobHandles.spec.ts | 10 +- .../src/test/containerRuntime.spec.ts | 7 +- .../runtime-definitions.legacy.alpha.api.md | 18 - .../runtime-definitions.legacy.beta.api.md | 18 - .../runtime/runtime-definitions/package.json | 15 +- .../runtime-definitions/src/protocol.ts | 79 - ...ateRuntimeDefinitionsPrevious.generated.ts | 4 + .../runtime/test-runtime-utils/package.json | 9 +- ...idateTestRuntimeUtilsPrevious.generated.ts | 2 + .../test/deRehydrateContainerTests.spec.ts | 17 +- packages/test/test-utils/package.json | 9 +- .../validateTestUtilsPrevious.generated.ts | 2 + 26 files changed, 1397 insertions(+), 275 deletions(-) create mode 100644 .changeset/heavy-bugs-thank.md create mode 100644 packages/runtime/container-runtime/src/test/blobManager.spec.ts diff --git a/.changeset/heavy-bugs-thank.md b/.changeset/heavy-bugs-thank.md new file mode 100644 index 000000000000..3e699adab49a --- /dev/null +++ b/.changeset/heavy-bugs-thank.md @@ -0,0 +1,26 @@ +--- +"@fluidframework/container-definitions": minor +"@fluidframework/runtime-definitions": minor +"__section": breaking +--- +Removed deprecated properties from "IRuntimeStorageService" and "IContainerStorageService" + +The following deprecated properties have been removed from `IRuntimeStorageService`: + +- `disposed` +- `dispose` +- `policies` +- `getSnapshotTree` +- `getSnapshot` +- `getVersions` +- `createBlob` +- `uploadSummaryWithContext` +- `downloadSummary` + +The following deprecated properties have been removed from `IContainerStorageService`: + +- `downloadSummary` +- `disposed` +- `dispose` + +The deprecations were announced in release 2.52.0 [here](https://github.com/microsoft/FluidFramework/releases/tag/client_v2.52.0). diff --git a/packages/common/container-definitions/api-report/container-definitions.legacy.beta.api.md b/packages/common/container-definitions/api-report/container-definitions.legacy.beta.api.md index 8930a85f63d5..25ce479df09a 100644 --- a/packages/common/container-definitions/api-report/container-definitions.legacy.beta.api.md +++ b/packages/common/container-definitions/api-report/container-definitions.legacy.beta.api.md @@ -209,12 +209,6 @@ export type IContainerPolicies = { // @beta @legacy export interface IContainerStorageService { createBlob(file: ArrayBufferLike): Promise; - // @deprecated - dispose?(error?: Error): void; - // @deprecated - readonly disposed?: boolean; - // @deprecated - downloadSummary(handle: ISummaryHandle): Promise; getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions): Promise; getSnapshotTree(version?: IVersion, scenarioName?: string): Promise; getVersions(versionId: string | null, count: number, scenarioName?: string, fetchSource?: FetchSource): Promise; diff --git a/packages/common/container-definitions/package.json b/packages/common/container-definitions/package.json index 06a1c4483c9d..e5c4341b65cf 100644 --- a/packages/common/container-definitions/package.json +++ b/packages/common/container-definitions/package.json @@ -110,7 +110,14 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Interface_IContainerContext": { + "backCompat": false + }, + "Interface_IContainerStorageService": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/common/container-definitions/src/runtime.ts b/packages/common/container-definitions/src/runtime.ts index 9130981a58b5..9401dde004cb 100644 --- a/packages/common/container-definitions/src/runtime.ts +++ b/packages/common/container-definitions/src/runtime.ts @@ -26,7 +26,6 @@ import type { ISnapshotFetchOptions, FetchSource, IDocumentStorageServicePolicies, - ISummaryHandle, } from "@fluidframework/driver-definitions/internal"; import type { IAudience } from "./audience.js"; @@ -143,25 +142,6 @@ export interface IBatchMessage { * @legacy @beta */ export interface IContainerStorageService { - /** - * Whether or not the object has been disposed. - * If true, the object should be considered invalid, and its other state should be disregarded. - * - * @deprecated - This API is deprecated and will be removed in a future release. No replacement is planned as - * it is unused in the Runtime layer. - */ - readonly disposed?: boolean; - - /** - * Dispose of the object and its resources. - * @param error - Optional error indicating the reason for the disposal, if the object was - * disposed as the result of an error. - * - * @deprecated - This API is deprecated and will be removed in a future release. No replacement is planned as - * it is unused in the Runtime layer. - */ - dispose?(error?: Error): void; - /** * Policies implemented/instructed by driver. * @@ -230,15 +210,6 @@ export interface IContainerStorageService { * Returns the uploaded summary handle. */ uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise; - - /** - * Retrieves the commit that matches the packfile handle. If the packfile has already been committed and the - * server has deleted it this call may result in a broken promise. - * - * @deprecated - This API is deprecated and will be removed in a future release. No replacement is planned as - * it is unused in the Runtime and below layers. - */ - downloadSummary(handle: ISummaryHandle): Promise; } /** diff --git a/packages/common/container-definitions/src/test/types/validateContainerDefinitionsPrevious.generated.ts b/packages/common/container-definitions/src/test/types/validateContainerDefinitionsPrevious.generated.ts index c70852f2d416..ad1fcf963900 100644 --- a/packages/common/container-definitions/src/test/types/validateContainerDefinitionsPrevious.generated.ts +++ b/packages/common/container-definitions/src/test/types/validateContainerDefinitionsPrevious.generated.ts @@ -211,6 +211,7 @@ declare type old_as_current_for_Interface_IContainerContext = requireAssignableT * typeValidation.broken: * "Interface_IContainerContext": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IContainerContext = requireAssignableTo, TypeOnly> /* @@ -265,6 +266,7 @@ declare type old_as_current_for_Interface_IContainerStorageService = requireAssi * typeValidation.broken: * "Interface_IContainerStorageService": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IContainerStorageService = requireAssignableTo, TypeOnly> /* diff --git a/packages/framework/aqueduct/package.json b/packages/framework/aqueduct/package.json index c9ba96239879..6e9fb44256b6 100644 --- a/packages/framework/aqueduct/package.json +++ b/packages/framework/aqueduct/package.json @@ -154,7 +154,11 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Interface_IDataObjectProps": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/framework/aqueduct/src/test/types/validateAqueductPrevious.generated.ts b/packages/framework/aqueduct/src/test/types/validateAqueductPrevious.generated.ts index 5871220938e4..5483e3cea0ed 100644 --- a/packages/framework/aqueduct/src/test/types/validateAqueductPrevious.generated.ts +++ b/packages/framework/aqueduct/src/test/types/validateAqueductPrevious.generated.ts @@ -319,4 +319,5 @@ declare type old_as_current_for_Interface_IDataObjectProps = requireAssignableTo * typeValidation.broken: * "Interface_IDataObjectProps": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IDataObjectProps = requireAssignableTo, TypeOnly> diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index 35ec5f477fbc..cbd68e6bdac7 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -32,6 +32,7 @@ import type { ReadOnlyInfo, ILoader, ILoaderOptions, + IContainerStorageService, } from "@fluidframework/container-definitions/internal"; import { isFluidCodeDetails } from "@fluidframework/container-definitions/internal"; import { @@ -54,7 +55,6 @@ import { import { type IDocumentService, type IDocumentServiceFactory, - type IDocumentStorageService, type IResolvedUrl, type ISnapshot, type IThrottlingWarning, @@ -1887,7 +1887,7 @@ export class Container private async initializeProtocolStateFromSnapshot( attributes: IDocumentAttributes, - storage: IDocumentStorageService, + storage: IContainerStorageService, snapshot: ISnapshotTree | undefined, ): Promise { const quorumSnapshot: IQuorumSnapshot = { diff --git a/packages/loader/container-loader/src/containerStorageAdapter.ts b/packages/loader/container-loader/src/containerStorageAdapter.ts index f491fdca19b1..9e4262eb43d4 100644 --- a/packages/loader/container-loader/src/containerStorageAdapter.ts +++ b/packages/loader/container-loader/src/containerStorageAdapter.ts @@ -10,7 +10,7 @@ import type { } from "@fluidframework/container-definitions/internal"; import type { IDisposable } from "@fluidframework/core-interfaces"; import { assert } from "@fluidframework/core-utils/internal"; -import type { ISummaryHandle, ISummaryTree } from "@fluidframework/driver-definitions"; +import type { ISummaryTree } from "@fluidframework/driver-definitions"; import type { FetchSource, IDocumentService, @@ -247,16 +247,6 @@ export class ContainerStorageAdapter public async createBlob(file: ArrayBufferLike): Promise { return this._storageService.createBlob(file); } - - /** - * {@link IRuntimeStorageService.downloadSummary}. - * - * @deprecated - This API is deprecated and will be removed in a future release. No replacement is planned as - * it is unused in the Runtime and below layers. - */ - public async downloadSummary(handle: ISummaryHandle): Promise { - return this._storageService.downloadSummary(handle); - } } /** diff --git a/packages/runtime/container-runtime-definitions/package.json b/packages/runtime/container-runtime-definitions/package.json index 1a68cd013f6d..3040b0c9c638 100644 --- a/packages/runtime/container-runtime-definitions/package.json +++ b/packages/runtime/container-runtime-definitions/package.json @@ -101,7 +101,14 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Interface_IContainerRuntime": { + "backCompat": false + }, + "Interface_IContainerRuntimeWithResolveHandle_Deprecated": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/runtime/container-runtime-definitions/src/test/types/validateContainerRuntimeDefinitionsPrevious.generated.ts b/packages/runtime/container-runtime-definitions/src/test/types/validateContainerRuntimeDefinitionsPrevious.generated.ts index 13f5bcb7b763..679fc62b0a35 100644 --- a/packages/runtime/container-runtime-definitions/src/test/types/validateContainerRuntimeDefinitionsPrevious.generated.ts +++ b/packages/runtime/container-runtime-definitions/src/test/types/validateContainerRuntimeDefinitionsPrevious.generated.ts @@ -22,6 +22,7 @@ declare type MakeUnusedImportErrorsGoAway = TypeOnly | MinimalType | Fu * typeValidation.broken: * "Interface_IContainerRuntime": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IContainerRuntime = requireAssignableTo, TypeOnly> /* @@ -49,6 +50,7 @@ declare type old_as_current_for_Interface_IContainerRuntimeWithResolveHandle_Dep * typeValidation.broken: * "Interface_IContainerRuntimeWithResolveHandle_Deprecated": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IContainerRuntimeWithResolveHandle_Deprecated = requireAssignableTo, TypeOnly> /* diff --git a/packages/runtime/container-runtime/package.json b/packages/runtime/container-runtime/package.json index 3402383516d8..60f1985c28bc 100644 --- a/packages/runtime/container-runtime/package.json +++ b/packages/runtime/container-runtime/package.json @@ -218,7 +218,11 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Interface_LoadContainerRuntimeParams": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/runtime/container-runtime/src/storageServiceWithAttachBlobs.ts b/packages/runtime/container-runtime/src/storageServiceWithAttachBlobs.ts index 8cada008b0d7..1986fb8e20ff 100644 --- a/packages/runtime/container-runtime/src/storageServiceWithAttachBlobs.ts +++ b/packages/runtime/container-runtime/src/storageServiceWithAttachBlobs.ts @@ -3,20 +3,7 @@ * Licensed under the MIT License. */ -import type { - FetchSource, - ICreateBlobResponse, - IDocumentStorageServicePolicies, - ISnapshot, - ISnapshotFetchOptions, - ISnapshotTree, - ISummaryContext, - ISummaryHandle, - ISummaryTree, - IVersion, -} from "@fluidframework/driver-definitions/internal"; import type { IRuntimeStorageService } from "@fluidframework/runtime-definitions/internal"; -import { UsageError } from "@fluidframework/telemetry-utils/internal"; /** * IRuntimeStorageService proxy which intercepts requests if they can be satisfied by the blobs received in the @@ -28,14 +15,6 @@ export class StorageServiceWithAttachBlobs implements IRuntimeStorageService { private readonly attachBlobs: Map, ) {} - /** - * {@link IRuntimeStorageService.policies}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public get policies(): IDocumentStorageServicePolicies | undefined { - return this.internalStorageService.policies; - } - public async readBlob(id: string): Promise { const blob = this.attachBlobs.get(id); if (blob !== undefined) { @@ -46,75 +25,4 @@ export class StorageServiceWithAttachBlobs implements IRuntimeStorageService { // IRuntimeStorageService to cache appropriately, no need to double-cache. return this.internalStorageService.readBlob(id); } - - /** - * {@link IRuntimeStorageService.getSnapshotTree}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public async getSnapshotTree( - version?: IVersion, - scenarioName?: string, - // eslint-disable-next-line @rushstack/no-new-null - ): Promise { - return this.internalStorageService.getSnapshotTree(version, scenarioName); - } - - /** - * {@link IRuntimeStorageService.getSnapshot}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public async getSnapshot(snapshotFetchOptions?: ISnapshotFetchOptions): Promise { - if (this.internalStorageService.getSnapshot !== undefined) { - return this.internalStorageService.getSnapshot(snapshotFetchOptions); - } - throw new UsageError( - "getSnapshot api should exist on internal storage in documentStorageServiceProxy class", - ); - } - - /** - * {@link IRuntimeStorageService.getVersions}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public async getVersions( - // eslint-disable-next-line @rushstack/no-new-null - versionId: string | null, - count: number, - scenarioName?: string, - fetchSource?: FetchSource, - ): Promise { - return this.internalStorageService.getVersions( - versionId, - count, - scenarioName, - fetchSource, - ); - } - - /** - * {@link IRuntimeStorageService.uploadSummaryWithContext}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public async uploadSummaryWithContext( - summary: ISummaryTree, - context: ISummaryContext, - ): Promise { - return this.internalStorageService.uploadSummaryWithContext(summary, context); - } - - /** - * {@link IRuntimeStorageService.createBlob}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public async createBlob(file: ArrayBufferLike): Promise { - return this.internalStorageService.createBlob(file); - } - - /** - * {@link IRuntimeStorageService.downloadSummary}. - * @deprecated - This will be removed in a future release. The DataStore layer does not need this. - */ - public async downloadSummary(handle: ISummaryHandle): Promise { - return this.internalStorageService.downloadSummary(handle); - } } diff --git a/packages/runtime/container-runtime/src/test/blobManager.spec.ts b/packages/runtime/container-runtime/src/test/blobManager.spec.ts new file mode 100644 index 000000000000..21f325d8b2e3 --- /dev/null +++ b/packages/runtime/container-runtime/src/test/blobManager.spec.ts @@ -0,0 +1,1278 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +import { + IsoBuffer, + TypedEventEmitter, + bufferToString, + gitHashFile, +} from "@fluid-internal/client-utils"; +import { + AttachState, + type IContainerStorageService, +} from "@fluidframework/container-definitions/internal"; +import type { IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions/internal"; +import type { + ConfigTypes, + IConfigProviderBase, + IErrorBase, +} from "@fluidframework/core-interfaces"; +import type { + IFluidHandleContext, + IFluidHandleInternal, +} from "@fluidframework/core-interfaces/internal"; +import { Deferred } from "@fluidframework/core-utils/internal"; +import { type IClientDetails, SummaryType } from "@fluidframework/driver-definitions"; +import type { ISequencedMessageEnvelope } from "@fluidframework/runtime-definitions/internal"; +import { + isFluidHandleInternalPayloadPending, + isFluidHandlePayloadPending, + isLocalFluidHandle, +} from "@fluidframework/runtime-utils/internal"; +import { + LoggingError, + MockLogger, + type MonitoringContext, + createChildLogger, + mixinMonitoringContext, + type ITelemetryLoggerExt, +} from "@fluidframework/telemetry-utils/internal"; +import Sinon from "sinon"; +import { v4 as uuid } from "uuid"; + +import { + BlobManager, + type IBlobManagerLoadInfo, + type IBlobManagerRuntime, + blobManagerBasePath, + redirectTableBlobName, + type IPendingBlobs, +} from "../blobManager/index.js"; + +const MIN_TTL = 24 * 60 * 60; // same as ODSP +abstract class BaseMockBlobStorage + implements Pick +{ + public blobs: Map = new Map(); + public abstract createBlob(blob: ArrayBufferLike); + public async readBlob(id: string) { + const blob = this.blobs.get(id); + assert(!!blob); + return blob; + } +} + +class DedupeStorage extends BaseMockBlobStorage { + public minTTL: number = MIN_TTL; + + public async createBlob(blob: ArrayBufferLike) { + const s = bufferToString(blob, "base64"); + const id = await gitHashFile(IsoBuffer.from(s, "base64")); + this.blobs.set(id, blob); + return { id, minTTLInSeconds: this.minTTL }; + } +} + +class NonDedupeStorage extends BaseMockBlobStorage { + public async createBlob(blob: ArrayBufferLike) { + const id = this.blobs.size.toString(); + this.blobs.set(id, blob); + return { id, minTTLInSeconds: MIN_TTL }; + } +} + +export class MockRuntime + extends TypedEventEmitter + implements IBlobManagerRuntime +{ + public readonly clientDetails: IClientDetails = { capabilities: { interactive: true } }; + constructor( + public mc: MonitoringContext, + createBlobPayloadPending: boolean, + blobManagerLoadInfo: IBlobManagerLoadInfo = {}, + attached = false, + stashed: unknown[] = [[], {}], + ) { + super(); + this.attachState = attached ? AttachState.Attached : AttachState.Detached; + this.ops = stashed[0] as unknown[]; + this.baseLogger = mc.logger; + this.blobManager = new BlobManager({ + routeContext: undefined as unknown as IFluidHandleContext, + blobManagerLoadInfo, + storage: this.getStorage(), + sendBlobAttachOp: (localId: string, blobId?: string) => + this.sendBlobAttachOp(localId, blobId), + blobRequested: () => undefined, + isBlobDeleted: (blobPath: string) => this.isBlobDeleted(blobPath), + runtime: this, + stashedBlobs: stashed[1] as IPendingBlobs | undefined, + createBlobPayloadPending, + }); + } + + public disposed: boolean = false; + + public get storage(): IContainerStorageService { + return (this.attachState === AttachState.Detached + ? this.detachedStorage + : this.attachedStorage) as unknown as IContainerStorageService; + } + + private processing = false; + public unprocessedBlobs = new Set(); + + public getStorage(): IContainerStorageService { + return { + createBlob: async (blob: ArrayBufferLike) => { + if (this.processing) { + return this.storage.createBlob(blob); + } + const P = this.processBlobsP.promise.then(async () => { + if (!this.connected && this.attachState === AttachState.Attached) { + this.unprocessedBlobs.delete(blob); + throw new Error("fake error due to having no connection to storage service"); + } else { + this.unprocessedBlobs.delete(blob); + return this.storage.createBlob(blob); + } + }); + this.unprocessedBlobs.add(blob); + this.emit("blob"); + this.blobPs.push(P); + return P; + }, + readBlob: async (id: string) => this.storage.readBlob(id), + } as unknown as IContainerStorageService; + } + + public sendBlobAttachOp(localId: string, blobId?: string): void { + this.ops.push({ metadata: { localId, blobId } }); + } + + public async createBlob( + blob: ArrayBufferLike, + signal?: AbortSignal, + ): Promise> { + const P = this.blobManager.createBlob(blob, signal); + this.handlePs.push(P); + return P; + } + + public async getBlob( + blobHandle: IFluidHandleInternal, + ): Promise { + const pathParts = blobHandle.absolutePath.split("/"); + const blobId = pathParts[2]; + const payloadPending = isFluidHandleInternalPayloadPending(blobHandle) + ? blobHandle.payloadPending + : false; + return this.blobManager.getBlob(blobId, payloadPending); + } + + public async getPendingLocalState(): Promise<(unknown[] | IPendingBlobs | undefined)[]> { + const pendingBlobs = await this.blobManager.attachAndGetPendingBlobs(); + return [[...this.ops], pendingBlobs]; + } + + public blobManager: BlobManager; + public connected = false; + public closed = false; + public attachState: AttachState; + public attachedStorage = new DedupeStorage(); + public detachedStorage = new NonDedupeStorage(); + public baseLogger: ITelemetryLoggerExt; + + private ops: unknown[] = []; + private processBlobsP = new Deferred(); + private blobPs: Promise[] = []; + private handlePs: Promise[] = []; + private readonly deletedBlobs: string[] = []; + + public processOps(): void { + assert(this.connected || this.ops.length === 0); + for (const op of this.ops) { + this.blobManager.processBlobAttachMessage(op as ISequencedMessageEnvelope, true); + } + this.ops = []; + } + + public async processBlobs( + resolve: boolean, + canRetry: boolean = false, + retryAfterSeconds?: number, + ): Promise { + const blobPs = this.blobPs; + this.blobPs = []; + if (resolve) { + this.processBlobsP.resolve(); + } else { + this.processBlobsP.reject( + new LoggingError("fake driver error", { canRetry, retryAfterSeconds }), + ); + } + this.processBlobsP = new Deferred(); + await Promise.allSettled(blobPs).catch(() => {}); + } + + public async processHandles(): Promise { + const handlePs = this.handlePs; + this.handlePs = []; + const handles = (await Promise.all(handlePs)) as IFluidHandleInternal[]; + for (const handle of handles) { + handle.attachGraph(); + } + } + + public async processAll(): Promise { + while (this.blobPs.length + this.handlePs.length + this.ops.length > 0) { + const p1 = this.processBlobs(true); + const p2 = this.processHandles(); + this.processOps(); + await Promise.race([p1, p2]); + this.processOps(); + await Promise.all([p1, p2]); + } + } + + public async attach(): Promise<{ + ids: string[]; + redirectTable: [string, string][] | undefined; + }> { + if (this.detachedStorage.blobs.size > 0) { + const table = new Map(); + for (const [detachedId, blob] of this.detachedStorage.blobs) { + const { id } = await this.attachedStorage.createBlob(blob); + table.set(detachedId, id); + } + this.detachedStorage.blobs.clear(); + this.blobManager.setRedirectTable(table); + } + const summary = validateSummary(this); + this.attachState = AttachState.Attached; + this.emit("attached"); + return summary; + } + + public async connect(delay = 0, processStashedWithRetry?: boolean): Promise { + assert(!this.connected); + await new Promise((resolve) => setTimeout(resolve, delay)); + this.connected = true; + this.emit("connected", "client ID"); + await this.processStashed(processStashedWithRetry); + const ops = this.ops; + this.ops = []; + for (const op of ops) { + // TODO: better typing + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + this.blobManager.reSubmit((op as any).metadata as Record | undefined); + } + } + + public async processStashed(processStashedWithRetry?: boolean): Promise { + // const uploadP = this.blobManager.stashedBlobsUploadP; + this.processing = true; + if (processStashedWithRetry) { + await this.processBlobs(false, false, 0); + // wait till next retry + await new Promise((resolve) => setTimeout(resolve, 1)); + // try again successfully + await this.processBlobs(true); + } else { + await this.processBlobs(true); + } + // await uploadP; + this.processing = false; + } + + public disconnect(): void { + assert(this.connected); + this.connected = false; + this.emit("disconnected"); + } + + public async remoteUpload( + blob: ArrayBufferLike, + ): Promise<{ metadata: { localId: string; blobId: string } }> { + const response = await this.storage.createBlob(blob); + const op = { metadata: { localId: uuid(), blobId: response.id } }; + this.blobManager.processBlobAttachMessage(op as ISequencedMessageEnvelope, false); + return op; + } + + public deleteBlob(blobHandle: IFluidHandleInternal): void { + this.deletedBlobs.push(blobHandle.absolutePath); + } + + public isBlobDeleted(blobPath: string): boolean { + return this.deletedBlobs.includes(blobPath); + } +} + +export const validateSummary = ( + runtime: MockRuntime, +): { + ids: string[]; + redirectTable: [string, string][] | undefined; +} => { + const summary = runtime.blobManager.summarize(); + const ids: string[] = []; + let redirectTable: [string, string][] | undefined; + for (const [key, attachment] of Object.entries(summary.summary.tree)) { + if (attachment.type === SummaryType.Attachment) { + ids.push(attachment.id); + } else { + assert.strictEqual(key, redirectTableBlobName); + assert(attachment.type === SummaryType.Blob); + assert(typeof attachment.content === "string"); + redirectTable = [ + ...new Map( + JSON.parse(attachment.content) as [string, string][], + ).entries(), + ]; + } + } + return { ids, redirectTable }; +}; + +for (const createBlobPayloadPending of [false, true]) { + describe(`BlobManager (pending payloads): ${createBlobPayloadPending}`, () => { + const mockLogger = new MockLogger(); + let runtime: MockRuntime; + let createBlob: (blob: ArrayBufferLike, signal?: AbortSignal) => Promise; + let waitForBlob: (blob: ArrayBufferLike) => Promise; + let mc: MonitoringContext; + let injectedSettings: Record = {}; + + beforeEach(() => { + const configProvider = (settings: Record): IConfigProviderBase => ({ + getRawConfig: (name: string): ConfigTypes => settings[name], + }); + mc = mixinMonitoringContext( + createChildLogger({ logger: mockLogger }), + configProvider(injectedSettings), + ); + runtime = new MockRuntime(mc, createBlobPayloadPending); + + // ensures this blob will be processed next time runtime.processBlobs() is called + waitForBlob = async (blob) => { + if (!runtime.unprocessedBlobs.has(blob)) { + await new Promise((resolve) => + runtime.on("blob", () => { + if (runtime.unprocessedBlobs.has(blob)) { + resolve(); + } + }), + ); + } + }; + + // create blob and await the handle after the test + createBlob = async (blob: ArrayBufferLike, signal?: AbortSignal) => { + runtime + .createBlob(blob, signal) + .then((handle) => { + if (createBlobPayloadPending) { + handle.attachGraph(); + } + return handle; + }) + // Suppress errors here, we expect them to be detected elsewhere + .catch(() => {}); + await waitForBlob(blob); + }; + + const onNoPendingBlobs = () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access -- Accessing private property + assert((runtime.blobManager as any).pendingBlobs.size === 0); + }; + + runtime.blobManager.events.on("noPendingBlobs", () => onNoPendingBlobs()); + }); + + afterEach(async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access -- Accessing private property + assert.strictEqual((runtime.blobManager as any).pendingBlobs.size, 0); + injectedSettings = {}; + mockLogger.clear(); + }); + + it("empty snapshot", () => { + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("non empty snapshot", async () => { + await runtime.attach(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it("hasPendingBlobs", async () => { + await runtime.attach(); + await runtime.connect(); + + assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob2", "utf8")); + assert.strictEqual(runtime.blobManager.hasPendingBlobs, true); + await runtime.processAll(); + assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 2); + assert.strictEqual(summaryData.redirectTable?.length, 2); + }); + + it("NoPendingBlobs count", async () => { + await runtime.attach(); + await runtime.connect(); + let count = 0; + runtime.blobManager.events.on("noPendingBlobs", () => count++); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + assert.strictEqual(count, 1); + await createBlob(IsoBuffer.from("blob2", "utf8")); + await createBlob(IsoBuffer.from("blob3", "utf8")); + await runtime.processAll(); + assert.strictEqual(count, 2); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 3); + assert.strictEqual(summaryData.redirectTable?.length, 3); + }); + + it("detached snapshot", async () => { + assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + assert.strictEqual(runtime.blobManager.hasPendingBlobs, true); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("detached->attached snapshot", async () => { + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + assert.strictEqual(runtime.blobManager.hasPendingBlobs, true); + await runtime.attach(); + assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it("uploads while disconnected", async () => { + await runtime.attach(); + const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.connect(); + await runtime.processAll(); + await assert.doesNotReject(handleP); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it("reupload blob if expired", async () => { + await runtime.attach(); + await runtime.connect(); + runtime.attachedStorage.minTTL = 0.001; // force expired TTL being less than connection time (50ms) + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processBlobs(true); + runtime.disconnect(); + await new Promise((resolve) => setTimeout(resolve, 50)); + await runtime.connect(); + await runtime.processAll(); + }); + + it("completes after disconnection while upload pending", async () => { + await runtime.attach(); + await runtime.connect(); + + const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); + runtime.disconnect(); + await runtime.connect(10); // adding some delay to reconnection + await runtime.processAll(); + await assert.doesNotReject(handleP); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it("upload fails gracefully", async () => { + await runtime.attach(); + await runtime.connect(); + + if (createBlobPayloadPending) { + const handle = await runtime.createBlob(IsoBuffer.from("blob", "utf8")); + assert.strict(isFluidHandlePayloadPending(handle)); + assert.strict(isLocalFluidHandle(handle)); + assert.strictEqual( + handle.payloadState, + "pending", + "Handle should be in pending state", + ); + assert.strictEqual( + handle.payloadShareError, + undefined, + "handle should not have an error yet", + ); + let failed = false; + const onPayloadShareFailed = (error: unknown): void => { + failed = true; + assert.strictEqual( + (error as Error).message, + "fake driver error", + "Did not receive the expected error", + ); + handle.events.off("payloadShareFailed", onPayloadShareFailed); + }; + handle.events.on("payloadShareFailed", onPayloadShareFailed); + await runtime.processHandles(); + await runtime.processBlobs(false); + runtime.processOps(); + assert.strict(failed, "should fail"); + assert.strictEqual( + handle.payloadState, + "pending", + "Handle should still be in pending state", + ); + assert.strictEqual( + (handle.payloadShareError as unknown as Error).message, + "fake driver error", + "Handle did not have the expected error", + ); + } else { + // If the blobs are created without pending payloads, we don't get to see the handle at + // all so we can't inspect its state. + const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processBlobs(false); + runtime.processOps(); + try { + await handleP; + assert.fail("should fail"); + } catch (error: unknown) { + assert.strictEqual((error as Error).message, "fake driver error"); + } + await assert.rejects(handleP); + } + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("updates handle state after success", async () => { + await runtime.attach(); + await runtime.connect(); + + if (createBlobPayloadPending) { + const handle = await runtime.createBlob(IsoBuffer.from("blob", "utf8")); + assert.strict(isFluidHandlePayloadPending(handle)); + assert.strictEqual( + handle.payloadState, + "pending", + "Handle should be in pending state", + ); + let shared = false; + const onPayloadShared = (): void => { + shared = true; + handle.events.off("payloadShared", onPayloadShared); + }; + handle.events.on("payloadShared", onPayloadShared); + await runtime.processHandles(); + await runtime.processBlobs(true); + runtime.processOps(); + assert.strict(shared, "should become shared"); + assert.strictEqual(handle.payloadState, "shared", "Handle should be in shared state"); + } else { + // Without placeholder blobs, we don't get to see the handle before it reaches "shared" state + // but we can still verify it's in the expected state when we get it. + const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + const handle = await handleP; + assert.strict(isFluidHandlePayloadPending(handle)); + assert.strictEqual(handle.payloadState, "shared", "Handle should be in shared state"); + } + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it.skip("upload fails and retries for retriable errors", async () => { + // Needs to use some sort of fake timer or write test in a different way as it is waiting + // for actual time which is causing timeouts. + await runtime.attach(); + await runtime.connect(); + const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processBlobs(false, true, 0); + // wait till next retry + await new Promise((resolve) => setTimeout(resolve, 1)); + // try again successfully + await runtime.processBlobs(true); + runtime.processOps(); + await runtime.processHandles(); + assert(handleP); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it("completes after disconnection while op in flight", async () => { + await runtime.attach(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processBlobs(true); + + runtime.disconnect(); + await runtime.connect(); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 2); + }); + + it("multiple disconnect/connects", async () => { + await runtime.attach(); + await runtime.connect(); + + const blob = IsoBuffer.from("blob", "utf8"); + const handleP = runtime.createBlob(blob); + runtime.disconnect(); + await runtime.connect(10); + + const blob2 = IsoBuffer.from("blob2", "utf8"); + const handleP2 = runtime.createBlob(blob2); + runtime.disconnect(); + + await runtime.connect(10); + await runtime.processAll(); + await assert.doesNotReject(handleP); + await assert.doesNotReject(handleP2); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 2); + assert.strictEqual(summaryData.redirectTable?.length, 2); + }); + + it("handles deduped IDs", async () => { + await runtime.attach(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + runtime.disconnect(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processBlobs(true); + + runtime.disconnect(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 6); + }); + + it("handles deduped IDs in detached", async () => { + runtime.detachedStorage = new DedupeStorage(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("handles deduped IDs in detached->attached", async () => { + runtime.detachedStorage = new DedupeStorage(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + await runtime.attach(); + await runtime.connect(); + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + + runtime.disconnect(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await createBlob(IsoBuffer.from("blob", "utf8")); + + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 4); + }); + + it("can load from summary", async () => { + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + await runtime.attach(); + const handle = runtime.createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.connect(); + + await runtime.processAll(); + await assert.doesNotReject(handle); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 3); + + const runtime2 = new MockRuntime(mc, createBlobPayloadPending, summaryData, true); + const summaryData2 = validateSummary(runtime2); + assert.strictEqual(summaryData2.ids.length, 1); + assert.strictEqual(summaryData2.redirectTable?.length, 3); + }); + + it("handles duplicate remote upload", async () => { + await runtime.attach(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.remoteUpload(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 2); + }); + + it("handles duplicate remote upload between upload and op", async () => { + await runtime.attach(); + await runtime.connect(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.processBlobs(true); + await runtime.remoteUpload(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 2); + }); + + it("handles duplicate remote upload with local ID", async () => { + await runtime.attach(); + + await createBlob(IsoBuffer.from("blob", "utf8")); + await runtime.connect(); + await runtime.processBlobs(true); + await runtime.remoteUpload(IsoBuffer.from("blob", "utf8")); + await runtime.processAll(); + + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 2); + }); + + it("includes blob IDs in summary while attaching", async () => { + await createBlob(IsoBuffer.from("blob1", "utf8")); + await createBlob(IsoBuffer.from("blob2", "utf8")); + await createBlob(IsoBuffer.from("blob3", "utf8")); + await runtime.processAll(); + + // While attaching with blobs, Container takes a summary while still in "Detached" + // state. BlobManager should know to include the list of attached blob + // IDs since this summary will be used to create the document + const summaryData = await runtime.attach(); + assert.strictEqual(summaryData?.ids.length, 3); + assert.strictEqual(summaryData?.redirectTable?.length, 3); + }); + + it("all blobs attached", async () => { + await runtime.attach(); + await runtime.connect(); + assert.strictEqual(runtime.blobManager.allBlobsAttached, true); + await createBlob(IsoBuffer.from("blob1", "utf8")); + // We immediately attach the handle in createBlob if pending payloads are enabled + assert.strictEqual(runtime.blobManager.allBlobsAttached, createBlobPayloadPending); + await runtime.processBlobs(true); + assert.strictEqual(runtime.blobManager.allBlobsAttached, createBlobPayloadPending); + await runtime.processAll(); + assert.strictEqual(runtime.blobManager.allBlobsAttached, true); + await createBlob(IsoBuffer.from("blob1", "utf8")); + await createBlob(IsoBuffer.from("blob2", "utf8")); + await createBlob(IsoBuffer.from("blob3", "utf8")); + assert.strictEqual(runtime.blobManager.allBlobsAttached, createBlobPayloadPending); + await runtime.processAll(); + assert.strictEqual(runtime.blobManager.allBlobsAttached, true); + }); + + it("runtime disposed during readBlob - log no error", async () => { + const someId = "someId"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call -- Accessing private property + (runtime.blobManager as any).setRedirection(someId, undefined); // To appease an assert + + // Mock storage.readBlob to dispose the runtime and throw an error + Sinon.stub(runtime.storage, "readBlob").callsFake(async (_id: string) => { + runtime.disposed = true; + throw new Error("BOOM!"); + }); + + await assert.rejects( + async () => runtime.blobManager.getBlob(someId, false), + (e: Error) => e.message === "BOOM!", + "Expected getBlob to throw with test error message", + ); + assert(runtime.disposed, "Runtime should be disposed"); + mockLogger.assertMatchNone( + [{ category: "error" }], + "Should not have logged any errors", + undefined, + false /* clearEventsAfterCheck */, + ); + mockLogger.assertMatch( + [{ category: "generic", eventName: "BlobManager:AttachmentReadBlob_cancel" }], + "Expected the _cancel event to be logged with 'generic' category", + ); + }); + + it("waits for blobs from handles with pending payloads without error", async () => { + await runtime.attach(); + + // Part of remoteUpload, but stop short of processing the message + const response = await runtime.storage.createBlob(IsoBuffer.from("blob", "utf8")); + const op = { metadata: { localId: uuid(), blobId: response.id } }; + + await assert.rejects( + runtime.blobManager.getBlob(op.metadata.localId, false), + "Rejects when attempting to get non-existent, shared-payload blobs", + ); + + // Try to get the blob that we haven't processed the attach op for yet. + // This simulates having found this ID in a handle with a pending payload that the remote client would have sent + const blobP = runtime.blobManager.getBlob(op.metadata.localId, true); + + // Process the op as if it were arriving from the remote client, which should cause the blobP promise to resolve + runtime.blobManager.processBlobAttachMessage(op as ISequencedMessageEnvelope, false); + + // Await the promise to confirm it settles and does not reject + await blobP; + }); + + describe("Abort Signal", () => { + it("abort before upload", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + ac.abort("abort test"); + try { + const blob = IsoBuffer.from("blob", "utf8"); + await runtime.createBlob(blob, ac.signal); + assert.fail("Should not succeed"); + + // TODO: better typing + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.status, undefined); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.uploadTime, undefined); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.acked, undefined); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.message, "uploadBlob aborted"); + } + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("abort while upload", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + const blob = IsoBuffer.from("blob", "utf8"); + const handleP = runtime.createBlob(blob, ac.signal); + ac.abort("abort test"); + assert.strictEqual(runtime.unprocessedBlobs.size, 1); + await runtime.processBlobs(true); + try { + await handleP; + assert.fail("Should not succeed"); + // TODO: better typing + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.uploadTime, undefined); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.acked, false); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.message, "uploadBlob aborted"); + } + assert(handleP); + await assert.rejects(handleP); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("abort while failed upload", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + const blob = IsoBuffer.from("blob", "utf8"); + const handleP = runtime.createBlob(blob, ac.signal); + const handleP2 = runtime.createBlob(IsoBuffer.from("blob2", "utf8")); + ac.abort("abort test"); + assert.strictEqual(runtime.unprocessedBlobs.size, 2); + await runtime.processBlobs(false); + try { + await handleP; + assert.fail("Should not succeed"); + } catch (error: unknown) { + assert.strictEqual((error as Error).message, "uploadBlob aborted"); + } + try { + await handleP2; + assert.fail("Should not succeed"); + } catch (error: unknown) { + assert.strictEqual((error as Error).message, "fake driver error"); + } + await assert.rejects(handleP); + await assert.rejects(handleP2); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("abort while disconnected", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + const blob = IsoBuffer.from("blob", "utf8"); + const handleP = runtime.createBlob(blob, ac.signal); + runtime.disconnect(); + ac.abort(); + await runtime.processBlobs(true); + try { + await handleP; + assert.fail("Should not succeed"); + } catch (error: unknown) { + assert.strictEqual((error as Error).message, "uploadBlob aborted"); + } + await assert.rejects(handleP); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("abort after blob succeeds", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + let handleP: Promise> | undefined; + try { + const blob = IsoBuffer.from("blob", "utf8"); + handleP = runtime.createBlob(blob, ac.signal); + await runtime.processAll(); + ac.abort(); + } catch { + assert.fail("abort after processing should not throw"); + } + assert(handleP); + await assert.doesNotReject(handleP); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 1); + assert.strictEqual(summaryData.redirectTable?.length, 1); + }); + + it("abort while waiting for op", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + const blob = IsoBuffer.from("blob", "utf8"); + const handleP = runtime.createBlob(blob, ac.signal); + const p1 = runtime.processBlobs(true); + const p2 = runtime.processHandles(); + // finish upload + await Promise.race([p1, p2]); + ac.abort(); + runtime.processOps(); + try { + // finish op + await handleP; + assert.fail("Should not succeed"); + + // TODO: better typing + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.message, "uploadBlob aborted"); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.ok(error.uploadTime); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.acked, false); + } + await assert.rejects(handleP); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + + it("resubmit on aborted pending op", async function () { + if (createBlobPayloadPending) { + // Blob creation with pending payload doesn't support abort + this.skip(); + } + await runtime.attach(); + await runtime.connect(); + const ac = new AbortController(); + let handleP: Promise> | undefined; + try { + handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8"), ac.signal); + const p1 = runtime.processBlobs(true); + const p2 = runtime.processHandles(); + // finish upload + await Promise.race([p1, p2]); + runtime.disconnect(); + ac.abort(); + await handleP; + assert.fail("Should not succeed"); + // TODO: better typing + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.message, "uploadBlob aborted"); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.ok(error.uploadTime); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + assert.strictEqual(error.acked, false); + } + await runtime.connect(); + runtime.processOps(); + + // TODO: `handleP` can be `undefined`; this should be made safer. + await assert.rejects(handleP as Promise>); + const summaryData = validateSummary(runtime); + assert.strictEqual(summaryData.ids.length, 0); + assert.strictEqual(summaryData.redirectTable, undefined); + }); + }); + + describe("Garbage Collection", () => { + let redirectTable: Map; + + /** + * Creates a blob with the given content and returns its local and storage id. + */ + async function createBlobAndGetIds(content: string) { + // For a given blob's GC node id, returns the blob id. + const getBlobIdFromGCNodeId = (gcNodeId: string) => { + const pathParts = gcNodeId.split("/"); + assert( + pathParts.length === 3 && pathParts[1] === blobManagerBasePath, + "Invalid blob node path", + ); + return pathParts[2]; + }; + + // For a given blob's id, returns the GC node id. + const getGCNodeIdFromBlobId = (blobId: string) => { + return `/${blobManagerBasePath}/${blobId}`; + }; + + const blobContents = IsoBuffer.from(content, "utf8"); + const handleP = runtime.createBlob(blobContents); + await runtime.processAll(); + + const blobHandle = await handleP; + const localId = getBlobIdFromGCNodeId(blobHandle.absolutePath); + assert(redirectTable.has(localId), "blob not found in redirect table"); + const storageId = redirectTable.get(localId); + assert(storageId !== undefined, "storage id not found in redirect table"); + return { + localId, + localGCNodeId: getGCNodeIdFromBlobId(localId), + storageId, + storageGCNodeId: getGCNodeIdFromBlobId(storageId), + }; + } + + beforeEach(() => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment -- Mutating private property + redirectTable = (runtime.blobManager as any).redirectTable; + }); + + it("fetching deleted blob fails", async () => { + await runtime.attach(); + await runtime.connect(); + const blob1Contents = IsoBuffer.from("blob1", "utf8"); + const blob2Contents = IsoBuffer.from("blob2", "utf8"); + const handle1P = runtime.createBlob(blob1Contents); + const handle2P = runtime.createBlob(blob2Contents); + await runtime.processAll(); + + const blob1Handle = await handle1P; + const blob2Handle = await handle2P; + + // Validate that the blobs can be retrieved. + assert.strictEqual(await runtime.getBlob(blob1Handle), blob1Contents); + assert.strictEqual(await runtime.getBlob(blob2Handle), blob2Contents); + + // Delete blob1. Retrieving it should result in an error. + runtime.deleteBlob(blob1Handle); + await assert.rejects( + async () => runtime.getBlob(blob1Handle), + (error: IErrorBase & { code: number | undefined }) => { + const blob1Id = blob1Handle.absolutePath.split("/")[2]; + const correctErrorType = error.code === 404; + const correctErrorMessage = error.message === `Blob was deleted: ${blob1Id}`; + return correctErrorType && correctErrorMessage; + }, + "Deleted blob2 fetch should have failed", + ); + + // Delete blob2. Retrieving it should result in an error. + runtime.deleteBlob(blob2Handle); + await assert.rejects( + async () => runtime.getBlob(blob2Handle), + (error: IErrorBase & { code: number | undefined }) => { + const blob2Id = blob2Handle.absolutePath.split("/")[2]; + const correctErrorType = error.code === 404; + const correctErrorMessage = error.message === `Blob was deleted: ${blob2Id}`; + return correctErrorType && correctErrorMessage; + }, + "Deleted blob2 fetch should have failed", + ); + }); + + // Support for this config has been removed. + const legacyKey_disableAttachmentBlobSweep = + "Fluid.GarbageCollection.DisableAttachmentBlobSweep"; + for (const disableAttachmentBlobsSweep of [true, undefined]) + it(`deletes unused blobs regardless of DisableAttachmentBlobsSweep setting [DisableAttachmentBlobsSweep=${disableAttachmentBlobsSweep}]`, async () => { + injectedSettings[legacyKey_disableAttachmentBlobSweep] = disableAttachmentBlobsSweep; + + await runtime.attach(); + await runtime.connect(); + + const blob1 = await createBlobAndGetIds("blob1"); + const blob2 = await createBlobAndGetIds("blob2"); + + // Delete blob1's local id. The local id and the storage id should both be deleted from the redirect table + // since the blob only had one reference. + runtime.blobManager.deleteSweepReadyNodes([blob1.localGCNodeId]); + assert(!redirectTable.has(blob1.localId)); + assert(!redirectTable.has(blob1.storageId)); + + // Delete blob2's local id. The local id and the storage id should both be deleted from the redirect table + // since the blob only had one reference. + runtime.blobManager.deleteSweepReadyNodes([blob2.localGCNodeId]); + assert(!redirectTable.has(blob2.localId)); + assert(!redirectTable.has(blob2.storageId)); + }); + + it("deletes unused de-duped blobs", async () => { + await runtime.attach(); + await runtime.connect(); + + // Create 2 blobs with the same content. They should get de-duped. + const blob1 = await createBlobAndGetIds("blob1"); + const blob1Duplicate = await createBlobAndGetIds("blob1"); + assert(blob1.storageId === blob1Duplicate.storageId, "blob1 not de-duped"); + + // Create another 2 blobs with the same content. They should get de-duped. + const blob2 = await createBlobAndGetIds("blob2"); + const blob2Duplicate = await createBlobAndGetIds("blob2"); + assert(blob2.storageId === blob2Duplicate.storageId, "blob2 not de-duped"); + + // Delete blob1's local id. The local id should both be deleted from the redirect table but the storage id + // should not because the blob has another referenced from the de-duped blob. + runtime.blobManager.deleteSweepReadyNodes([blob1.localGCNodeId]); + assert(!redirectTable.has(blob1.localId), "blob1 localId should have been deleted"); + assert( + redirectTable.has(blob1.storageId), + "blob1 storageId should not have been deleted", + ); + // Delete blob1's de-duped local id. The local id and the storage id should both be deleted from the redirect table + // since all the references for the blob are now deleted. + runtime.blobManager.deleteSweepReadyNodes([blob1Duplicate.localGCNodeId]); + assert( + !redirectTable.has(blob1Duplicate.localId), + "blob1Duplicate localId should have been deleted", + ); + assert( + !redirectTable.has(blob1.storageId), + "blob1 storageId should have been deleted", + ); + + // Delete blob2's local id. The local id should both be deleted from the redirect table but the storage id + // should not because the blob has another referenced from the de-duped blob. + runtime.blobManager.deleteSweepReadyNodes([blob2.localGCNodeId]); + assert(!redirectTable.has(blob2.localId), "blob2 localId should have been deleted"); + assert( + redirectTable.has(blob2.storageId), + "blob2 storageId should not have been deleted", + ); + // Delete blob2's de-duped local id. The local id and the storage id should both be deleted from the redirect table + // since all the references for the blob are now deleted. + runtime.blobManager.deleteSweepReadyNodes([blob2Duplicate.localGCNodeId]); + assert( + !redirectTable.has(blob2Duplicate.localId), + "blob2Duplicate localId should have been deleted", + ); + assert( + !redirectTable.has(blob2.storageId), + "blob2 storageId should have been deleted", + ); + }); + }); + }); +} diff --git a/packages/runtime/container-runtime/src/test/blobs/blobHandles.spec.ts b/packages/runtime/container-runtime/src/test/blobs/blobHandles.spec.ts index 5c89d38b1d09..37c48e2e007d 100644 --- a/packages/runtime/container-runtime/src/test/blobs/blobHandles.spec.ts +++ b/packages/runtime/container-runtime/src/test/blobs/blobHandles.spec.ts @@ -5,9 +5,11 @@ import { strict as assert } from "node:assert"; -import { AttachState } from "@fluidframework/container-definitions/internal"; +import { + AttachState, + type IContainerStorageService, +} from "@fluidframework/container-definitions/internal"; import { Deferred } from "@fluidframework/core-utils/internal"; -import type { IRuntimeStorageService } from "@fluidframework/runtime-definitions/internal"; import { createChildLogger } from "@fluidframework/telemetry-utils/internal"; import { BlobManager, type IBlobManagerRuntime } from "../../blobManager/index.js"; @@ -81,7 +83,7 @@ describe("BlobHandles", () => { pendingBlobs: {}, localIdGenerator: () => "localId", isBlobDeleted: () => false, - storage: failProxy({ + storage: failProxy>({ createBlob: async () => { return { id: "blobId" }; }, @@ -120,7 +122,7 @@ describe("BlobHandles", () => { }, pendingBlobs: {}, localIdGenerator: () => "localId", - storage: failProxy({ + storage: failProxy>({ createBlob: async () => { count++; return { id: "blobId", minTTLInSeconds: count < 3 ? -1 : undefined }; diff --git a/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts b/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts index c2ed646ab865..64f2a12d0eb3 100644 --- a/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts +++ b/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts @@ -18,6 +18,7 @@ import { ContainerErrorTypes, type IContainerContext, type IBatchMessage, + type IContainerStorageService, } from "@fluidframework/container-definitions/internal"; import type { IContainerRuntime } from "@fluidframework/container-runtime-definitions/internal"; import type { @@ -242,7 +243,7 @@ describe("Runtime", () => { const mockClientId = "mockClientId"; // Mock the storage layer so "submitSummary" works. - const defaultMockStorage: Partial = { + const defaultMockStorage: Partial = { uploadSummaryWithContext: async (summary: ISummaryTree, context: ISummaryContext) => { return "fakeHandle"; }, @@ -251,7 +252,7 @@ describe("Runtime", () => { params: { settings?: Record; logger?: ITelemetryBaseLogger; - mockStorage?: Partial; + mockStorage?: Partial; loadedFromVersion?: IVersion; baseSnapshot?: ISnapshotTree; connected?: boolean; @@ -298,7 +299,7 @@ describe("Runtime", () => { }, clientId, connected, - storage: mockStorage as IRuntimeStorageService, + storage: mockStorage as IContainerStorageService, baseSnapshot, } satisfies Partial; diff --git a/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.alpha.api.md b/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.alpha.api.md index 70869315eb85..662f01cfa35d 100644 --- a/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.alpha.api.md +++ b/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.alpha.api.md @@ -309,25 +309,7 @@ export interface IRuntimeMessagesContent { // @beta @legacy export interface IRuntimeStorageService { - // @deprecated (undocumented) - createBlob(file: ArrayBufferLike): Promise; - // @deprecated - dispose?(error?: Error): void; - // @deprecated - readonly disposed?: boolean; - // @deprecated (undocumented) - downloadSummary(handle: ISummaryHandle): Promise; - // @deprecated (undocumented) - getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions): Promise; - // @deprecated (undocumented) - getSnapshotTree(version?: IVersion, scenarioName?: string): Promise; - // @deprecated (undocumented) - getVersions(versionId: string | null, count: number, scenarioName?: string, fetchSource?: FetchSource): Promise; - // @deprecated (undocumented) - readonly policies?: IDocumentStorageServicePolicies | undefined; readBlob(id: string): Promise; - // @deprecated (undocumented) - uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise; } // @beta @legacy diff --git a/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.beta.api.md b/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.beta.api.md index e6825e857073..4b163d30f6da 100644 --- a/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.beta.api.md +++ b/packages/runtime/runtime-definitions/api-report/runtime-definitions.legacy.beta.api.md @@ -300,25 +300,7 @@ export interface IRuntimeMessagesContent { // @beta @legacy export interface IRuntimeStorageService { - // @deprecated (undocumented) - createBlob(file: ArrayBufferLike): Promise; - // @deprecated - dispose?(error?: Error): void; - // @deprecated - readonly disposed?: boolean; - // @deprecated (undocumented) - downloadSummary(handle: ISummaryHandle): Promise; - // @deprecated (undocumented) - getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions): Promise; - // @deprecated (undocumented) - getSnapshotTree(version?: IVersion, scenarioName?: string): Promise; - // @deprecated (undocumented) - getVersions(versionId: string | null, count: number, scenarioName?: string, fetchSource?: FetchSource): Promise; - // @deprecated (undocumented) - readonly policies?: IDocumentStorageServicePolicies | undefined; readBlob(id: string): Promise; - // @deprecated (undocumented) - uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise; } // @beta @legacy diff --git a/packages/runtime/runtime-definitions/package.json b/packages/runtime/runtime-definitions/package.json index e9b3b3ff0712..1413a4c95202 100644 --- a/packages/runtime/runtime-definitions/package.json +++ b/packages/runtime/runtime-definitions/package.json @@ -119,7 +119,20 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Interface_IFluidDataStoreContext": { + "backCompat": false + }, + "Interface_IFluidDataStoreContextDetached": { + "backCompat": false + }, + "Interface_IFluidParentContext": { + "backCompat": false + }, + "Interface_IRuntimeStorageService": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/runtime/runtime-definitions/src/protocol.ts b/packages/runtime/runtime-definitions/src/protocol.ts index 027bf72ee633..101ac78289fd 100644 --- a/packages/runtime/runtime-definitions/src/protocol.ts +++ b/packages/runtime/runtime-definitions/src/protocol.ts @@ -8,16 +8,6 @@ import type { ITree, ISignalMessage, ISequencedDocumentMessage, - IDocumentStorageServicePolicies, - IVersion, - ISnapshotTree, - ISnapshotFetchOptions, - ISnapshot, - FetchSource, - ICreateBlobResponse, - ISummaryTree, - ISummaryHandle, - ISummaryContext, } from "@fluidframework/driver-definitions/internal"; /** @@ -142,73 +132,4 @@ export interface IRuntimeStorageService { * Reads the object with the given ID, returns content in arrayBufferLike */ readBlob(id: string): Promise; - - /** - * Whether or not the object has been disposed. - * If true, the object should be considered invalid, and its other state should be disregarded. - * - * @deprecated - This API is deprecated and will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - readonly disposed?: boolean; - - /** - * Dispose of the object and its resources. - * @param error - Optional error indicating the reason for the disposal, if the object was - * disposed as the result of an error. - * - * @deprecated - This API is deprecated and will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - dispose?(error?: Error): void; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - readonly policies?: IDocumentStorageServicePolicies | undefined; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - // eslint-disable-next-line @rushstack/no-new-null - getSnapshotTree(version?: IVersion, scenarioName?: string): Promise; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions): Promise; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - getVersions( - // TODO: use `undefined` instead. - // eslint-disable-next-line @rushstack/no-new-null - versionId: string | null, - count: number, - scenarioName?: string, - fetchSource?: FetchSource, - ): Promise; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - createBlob(file: ArrayBufferLike): Promise; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise; - - /** - * @deprecated - This will be removed in a future release. No replacement is planned as - * it is unused in the DataStore layer. - */ - downloadSummary(handle: ISummaryHandle): Promise; } diff --git a/packages/runtime/runtime-definitions/src/test/types/validateRuntimeDefinitionsPrevious.generated.ts b/packages/runtime/runtime-definitions/src/test/types/validateRuntimeDefinitionsPrevious.generated.ts index b4d0161a7566..ed50eb7205da 100644 --- a/packages/runtime/runtime-definitions/src/test/types/validateRuntimeDefinitionsPrevious.generated.ts +++ b/packages/runtime/runtime-definitions/src/test/types/validateRuntimeDefinitionsPrevious.generated.ts @@ -229,6 +229,7 @@ declare type old_as_current_for_Interface_IFluidDataStoreContext = requireAssign * typeValidation.broken: * "Interface_IFluidDataStoreContext": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IFluidDataStoreContext = requireAssignableTo, TypeOnly> /* @@ -247,6 +248,7 @@ declare type old_as_current_for_Interface_IFluidDataStoreContextDetached = requi * typeValidation.broken: * "Interface_IFluidDataStoreContextDetached": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IFluidDataStoreContextDetached = requireAssignableTo, TypeOnly> /* @@ -319,6 +321,7 @@ declare type old_as_current_for_Interface_IFluidParentContext = requireAssignabl * typeValidation.broken: * "Interface_IFluidParentContext": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IFluidParentContext = requireAssignableTo, TypeOnly> /* @@ -445,6 +448,7 @@ declare type old_as_current_for_Interface_IRuntimeStorageService = requireAssign * typeValidation.broken: * "Interface_IRuntimeStorageService": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IRuntimeStorageService = requireAssignableTo, TypeOnly> /* diff --git a/packages/runtime/test-runtime-utils/package.json b/packages/runtime/test-runtime-utils/package.json index db0e8beb871f..d9915b8bb164 100644 --- a/packages/runtime/test-runtime-utils/package.json +++ b/packages/runtime/test-runtime-utils/package.json @@ -153,7 +153,14 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Class_MockFluidDataStoreContext": { + "backCompat": false + }, + "ClassStatics_MockFluidDataStoreContext": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/runtime/test-runtime-utils/src/test/types/validateTestRuntimeUtilsPrevious.generated.ts b/packages/runtime/test-runtime-utils/src/test/types/validateTestRuntimeUtilsPrevious.generated.ts index 60347db4f053..a88ba30aba20 100644 --- a/packages/runtime/test-runtime-utils/src/test/types/validateTestRuntimeUtilsPrevious.generated.ts +++ b/packages/runtime/test-runtime-utils/src/test/types/validateTestRuntimeUtilsPrevious.generated.ts @@ -175,6 +175,7 @@ declare type old_as_current_for_Class_MockFluidDataStoreContext = requireAssigna * typeValidation.broken: * "Class_MockFluidDataStoreContext": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Class_MockFluidDataStoreContext = requireAssignableTo, TypeOnly> /* @@ -364,6 +365,7 @@ declare type current_as_old_for_ClassStatics_MockDeltaQueue = requireAssignableT * typeValidation.broken: * "ClassStatics_MockFluidDataStoreContext": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_ClassStatics_MockFluidDataStoreContext = requireAssignableTo, TypeOnly> /* diff --git a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index 40e01889f96b..b2a5436509a3 100644 --- a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -10,6 +10,7 @@ import { describeCompat } from "@fluid-private/test-version-utils"; import type { ISharedCell } from "@fluidframework/cell/internal"; import { IContainer, IFluidCodeDetails } from "@fluidframework/container-definitions/internal"; import { Loader } from "@fluidframework/container-loader/internal"; +import { IContainerRuntime } from "@fluidframework/container-runtime-definitions/internal"; import { IFluidHandle, IRequest } from "@fluidframework/core-interfaces"; import type { SharedCounter } from "@fluidframework/counter/internal"; import { ISummaryTree, SummaryType } from "@fluidframework/driver-definitions"; @@ -542,9 +543,11 @@ describeCompat( "Storage should be present in detached data store", ); let success1: boolean | undefined; - await defaultDataStore.context.storage.getSnapshotTree(undefined).catch((err) => { - success1 = false; - }); + await (defaultDataStore.context.containerRuntime as IContainerRuntime).storage + .getSnapshotTree(undefined) + .catch((err) => { + success1 = false; + }); assert( success1 === false, "Snapshot fetch should not be allowed in detached data store", @@ -559,9 +562,11 @@ describeCompat( "Storage should be present in rehydrated data store", ); let success2: boolean | undefined; - await defaultDataStore2.context.storage.getSnapshotTree(undefined).catch((err) => { - success2 = false; - }); + await (defaultDataStore2.context.containerRuntime as IContainerRuntime).storage + .getSnapshotTree(undefined) + .catch((err) => { + success2 = false; + }); assert( success2 === false, "Snapshot fetch should not be allowed in rehydrated data store", diff --git a/packages/test/test-utils/package.json b/packages/test/test-utils/package.json index 3cbcc1c35fb4..7777004d1fee 100644 --- a/packages/test/test-utils/package.json +++ b/packages/test/test-utils/package.json @@ -162,7 +162,14 @@ "typescript": "~5.4.5" }, "typeValidation": { - "broken": {}, + "broken": { + "Interface_IProvideTestFluidObject": { + "backCompat": false + }, + "Interface_ITestFluidObject": { + "backCompat": false + } + }, "entrypoint": "legacy" } } diff --git a/packages/test/test-utils/src/test/types/validateTestUtilsPrevious.generated.ts b/packages/test/test-utils/src/test/types/validateTestUtilsPrevious.generated.ts index 23bb8b2365d0..d14180f98964 100644 --- a/packages/test/test-utils/src/test/types/validateTestUtilsPrevious.generated.ts +++ b/packages/test/test-utils/src/test/types/validateTestUtilsPrevious.generated.ts @@ -85,6 +85,7 @@ declare type old_as_current_for_Interface_IProvideTestFluidObject = requireAssig * typeValidation.broken: * "Interface_IProvideTestFluidObject": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_IProvideTestFluidObject = requireAssignableTo, TypeOnly> /* @@ -103,4 +104,5 @@ declare type old_as_current_for_Interface_ITestFluidObject = requireAssignableTo * typeValidation.broken: * "Interface_ITestFluidObject": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_ITestFluidObject = requireAssignableTo, TypeOnly> From afb8f1c53cdcd748d8dd39d881d76631e481bec5 Mon Sep 17 00:00:00 2001 From: Navin Agarwal Date: Wed, 15 Oct 2025 11:35:45 -0700 Subject: [PATCH 2/6] Remove test that is not needed anymore --- .../test/deRehydrateContainerTests.spec.ts | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index b2a5436509a3..c3c61c54f332 100644 --- a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -10,7 +10,6 @@ import { describeCompat } from "@fluid-private/test-version-utils"; import type { ISharedCell } from "@fluidframework/cell/internal"; import { IContainer, IFluidCodeDetails } from "@fluidframework/container-definitions/internal"; import { Loader } from "@fluidframework/container-loader/internal"; -import { IContainerRuntime } from "@fluidframework/container-runtime-definitions/internal"; import { IFluidHandle, IRequest } from "@fluidframework/core-interfaces"; import type { SharedCounter } from "@fluidframework/counter/internal"; import { ISummaryTree, SummaryType } from "@fluidframework/driver-definitions"; @@ -532,47 +531,6 @@ describeCompat( assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); }); - it("Storage in detached container", async () => { - const { container } = await createDetachedContainerAndGetEntryPoint(); - - const snapshotTree = container.serialize(); - const defaultDataStore = - await getContainerEntryPointBackCompat(container); - assert( - defaultDataStore.context.storage !== undefined, - "Storage should be present in detached data store", - ); - let success1: boolean | undefined; - await (defaultDataStore.context.containerRuntime as IContainerRuntime).storage - .getSnapshotTree(undefined) - .catch((err) => { - success1 = false; - }); - assert( - success1 === false, - "Snapshot fetch should not be allowed in detached data store", - ); - - const container2: IContainer = - await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); - const defaultDataStore2 = - await getContainerEntryPointBackCompat(container2); - assert( - defaultDataStore2.context.storage !== undefined, - "Storage should be present in rehydrated data store", - ); - let success2: boolean | undefined; - await (defaultDataStore2.context.containerRuntime as IContainerRuntime).storage - .getSnapshotTree(undefined) - .catch((err) => { - success2 = false; - }); - assert( - success2 === false, - "Snapshot fetch should not be allowed in rehydrated data store", - ); - }); - it("Change contents of dds, then rehydrate and then check summary", async () => { const { container } = await createDetachedContainerAndGetEntryPoint(); From 2234b95722ab5f88dea0cbccfbb17f14e165307a Mon Sep 17 00:00:00 2001 From: Navin Agarwal Date: Wed, 15 Oct 2025 12:44:12 -0700 Subject: [PATCH 3/6] Merge with main latest --- .../src/test/blobManager.spec.ts | 1278 ----------------- ...idateContainerRuntimePrevious.generated.ts | 1 + 2 files changed, 1 insertion(+), 1278 deletions(-) delete mode 100644 packages/runtime/container-runtime/src/test/blobManager.spec.ts diff --git a/packages/runtime/container-runtime/src/test/blobManager.spec.ts b/packages/runtime/container-runtime/src/test/blobManager.spec.ts deleted file mode 100644 index 21f325d8b2e3..000000000000 --- a/packages/runtime/container-runtime/src/test/blobManager.spec.ts +++ /dev/null @@ -1,1278 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -import { strict as assert } from "node:assert"; - -import { - IsoBuffer, - TypedEventEmitter, - bufferToString, - gitHashFile, -} from "@fluid-internal/client-utils"; -import { - AttachState, - type IContainerStorageService, -} from "@fluidframework/container-definitions/internal"; -import type { IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions/internal"; -import type { - ConfigTypes, - IConfigProviderBase, - IErrorBase, -} from "@fluidframework/core-interfaces"; -import type { - IFluidHandleContext, - IFluidHandleInternal, -} from "@fluidframework/core-interfaces/internal"; -import { Deferred } from "@fluidframework/core-utils/internal"; -import { type IClientDetails, SummaryType } from "@fluidframework/driver-definitions"; -import type { ISequencedMessageEnvelope } from "@fluidframework/runtime-definitions/internal"; -import { - isFluidHandleInternalPayloadPending, - isFluidHandlePayloadPending, - isLocalFluidHandle, -} from "@fluidframework/runtime-utils/internal"; -import { - LoggingError, - MockLogger, - type MonitoringContext, - createChildLogger, - mixinMonitoringContext, - type ITelemetryLoggerExt, -} from "@fluidframework/telemetry-utils/internal"; -import Sinon from "sinon"; -import { v4 as uuid } from "uuid"; - -import { - BlobManager, - type IBlobManagerLoadInfo, - type IBlobManagerRuntime, - blobManagerBasePath, - redirectTableBlobName, - type IPendingBlobs, -} from "../blobManager/index.js"; - -const MIN_TTL = 24 * 60 * 60; // same as ODSP -abstract class BaseMockBlobStorage - implements Pick -{ - public blobs: Map = new Map(); - public abstract createBlob(blob: ArrayBufferLike); - public async readBlob(id: string) { - const blob = this.blobs.get(id); - assert(!!blob); - return blob; - } -} - -class DedupeStorage extends BaseMockBlobStorage { - public minTTL: number = MIN_TTL; - - public async createBlob(blob: ArrayBufferLike) { - const s = bufferToString(blob, "base64"); - const id = await gitHashFile(IsoBuffer.from(s, "base64")); - this.blobs.set(id, blob); - return { id, minTTLInSeconds: this.minTTL }; - } -} - -class NonDedupeStorage extends BaseMockBlobStorage { - public async createBlob(blob: ArrayBufferLike) { - const id = this.blobs.size.toString(); - this.blobs.set(id, blob); - return { id, minTTLInSeconds: MIN_TTL }; - } -} - -export class MockRuntime - extends TypedEventEmitter - implements IBlobManagerRuntime -{ - public readonly clientDetails: IClientDetails = { capabilities: { interactive: true } }; - constructor( - public mc: MonitoringContext, - createBlobPayloadPending: boolean, - blobManagerLoadInfo: IBlobManagerLoadInfo = {}, - attached = false, - stashed: unknown[] = [[], {}], - ) { - super(); - this.attachState = attached ? AttachState.Attached : AttachState.Detached; - this.ops = stashed[0] as unknown[]; - this.baseLogger = mc.logger; - this.blobManager = new BlobManager({ - routeContext: undefined as unknown as IFluidHandleContext, - blobManagerLoadInfo, - storage: this.getStorage(), - sendBlobAttachOp: (localId: string, blobId?: string) => - this.sendBlobAttachOp(localId, blobId), - blobRequested: () => undefined, - isBlobDeleted: (blobPath: string) => this.isBlobDeleted(blobPath), - runtime: this, - stashedBlobs: stashed[1] as IPendingBlobs | undefined, - createBlobPayloadPending, - }); - } - - public disposed: boolean = false; - - public get storage(): IContainerStorageService { - return (this.attachState === AttachState.Detached - ? this.detachedStorage - : this.attachedStorage) as unknown as IContainerStorageService; - } - - private processing = false; - public unprocessedBlobs = new Set(); - - public getStorage(): IContainerStorageService { - return { - createBlob: async (blob: ArrayBufferLike) => { - if (this.processing) { - return this.storage.createBlob(blob); - } - const P = this.processBlobsP.promise.then(async () => { - if (!this.connected && this.attachState === AttachState.Attached) { - this.unprocessedBlobs.delete(blob); - throw new Error("fake error due to having no connection to storage service"); - } else { - this.unprocessedBlobs.delete(blob); - return this.storage.createBlob(blob); - } - }); - this.unprocessedBlobs.add(blob); - this.emit("blob"); - this.blobPs.push(P); - return P; - }, - readBlob: async (id: string) => this.storage.readBlob(id), - } as unknown as IContainerStorageService; - } - - public sendBlobAttachOp(localId: string, blobId?: string): void { - this.ops.push({ metadata: { localId, blobId } }); - } - - public async createBlob( - blob: ArrayBufferLike, - signal?: AbortSignal, - ): Promise> { - const P = this.blobManager.createBlob(blob, signal); - this.handlePs.push(P); - return P; - } - - public async getBlob( - blobHandle: IFluidHandleInternal, - ): Promise { - const pathParts = blobHandle.absolutePath.split("/"); - const blobId = pathParts[2]; - const payloadPending = isFluidHandleInternalPayloadPending(blobHandle) - ? blobHandle.payloadPending - : false; - return this.blobManager.getBlob(blobId, payloadPending); - } - - public async getPendingLocalState(): Promise<(unknown[] | IPendingBlobs | undefined)[]> { - const pendingBlobs = await this.blobManager.attachAndGetPendingBlobs(); - return [[...this.ops], pendingBlobs]; - } - - public blobManager: BlobManager; - public connected = false; - public closed = false; - public attachState: AttachState; - public attachedStorage = new DedupeStorage(); - public detachedStorage = new NonDedupeStorage(); - public baseLogger: ITelemetryLoggerExt; - - private ops: unknown[] = []; - private processBlobsP = new Deferred(); - private blobPs: Promise[] = []; - private handlePs: Promise[] = []; - private readonly deletedBlobs: string[] = []; - - public processOps(): void { - assert(this.connected || this.ops.length === 0); - for (const op of this.ops) { - this.blobManager.processBlobAttachMessage(op as ISequencedMessageEnvelope, true); - } - this.ops = []; - } - - public async processBlobs( - resolve: boolean, - canRetry: boolean = false, - retryAfterSeconds?: number, - ): Promise { - const blobPs = this.blobPs; - this.blobPs = []; - if (resolve) { - this.processBlobsP.resolve(); - } else { - this.processBlobsP.reject( - new LoggingError("fake driver error", { canRetry, retryAfterSeconds }), - ); - } - this.processBlobsP = new Deferred(); - await Promise.allSettled(blobPs).catch(() => {}); - } - - public async processHandles(): Promise { - const handlePs = this.handlePs; - this.handlePs = []; - const handles = (await Promise.all(handlePs)) as IFluidHandleInternal[]; - for (const handle of handles) { - handle.attachGraph(); - } - } - - public async processAll(): Promise { - while (this.blobPs.length + this.handlePs.length + this.ops.length > 0) { - const p1 = this.processBlobs(true); - const p2 = this.processHandles(); - this.processOps(); - await Promise.race([p1, p2]); - this.processOps(); - await Promise.all([p1, p2]); - } - } - - public async attach(): Promise<{ - ids: string[]; - redirectTable: [string, string][] | undefined; - }> { - if (this.detachedStorage.blobs.size > 0) { - const table = new Map(); - for (const [detachedId, blob] of this.detachedStorage.blobs) { - const { id } = await this.attachedStorage.createBlob(blob); - table.set(detachedId, id); - } - this.detachedStorage.blobs.clear(); - this.blobManager.setRedirectTable(table); - } - const summary = validateSummary(this); - this.attachState = AttachState.Attached; - this.emit("attached"); - return summary; - } - - public async connect(delay = 0, processStashedWithRetry?: boolean): Promise { - assert(!this.connected); - await new Promise((resolve) => setTimeout(resolve, delay)); - this.connected = true; - this.emit("connected", "client ID"); - await this.processStashed(processStashedWithRetry); - const ops = this.ops; - this.ops = []; - for (const op of ops) { - // TODO: better typing - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access - this.blobManager.reSubmit((op as any).metadata as Record | undefined); - } - } - - public async processStashed(processStashedWithRetry?: boolean): Promise { - // const uploadP = this.blobManager.stashedBlobsUploadP; - this.processing = true; - if (processStashedWithRetry) { - await this.processBlobs(false, false, 0); - // wait till next retry - await new Promise((resolve) => setTimeout(resolve, 1)); - // try again successfully - await this.processBlobs(true); - } else { - await this.processBlobs(true); - } - // await uploadP; - this.processing = false; - } - - public disconnect(): void { - assert(this.connected); - this.connected = false; - this.emit("disconnected"); - } - - public async remoteUpload( - blob: ArrayBufferLike, - ): Promise<{ metadata: { localId: string; blobId: string } }> { - const response = await this.storage.createBlob(blob); - const op = { metadata: { localId: uuid(), blobId: response.id } }; - this.blobManager.processBlobAttachMessage(op as ISequencedMessageEnvelope, false); - return op; - } - - public deleteBlob(blobHandle: IFluidHandleInternal): void { - this.deletedBlobs.push(blobHandle.absolutePath); - } - - public isBlobDeleted(blobPath: string): boolean { - return this.deletedBlobs.includes(blobPath); - } -} - -export const validateSummary = ( - runtime: MockRuntime, -): { - ids: string[]; - redirectTable: [string, string][] | undefined; -} => { - const summary = runtime.blobManager.summarize(); - const ids: string[] = []; - let redirectTable: [string, string][] | undefined; - for (const [key, attachment] of Object.entries(summary.summary.tree)) { - if (attachment.type === SummaryType.Attachment) { - ids.push(attachment.id); - } else { - assert.strictEqual(key, redirectTableBlobName); - assert(attachment.type === SummaryType.Blob); - assert(typeof attachment.content === "string"); - redirectTable = [ - ...new Map( - JSON.parse(attachment.content) as [string, string][], - ).entries(), - ]; - } - } - return { ids, redirectTable }; -}; - -for (const createBlobPayloadPending of [false, true]) { - describe(`BlobManager (pending payloads): ${createBlobPayloadPending}`, () => { - const mockLogger = new MockLogger(); - let runtime: MockRuntime; - let createBlob: (blob: ArrayBufferLike, signal?: AbortSignal) => Promise; - let waitForBlob: (blob: ArrayBufferLike) => Promise; - let mc: MonitoringContext; - let injectedSettings: Record = {}; - - beforeEach(() => { - const configProvider = (settings: Record): IConfigProviderBase => ({ - getRawConfig: (name: string): ConfigTypes => settings[name], - }); - mc = mixinMonitoringContext( - createChildLogger({ logger: mockLogger }), - configProvider(injectedSettings), - ); - runtime = new MockRuntime(mc, createBlobPayloadPending); - - // ensures this blob will be processed next time runtime.processBlobs() is called - waitForBlob = async (blob) => { - if (!runtime.unprocessedBlobs.has(blob)) { - await new Promise((resolve) => - runtime.on("blob", () => { - if (runtime.unprocessedBlobs.has(blob)) { - resolve(); - } - }), - ); - } - }; - - // create blob and await the handle after the test - createBlob = async (blob: ArrayBufferLike, signal?: AbortSignal) => { - runtime - .createBlob(blob, signal) - .then((handle) => { - if (createBlobPayloadPending) { - handle.attachGraph(); - } - return handle; - }) - // Suppress errors here, we expect them to be detected elsewhere - .catch(() => {}); - await waitForBlob(blob); - }; - - const onNoPendingBlobs = () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access -- Accessing private property - assert((runtime.blobManager as any).pendingBlobs.size === 0); - }; - - runtime.blobManager.events.on("noPendingBlobs", () => onNoPendingBlobs()); - }); - - afterEach(async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access -- Accessing private property - assert.strictEqual((runtime.blobManager as any).pendingBlobs.size, 0); - injectedSettings = {}; - mockLogger.clear(); - }); - - it("empty snapshot", () => { - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("non empty snapshot", async () => { - await runtime.attach(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it("hasPendingBlobs", async () => { - await runtime.attach(); - await runtime.connect(); - - assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob2", "utf8")); - assert.strictEqual(runtime.blobManager.hasPendingBlobs, true); - await runtime.processAll(); - assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 2); - assert.strictEqual(summaryData.redirectTable?.length, 2); - }); - - it("NoPendingBlobs count", async () => { - await runtime.attach(); - await runtime.connect(); - let count = 0; - runtime.blobManager.events.on("noPendingBlobs", () => count++); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - assert.strictEqual(count, 1); - await createBlob(IsoBuffer.from("blob2", "utf8")); - await createBlob(IsoBuffer.from("blob3", "utf8")); - await runtime.processAll(); - assert.strictEqual(count, 2); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 3); - assert.strictEqual(summaryData.redirectTable?.length, 3); - }); - - it("detached snapshot", async () => { - assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - assert.strictEqual(runtime.blobManager.hasPendingBlobs, true); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("detached->attached snapshot", async () => { - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - assert.strictEqual(runtime.blobManager.hasPendingBlobs, true); - await runtime.attach(); - assert.strictEqual(runtime.blobManager.hasPendingBlobs, false); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it("uploads while disconnected", async () => { - await runtime.attach(); - const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.connect(); - await runtime.processAll(); - await assert.doesNotReject(handleP); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it("reupload blob if expired", async () => { - await runtime.attach(); - await runtime.connect(); - runtime.attachedStorage.minTTL = 0.001; // force expired TTL being less than connection time (50ms) - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processBlobs(true); - runtime.disconnect(); - await new Promise((resolve) => setTimeout(resolve, 50)); - await runtime.connect(); - await runtime.processAll(); - }); - - it("completes after disconnection while upload pending", async () => { - await runtime.attach(); - await runtime.connect(); - - const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); - runtime.disconnect(); - await runtime.connect(10); // adding some delay to reconnection - await runtime.processAll(); - await assert.doesNotReject(handleP); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it("upload fails gracefully", async () => { - await runtime.attach(); - await runtime.connect(); - - if (createBlobPayloadPending) { - const handle = await runtime.createBlob(IsoBuffer.from("blob", "utf8")); - assert.strict(isFluidHandlePayloadPending(handle)); - assert.strict(isLocalFluidHandle(handle)); - assert.strictEqual( - handle.payloadState, - "pending", - "Handle should be in pending state", - ); - assert.strictEqual( - handle.payloadShareError, - undefined, - "handle should not have an error yet", - ); - let failed = false; - const onPayloadShareFailed = (error: unknown): void => { - failed = true; - assert.strictEqual( - (error as Error).message, - "fake driver error", - "Did not receive the expected error", - ); - handle.events.off("payloadShareFailed", onPayloadShareFailed); - }; - handle.events.on("payloadShareFailed", onPayloadShareFailed); - await runtime.processHandles(); - await runtime.processBlobs(false); - runtime.processOps(); - assert.strict(failed, "should fail"); - assert.strictEqual( - handle.payloadState, - "pending", - "Handle should still be in pending state", - ); - assert.strictEqual( - (handle.payloadShareError as unknown as Error).message, - "fake driver error", - "Handle did not have the expected error", - ); - } else { - // If the blobs are created without pending payloads, we don't get to see the handle at - // all so we can't inspect its state. - const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processBlobs(false); - runtime.processOps(); - try { - await handleP; - assert.fail("should fail"); - } catch (error: unknown) { - assert.strictEqual((error as Error).message, "fake driver error"); - } - await assert.rejects(handleP); - } - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("updates handle state after success", async () => { - await runtime.attach(); - await runtime.connect(); - - if (createBlobPayloadPending) { - const handle = await runtime.createBlob(IsoBuffer.from("blob", "utf8")); - assert.strict(isFluidHandlePayloadPending(handle)); - assert.strictEqual( - handle.payloadState, - "pending", - "Handle should be in pending state", - ); - let shared = false; - const onPayloadShared = (): void => { - shared = true; - handle.events.off("payloadShared", onPayloadShared); - }; - handle.events.on("payloadShared", onPayloadShared); - await runtime.processHandles(); - await runtime.processBlobs(true); - runtime.processOps(); - assert.strict(shared, "should become shared"); - assert.strictEqual(handle.payloadState, "shared", "Handle should be in shared state"); - } else { - // Without placeholder blobs, we don't get to see the handle before it reaches "shared" state - // but we can still verify it's in the expected state when we get it. - const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - const handle = await handleP; - assert.strict(isFluidHandlePayloadPending(handle)); - assert.strictEqual(handle.payloadState, "shared", "Handle should be in shared state"); - } - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it.skip("upload fails and retries for retriable errors", async () => { - // Needs to use some sort of fake timer or write test in a different way as it is waiting - // for actual time which is causing timeouts. - await runtime.attach(); - await runtime.connect(); - const handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processBlobs(false, true, 0); - // wait till next retry - await new Promise((resolve) => setTimeout(resolve, 1)); - // try again successfully - await runtime.processBlobs(true); - runtime.processOps(); - await runtime.processHandles(); - assert(handleP); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it("completes after disconnection while op in flight", async () => { - await runtime.attach(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processBlobs(true); - - runtime.disconnect(); - await runtime.connect(); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 2); - }); - - it("multiple disconnect/connects", async () => { - await runtime.attach(); - await runtime.connect(); - - const blob = IsoBuffer.from("blob", "utf8"); - const handleP = runtime.createBlob(blob); - runtime.disconnect(); - await runtime.connect(10); - - const blob2 = IsoBuffer.from("blob2", "utf8"); - const handleP2 = runtime.createBlob(blob2); - runtime.disconnect(); - - await runtime.connect(10); - await runtime.processAll(); - await assert.doesNotReject(handleP); - await assert.doesNotReject(handleP2); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 2); - assert.strictEqual(summaryData.redirectTable?.length, 2); - }); - - it("handles deduped IDs", async () => { - await runtime.attach(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - runtime.disconnect(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processBlobs(true); - - runtime.disconnect(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 6); - }); - - it("handles deduped IDs in detached", async () => { - runtime.detachedStorage = new DedupeStorage(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("handles deduped IDs in detached->attached", async () => { - runtime.detachedStorage = new DedupeStorage(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - await runtime.attach(); - await runtime.connect(); - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - - runtime.disconnect(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await createBlob(IsoBuffer.from("blob", "utf8")); - - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 4); - }); - - it("can load from summary", async () => { - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - await runtime.attach(); - const handle = runtime.createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.connect(); - - await runtime.processAll(); - await assert.doesNotReject(handle); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 3); - - const runtime2 = new MockRuntime(mc, createBlobPayloadPending, summaryData, true); - const summaryData2 = validateSummary(runtime2); - assert.strictEqual(summaryData2.ids.length, 1); - assert.strictEqual(summaryData2.redirectTable?.length, 3); - }); - - it("handles duplicate remote upload", async () => { - await runtime.attach(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.remoteUpload(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 2); - }); - - it("handles duplicate remote upload between upload and op", async () => { - await runtime.attach(); - await runtime.connect(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.processBlobs(true); - await runtime.remoteUpload(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 2); - }); - - it("handles duplicate remote upload with local ID", async () => { - await runtime.attach(); - - await createBlob(IsoBuffer.from("blob", "utf8")); - await runtime.connect(); - await runtime.processBlobs(true); - await runtime.remoteUpload(IsoBuffer.from("blob", "utf8")); - await runtime.processAll(); - - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 2); - }); - - it("includes blob IDs in summary while attaching", async () => { - await createBlob(IsoBuffer.from("blob1", "utf8")); - await createBlob(IsoBuffer.from("blob2", "utf8")); - await createBlob(IsoBuffer.from("blob3", "utf8")); - await runtime.processAll(); - - // While attaching with blobs, Container takes a summary while still in "Detached" - // state. BlobManager should know to include the list of attached blob - // IDs since this summary will be used to create the document - const summaryData = await runtime.attach(); - assert.strictEqual(summaryData?.ids.length, 3); - assert.strictEqual(summaryData?.redirectTable?.length, 3); - }); - - it("all blobs attached", async () => { - await runtime.attach(); - await runtime.connect(); - assert.strictEqual(runtime.blobManager.allBlobsAttached, true); - await createBlob(IsoBuffer.from("blob1", "utf8")); - // We immediately attach the handle in createBlob if pending payloads are enabled - assert.strictEqual(runtime.blobManager.allBlobsAttached, createBlobPayloadPending); - await runtime.processBlobs(true); - assert.strictEqual(runtime.blobManager.allBlobsAttached, createBlobPayloadPending); - await runtime.processAll(); - assert.strictEqual(runtime.blobManager.allBlobsAttached, true); - await createBlob(IsoBuffer.from("blob1", "utf8")); - await createBlob(IsoBuffer.from("blob2", "utf8")); - await createBlob(IsoBuffer.from("blob3", "utf8")); - assert.strictEqual(runtime.blobManager.allBlobsAttached, createBlobPayloadPending); - await runtime.processAll(); - assert.strictEqual(runtime.blobManager.allBlobsAttached, true); - }); - - it("runtime disposed during readBlob - log no error", async () => { - const someId = "someId"; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call -- Accessing private property - (runtime.blobManager as any).setRedirection(someId, undefined); // To appease an assert - - // Mock storage.readBlob to dispose the runtime and throw an error - Sinon.stub(runtime.storage, "readBlob").callsFake(async (_id: string) => { - runtime.disposed = true; - throw new Error("BOOM!"); - }); - - await assert.rejects( - async () => runtime.blobManager.getBlob(someId, false), - (e: Error) => e.message === "BOOM!", - "Expected getBlob to throw with test error message", - ); - assert(runtime.disposed, "Runtime should be disposed"); - mockLogger.assertMatchNone( - [{ category: "error" }], - "Should not have logged any errors", - undefined, - false /* clearEventsAfterCheck */, - ); - mockLogger.assertMatch( - [{ category: "generic", eventName: "BlobManager:AttachmentReadBlob_cancel" }], - "Expected the _cancel event to be logged with 'generic' category", - ); - }); - - it("waits for blobs from handles with pending payloads without error", async () => { - await runtime.attach(); - - // Part of remoteUpload, but stop short of processing the message - const response = await runtime.storage.createBlob(IsoBuffer.from("blob", "utf8")); - const op = { metadata: { localId: uuid(), blobId: response.id } }; - - await assert.rejects( - runtime.blobManager.getBlob(op.metadata.localId, false), - "Rejects when attempting to get non-existent, shared-payload blobs", - ); - - // Try to get the blob that we haven't processed the attach op for yet. - // This simulates having found this ID in a handle with a pending payload that the remote client would have sent - const blobP = runtime.blobManager.getBlob(op.metadata.localId, true); - - // Process the op as if it were arriving from the remote client, which should cause the blobP promise to resolve - runtime.blobManager.processBlobAttachMessage(op as ISequencedMessageEnvelope, false); - - // Await the promise to confirm it settles and does not reject - await blobP; - }); - - describe("Abort Signal", () => { - it("abort before upload", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - ac.abort("abort test"); - try { - const blob = IsoBuffer.from("blob", "utf8"); - await runtime.createBlob(blob, ac.signal); - assert.fail("Should not succeed"); - - // TODO: better typing - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.status, undefined); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.uploadTime, undefined); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.acked, undefined); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.message, "uploadBlob aborted"); - } - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("abort while upload", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - const blob = IsoBuffer.from("blob", "utf8"); - const handleP = runtime.createBlob(blob, ac.signal); - ac.abort("abort test"); - assert.strictEqual(runtime.unprocessedBlobs.size, 1); - await runtime.processBlobs(true); - try { - await handleP; - assert.fail("Should not succeed"); - // TODO: better typing - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.uploadTime, undefined); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.acked, false); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.message, "uploadBlob aborted"); - } - assert(handleP); - await assert.rejects(handleP); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("abort while failed upload", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - const blob = IsoBuffer.from("blob", "utf8"); - const handleP = runtime.createBlob(blob, ac.signal); - const handleP2 = runtime.createBlob(IsoBuffer.from("blob2", "utf8")); - ac.abort("abort test"); - assert.strictEqual(runtime.unprocessedBlobs.size, 2); - await runtime.processBlobs(false); - try { - await handleP; - assert.fail("Should not succeed"); - } catch (error: unknown) { - assert.strictEqual((error as Error).message, "uploadBlob aborted"); - } - try { - await handleP2; - assert.fail("Should not succeed"); - } catch (error: unknown) { - assert.strictEqual((error as Error).message, "fake driver error"); - } - await assert.rejects(handleP); - await assert.rejects(handleP2); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("abort while disconnected", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - const blob = IsoBuffer.from("blob", "utf8"); - const handleP = runtime.createBlob(blob, ac.signal); - runtime.disconnect(); - ac.abort(); - await runtime.processBlobs(true); - try { - await handleP; - assert.fail("Should not succeed"); - } catch (error: unknown) { - assert.strictEqual((error as Error).message, "uploadBlob aborted"); - } - await assert.rejects(handleP); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("abort after blob succeeds", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - let handleP: Promise> | undefined; - try { - const blob = IsoBuffer.from("blob", "utf8"); - handleP = runtime.createBlob(blob, ac.signal); - await runtime.processAll(); - ac.abort(); - } catch { - assert.fail("abort after processing should not throw"); - } - assert(handleP); - await assert.doesNotReject(handleP); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 1); - assert.strictEqual(summaryData.redirectTable?.length, 1); - }); - - it("abort while waiting for op", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - const blob = IsoBuffer.from("blob", "utf8"); - const handleP = runtime.createBlob(blob, ac.signal); - const p1 = runtime.processBlobs(true); - const p2 = runtime.processHandles(); - // finish upload - await Promise.race([p1, p2]); - ac.abort(); - runtime.processOps(); - try { - // finish op - await handleP; - assert.fail("Should not succeed"); - - // TODO: better typing - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.message, "uploadBlob aborted"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.ok(error.uploadTime); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.acked, false); - } - await assert.rejects(handleP); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - - it("resubmit on aborted pending op", async function () { - if (createBlobPayloadPending) { - // Blob creation with pending payload doesn't support abort - this.skip(); - } - await runtime.attach(); - await runtime.connect(); - const ac = new AbortController(); - let handleP: Promise> | undefined; - try { - handleP = runtime.createBlob(IsoBuffer.from("blob", "utf8"), ac.signal); - const p1 = runtime.processBlobs(true); - const p2 = runtime.processHandles(); - // finish upload - await Promise.race([p1, p2]); - runtime.disconnect(); - ac.abort(); - await handleP; - assert.fail("Should not succeed"); - // TODO: better typing - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (error: any) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.message, "uploadBlob aborted"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.ok(error.uploadTime); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - assert.strictEqual(error.acked, false); - } - await runtime.connect(); - runtime.processOps(); - - // TODO: `handleP` can be `undefined`; this should be made safer. - await assert.rejects(handleP as Promise>); - const summaryData = validateSummary(runtime); - assert.strictEqual(summaryData.ids.length, 0); - assert.strictEqual(summaryData.redirectTable, undefined); - }); - }); - - describe("Garbage Collection", () => { - let redirectTable: Map; - - /** - * Creates a blob with the given content and returns its local and storage id. - */ - async function createBlobAndGetIds(content: string) { - // For a given blob's GC node id, returns the blob id. - const getBlobIdFromGCNodeId = (gcNodeId: string) => { - const pathParts = gcNodeId.split("/"); - assert( - pathParts.length === 3 && pathParts[1] === blobManagerBasePath, - "Invalid blob node path", - ); - return pathParts[2]; - }; - - // For a given blob's id, returns the GC node id. - const getGCNodeIdFromBlobId = (blobId: string) => { - return `/${blobManagerBasePath}/${blobId}`; - }; - - const blobContents = IsoBuffer.from(content, "utf8"); - const handleP = runtime.createBlob(blobContents); - await runtime.processAll(); - - const blobHandle = await handleP; - const localId = getBlobIdFromGCNodeId(blobHandle.absolutePath); - assert(redirectTable.has(localId), "blob not found in redirect table"); - const storageId = redirectTable.get(localId); - assert(storageId !== undefined, "storage id not found in redirect table"); - return { - localId, - localGCNodeId: getGCNodeIdFromBlobId(localId), - storageId, - storageGCNodeId: getGCNodeIdFromBlobId(storageId), - }; - } - - beforeEach(() => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment -- Mutating private property - redirectTable = (runtime.blobManager as any).redirectTable; - }); - - it("fetching deleted blob fails", async () => { - await runtime.attach(); - await runtime.connect(); - const blob1Contents = IsoBuffer.from("blob1", "utf8"); - const blob2Contents = IsoBuffer.from("blob2", "utf8"); - const handle1P = runtime.createBlob(blob1Contents); - const handle2P = runtime.createBlob(blob2Contents); - await runtime.processAll(); - - const blob1Handle = await handle1P; - const blob2Handle = await handle2P; - - // Validate that the blobs can be retrieved. - assert.strictEqual(await runtime.getBlob(blob1Handle), blob1Contents); - assert.strictEqual(await runtime.getBlob(blob2Handle), blob2Contents); - - // Delete blob1. Retrieving it should result in an error. - runtime.deleteBlob(blob1Handle); - await assert.rejects( - async () => runtime.getBlob(blob1Handle), - (error: IErrorBase & { code: number | undefined }) => { - const blob1Id = blob1Handle.absolutePath.split("/")[2]; - const correctErrorType = error.code === 404; - const correctErrorMessage = error.message === `Blob was deleted: ${blob1Id}`; - return correctErrorType && correctErrorMessage; - }, - "Deleted blob2 fetch should have failed", - ); - - // Delete blob2. Retrieving it should result in an error. - runtime.deleteBlob(blob2Handle); - await assert.rejects( - async () => runtime.getBlob(blob2Handle), - (error: IErrorBase & { code: number | undefined }) => { - const blob2Id = blob2Handle.absolutePath.split("/")[2]; - const correctErrorType = error.code === 404; - const correctErrorMessage = error.message === `Blob was deleted: ${blob2Id}`; - return correctErrorType && correctErrorMessage; - }, - "Deleted blob2 fetch should have failed", - ); - }); - - // Support for this config has been removed. - const legacyKey_disableAttachmentBlobSweep = - "Fluid.GarbageCollection.DisableAttachmentBlobSweep"; - for (const disableAttachmentBlobsSweep of [true, undefined]) - it(`deletes unused blobs regardless of DisableAttachmentBlobsSweep setting [DisableAttachmentBlobsSweep=${disableAttachmentBlobsSweep}]`, async () => { - injectedSettings[legacyKey_disableAttachmentBlobSweep] = disableAttachmentBlobsSweep; - - await runtime.attach(); - await runtime.connect(); - - const blob1 = await createBlobAndGetIds("blob1"); - const blob2 = await createBlobAndGetIds("blob2"); - - // Delete blob1's local id. The local id and the storage id should both be deleted from the redirect table - // since the blob only had one reference. - runtime.blobManager.deleteSweepReadyNodes([blob1.localGCNodeId]); - assert(!redirectTable.has(blob1.localId)); - assert(!redirectTable.has(blob1.storageId)); - - // Delete blob2's local id. The local id and the storage id should both be deleted from the redirect table - // since the blob only had one reference. - runtime.blobManager.deleteSweepReadyNodes([blob2.localGCNodeId]); - assert(!redirectTable.has(blob2.localId)); - assert(!redirectTable.has(blob2.storageId)); - }); - - it("deletes unused de-duped blobs", async () => { - await runtime.attach(); - await runtime.connect(); - - // Create 2 blobs with the same content. They should get de-duped. - const blob1 = await createBlobAndGetIds("blob1"); - const blob1Duplicate = await createBlobAndGetIds("blob1"); - assert(blob1.storageId === blob1Duplicate.storageId, "blob1 not de-duped"); - - // Create another 2 blobs with the same content. They should get de-duped. - const blob2 = await createBlobAndGetIds("blob2"); - const blob2Duplicate = await createBlobAndGetIds("blob2"); - assert(blob2.storageId === blob2Duplicate.storageId, "blob2 not de-duped"); - - // Delete blob1's local id. The local id should both be deleted from the redirect table but the storage id - // should not because the blob has another referenced from the de-duped blob. - runtime.blobManager.deleteSweepReadyNodes([blob1.localGCNodeId]); - assert(!redirectTable.has(blob1.localId), "blob1 localId should have been deleted"); - assert( - redirectTable.has(blob1.storageId), - "blob1 storageId should not have been deleted", - ); - // Delete blob1's de-duped local id. The local id and the storage id should both be deleted from the redirect table - // since all the references for the blob are now deleted. - runtime.blobManager.deleteSweepReadyNodes([blob1Duplicate.localGCNodeId]); - assert( - !redirectTable.has(blob1Duplicate.localId), - "blob1Duplicate localId should have been deleted", - ); - assert( - !redirectTable.has(blob1.storageId), - "blob1 storageId should have been deleted", - ); - - // Delete blob2's local id. The local id should both be deleted from the redirect table but the storage id - // should not because the blob has another referenced from the de-duped blob. - runtime.blobManager.deleteSweepReadyNodes([blob2.localGCNodeId]); - assert(!redirectTable.has(blob2.localId), "blob2 localId should have been deleted"); - assert( - redirectTable.has(blob2.storageId), - "blob2 storageId should not have been deleted", - ); - // Delete blob2's de-duped local id. The local id and the storage id should both be deleted from the redirect table - // since all the references for the blob are now deleted. - runtime.blobManager.deleteSweepReadyNodes([blob2Duplicate.localGCNodeId]); - assert( - !redirectTable.has(blob2Duplicate.localId), - "blob2Duplicate localId should have been deleted", - ); - assert( - !redirectTable.has(blob2.storageId), - "blob2 storageId should have been deleted", - ); - }); - }); - }); -} diff --git a/packages/runtime/container-runtime/src/test/types/validateContainerRuntimePrevious.generated.ts b/packages/runtime/container-runtime/src/test/types/validateContainerRuntimePrevious.generated.ts index 5088661971bc..f07172c46c3c 100644 --- a/packages/runtime/container-runtime/src/test/types/validateContainerRuntimePrevious.generated.ts +++ b/packages/runtime/container-runtime/src/test/types/validateContainerRuntimePrevious.generated.ts @@ -697,6 +697,7 @@ declare type old_as_current_for_Interface_LoadContainerRuntimeParams = requireAs * typeValidation.broken: * "Interface_LoadContainerRuntimeParams": {"backCompat": false} */ +// @ts-expect-error compatibility expected to be broken declare type current_as_old_for_Interface_LoadContainerRuntimeParams = requireAssignableTo, TypeOnly> /* From bc8c507909d59bb66bac8f2481407e6c35517fd3 Mon Sep 17 00:00:00 2001 From: Navin Agarwal Date: Thu, 16 Oct 2025 10:08:15 -0700 Subject: [PATCH 4/6] Changeset update --- .changeset/heavy-bugs-thank.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.changeset/heavy-bugs-thank.md b/.changeset/heavy-bugs-thank.md index 3e699adab49a..e8a742d08221 100644 --- a/.changeset/heavy-bugs-thank.md +++ b/.changeset/heavy-bugs-thank.md @@ -3,24 +3,24 @@ "@fluidframework/runtime-definitions": minor "__section": breaking --- -Removed deprecated properties from "IRuntimeStorageService" and "IContainerStorageService" +Deprecated properties have been removed from IRuntimeStorageService and IContainerStorageService The following deprecated properties have been removed from `IRuntimeStorageService`: -- `disposed` +- `createBlob` - `dispose` -- `policies` -- `getSnapshotTree` +- `disposed` +- `downloadSummary` - `getSnapshot` +- `getSnapshotTree` - `getVersions` -- `createBlob` +- `policies` - `uploadSummaryWithContext` -- `downloadSummary` The following deprecated properties have been removed from `IContainerStorageService`: -- `downloadSummary` -- `disposed` - `dispose` +- `disposed` +- `downloadSummary` -The deprecations were announced in release 2.52.0 [here](https://github.com/microsoft/FluidFramework/releases/tag/client_v2.52.0). +The deprecations were announced in version [2.52.0](https://github.com/microsoft/FluidFramework/releases/tag/client_v2.52.0). From a81d27191ae834316295e838c0be7410c879db90 Mon Sep 17 00:00:00 2001 From: Navin Agarwal Date: Mon, 20 Oct 2025 16:42:27 -0700 Subject: [PATCH 5/6] Undo unrelated changes --- .../loader/container-loader/src/container.ts | 4 +- .../src/test/containerRuntime.spec.ts | 7 ++-- .../test/deRehydrateContainerTests.spec.ts | 37 +++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index cbd68e6bdac7..ce62a73bcc34 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -32,7 +32,7 @@ import type { ReadOnlyInfo, ILoader, ILoaderOptions, - IContainerStorageService, + IDocumentStorageService, } from "@fluidframework/container-definitions/internal"; import { isFluidCodeDetails } from "@fluidframework/container-definitions/internal"; import { @@ -1887,7 +1887,7 @@ export class Container private async initializeProtocolStateFromSnapshot( attributes: IDocumentAttributes, - storage: IContainerStorageService, + storage: IDocumentStorageService, snapshot: ISnapshotTree | undefined, ): Promise { const quorumSnapshot: IQuorumSnapshot = { diff --git a/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts b/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts index 64f2a12d0eb3..c2ed646ab865 100644 --- a/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts +++ b/packages/runtime/container-runtime/src/test/containerRuntime.spec.ts @@ -18,7 +18,6 @@ import { ContainerErrorTypes, type IContainerContext, type IBatchMessage, - type IContainerStorageService, } from "@fluidframework/container-definitions/internal"; import type { IContainerRuntime } from "@fluidframework/container-runtime-definitions/internal"; import type { @@ -243,7 +242,7 @@ describe("Runtime", () => { const mockClientId = "mockClientId"; // Mock the storage layer so "submitSummary" works. - const defaultMockStorage: Partial = { + const defaultMockStorage: Partial = { uploadSummaryWithContext: async (summary: ISummaryTree, context: ISummaryContext) => { return "fakeHandle"; }, @@ -252,7 +251,7 @@ describe("Runtime", () => { params: { settings?: Record; logger?: ITelemetryBaseLogger; - mockStorage?: Partial; + mockStorage?: Partial; loadedFromVersion?: IVersion; baseSnapshot?: ISnapshotTree; connected?: boolean; @@ -299,7 +298,7 @@ describe("Runtime", () => { }, clientId, connected, - storage: mockStorage as IContainerStorageService, + storage: mockStorage as IRuntimeStorageService, baseSnapshot, } satisfies Partial; diff --git a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts index c3c61c54f332..40e01889f96b 100644 --- a/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts +++ b/packages/test/test-end-to-end-tests/src/test/deRehydrateContainerTests.spec.ts @@ -531,6 +531,43 @@ describeCompat( assert.strictEqual(sparseMatrix.id, sparseMatrixId, "Sparse matrix should exist!!"); }); + it("Storage in detached container", async () => { + const { container } = await createDetachedContainerAndGetEntryPoint(); + + const snapshotTree = container.serialize(); + const defaultDataStore = + await getContainerEntryPointBackCompat(container); + assert( + defaultDataStore.context.storage !== undefined, + "Storage should be present in detached data store", + ); + let success1: boolean | undefined; + await defaultDataStore.context.storage.getSnapshotTree(undefined).catch((err) => { + success1 = false; + }); + assert( + success1 === false, + "Snapshot fetch should not be allowed in detached data store", + ); + + const container2: IContainer = + await loader.rehydrateDetachedContainerFromSnapshot(snapshotTree); + const defaultDataStore2 = + await getContainerEntryPointBackCompat(container2); + assert( + defaultDataStore2.context.storage !== undefined, + "Storage should be present in rehydrated data store", + ); + let success2: boolean | undefined; + await defaultDataStore2.context.storage.getSnapshotTree(undefined).catch((err) => { + success2 = false; + }); + assert( + success2 === false, + "Snapshot fetch should not be allowed in rehydrated data store", + ); + }); + it("Change contents of dds, then rehydrate and then check summary", async () => { const { container } = await createDetachedContainerAndGetEntryPoint(); From 4b3fd670ebb3e322b7eefa5334cb5800f9d2dfe8 Mon Sep 17 00:00:00 2001 From: Navin Agarwal Date: Mon, 20 Oct 2025 16:51:55 -0700 Subject: [PATCH 6/6] Update changeset --- .changeset/heavy-bugs-thank.md | 2 +- packages/loader/container-loader/src/container.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/heavy-bugs-thank.md b/.changeset/heavy-bugs-thank.md index e8a742d08221..dd476534b9a9 100644 --- a/.changeset/heavy-bugs-thank.md +++ b/.changeset/heavy-bugs-thank.md @@ -23,4 +23,4 @@ The following deprecated properties have been removed from `IContainerStorageSer - `disposed` - `downloadSummary` -The deprecations were announced in version [2.52.0](https://github.com/microsoft/FluidFramework/releases/tag/client_v2.52.0). +Please see [this Github issue](https://github.com/microsoft/FluidFramework/issues/25069) for more details. diff --git a/packages/loader/container-loader/src/container.ts b/packages/loader/container-loader/src/container.ts index ce62a73bcc34..35ec5f477fbc 100644 --- a/packages/loader/container-loader/src/container.ts +++ b/packages/loader/container-loader/src/container.ts @@ -32,7 +32,6 @@ import type { ReadOnlyInfo, ILoader, ILoaderOptions, - IDocumentStorageService, } from "@fluidframework/container-definitions/internal"; import { isFluidCodeDetails } from "@fluidframework/container-definitions/internal"; import { @@ -55,6 +54,7 @@ import { import { type IDocumentService, type IDocumentServiceFactory, + type IDocumentStorageService, type IResolvedUrl, type ISnapshot, type IThrottlingWarning,