diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 102c98f5931..cd816fb972e 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -201,7 +201,7 @@ export class AudioStreamController extends BaseStreamController implements Netwo // (undocumented) _handleFragmentLoadProgress(data: FragLoadedData): void; // (undocumented) - protected loadFragment(frag: Fragment, track: Level, targetBufferTime: number): void; + protected loadFragment(frag: MediaFragment, track: Level, targetBufferTime: number): void; get nextAudioTrack(): number; // (undocumented) protected onError(event: Events.ERROR, data: ErrorData): void; @@ -529,9 +529,9 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected getMaxBufferLength(levelBitrate?: number): number; // (undocumented) - protected getNextFragment(pos: number, levelDetails: LevelDetails): Fragment | null; + protected getNextFragment(pos: number, levelDetails: LevelDetails): MediaFragment | null; // (undocumented) - protected getNextFragmentLoopLoading(frag: Fragment, levelDetails: LevelDetails, bufferInfo: BufferInfo, playlistType: PlaylistLevelType, maxBufLen: number): Fragment | null; + protected getNextFragmentLoopLoading(frag: MediaFragment, levelDetails: LevelDetails, bufferInfo: BufferInfo, playlistType: PlaylistLevelType, maxBufLen: number): MediaFragment | null; // (undocumented) getNextPart(partList: Part[], frag: Fragment, targetBufferTime: number): number; // (undocumented) @@ -547,6 +547,8 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) get inFlightFrag(): InFlightData; // (undocumented) + protected initFragmentLoader: FragmentLoader; + // (undocumented) protected initPTS: TimestampOffset[]; // (undocumented) protected isLoopLoading(frag: Fragment, targetBufferTime: number): boolean; @@ -563,9 +565,7 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP // (undocumented) protected loadingParts: boolean; // (undocumented) - protected _loadInitSegment(fragment: Fragment, level: Level): void; - // (undocumented) - protected mapToInitFragWhenRequired(frag: T): T | Fragment; + protected _loadInitSegment(initFrag: Fragment): Promise; // (undocumented) protected media: HTMLMediaElement | null; // (undocumented) @@ -1991,7 +1991,7 @@ export class FragmentLoader { // (undocumented) destroy(): void; // (undocumented) - load(frag: Fragment, isIFrame?: boolean, onProgress?: FragmentLoadProgressCallback): Promise; + load(frag: Fragment, isIFrame?: boolean, onProgress?: FragmentLoadProgressCallback, progressGate?: Promise): Promise; // (undocumented) loadPart(frag: Fragment, part: Part, onProgress: FragmentLoadProgressCallback): Promise; } @@ -4905,7 +4905,7 @@ export class StreamController extends BaseStreamController implements NetworkCom // (undocumented) immediateLevelSwitch(): void; // (undocumented) - protected loadFragment(frag: Fragment, level: Level, targetBufferTime: number): void; + protected loadFragment(frag: MediaFragment, level: Level, targetBufferTime: number): void; // (undocumented) get maxBufferLength(): number; // (undocumented) @@ -5000,7 +5000,7 @@ export class SubtitleStreamController extends BaseStreamController implements Ne // (undocumented) _handleFragmentLoadComplete(fragLoadedData: FragLoadedData): void; // (undocumented) - protected loadFragment(frag: Fragment, level: Level, targetBufferTime: number): void; + protected loadFragment(frag: MediaFragment, level: Level, targetBufferTime: number): void; // (undocumented) get mediaBufferTimeRanges(): Bufferable; // (undocumented) diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index a1a727fcab0..0ac33ea6a78 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -1032,7 +1032,7 @@ class AudioStreamController } protected loadFragment( - frag: Fragment, + frag: MediaFragment, track: Level, targetBufferTime: number, ) { @@ -1045,9 +1045,7 @@ class AudioStreamController fragState === FragmentState.NOT_LOADED || fragState === FragmentState.PARTIAL ) { - if (!isMediaFragment(frag)) { - this._loadInitSegment(frag, track); - } else if (track.details?.live && !this.initPTS[frag.cc]) { + if (track.details?.live && !this.initPTS[frag.cc]) { this.log( `Waiting for video PTS in continuity counter ${frag.cc} of live stream before loading audio fragment ${frag.sn} of level ${this.trackId}`, ); diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 2e18f5e96f2..33511340fff 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -119,6 +119,7 @@ export default class BaseStreamController protected retryDate: number = 0; protected levels: Array | null = null; protected fragmentLoader: FragmentLoader; + protected initFragmentLoader: FragmentLoader; protected keyLoader: KeyLoader; protected levelLastLoaded: Level | null = null; protected startFragRequested: boolean = false; @@ -139,6 +140,7 @@ export default class BaseStreamController this.playlistType = playlistType; this.hls = hls; this.fragmentLoader = new FragmentLoader(hls.config); + this.initFragmentLoader = new FragmentLoader(hls.config); this.keyLoader = keyLoader; this.fragmentTracker = fragmentTracker; this.config = hls.config; @@ -176,6 +178,7 @@ export default class BaseStreamController return; } this.fragmentLoader.abort(); + this.initFragmentLoader.abort(); this.keyLoader.abort(this.playlistType); const frag = this.fragCurrent; if (frag?.loader) { @@ -511,6 +514,9 @@ export default class BaseStreamController if (this.fragmentLoader) { this.fragmentLoader.destroy(); } + if (this.initFragmentLoader) { + this.initFragmentLoader.destroy(); + } if (this.keyLoader) { this.keyLoader.destroy(); } @@ -521,7 +527,8 @@ export default class BaseStreamController //@ts-ignore this.hls = this.decrypter = this.keyLoader = null; //@ts-ignore - this.fragmentLoader = this.fragmentTracker = null; + this.fragmentLoader = this.initFragmentLoader = this.fragmentTracker = null; + super.onHandlerDestroyed(); } @@ -698,18 +705,24 @@ export default class BaseStreamController this.hls.trigger(Events.BUFFER_FLUSHING, flushScope); } - protected _loadInitSegment(fragment: Fragment, level: Level) { - this._doFragLoad(fragment, level) + protected _loadInitSegment(initFrag: Fragment): Promise { + const { hls } = this; + this.initFragmentLoader.abort(); + hls.trigger(Events.FRAG_LOADING, { frag: initFrag, targetBufferTime: 0 }); + return this.initFragmentLoader + .load(initFrag) .then((data) => { const frag = data?.frag; - if (!frag || this.fragContextChanged(frag) || !this.levels) { + if ( + !frag || + !fragmentsAreEqual(frag, this.fragCurrent?.initSegment) || + !this.levels + ) { throw new Error('init load aborted'); } - return data; }) .then((data: FragLoadedData) => { - const { hls } = this; const { frag, payload } = data; const decryptData = frag.decryptdata; @@ -744,6 +757,7 @@ export default class BaseStreamController reason: err.message, frag, }); + this.fragmentLoader.abort(); throw err; }) .then((decryptedData) => { @@ -761,29 +775,47 @@ export default class BaseStreamController }); } return this.completeInitSegmentLoad(data); - }) - .catch((reason) => { - if (this.state === State.STOPPED || this.state === State.ERROR) { - return; - } - this.warn(reason); - this.resetFragmentLoading(fragment); }); } private completeInitSegmentLoad(data: FragLoadedData) { - const { levels } = this; - if (!levels) { - throw new Error('init load aborted, missing levels'); - } const stats = data.frag.stats; - if (this.state !== State.STOPPED) { - this.state = State.IDLE; - } data.frag.data = new Uint8Array(data.payload); stats.parsing.start = stats.buffering.start = self.performance.now(); stats.parsing.end = stats.buffering.end = self.performance.now(); - this.tick(); + } + + private loadInitSegmentIfNeeded( + mediaFrag: Fragment, + ): Promise | undefined { + const { initSegment } = mediaFrag; + if ( + !this.bitrateTest && + isMediaFragment(mediaFrag) && + initSegment && + !initSegment.data + ) { + const initPromise = + initSegment.encrypted && !initSegment.decryptdata?.key + ? this.keyLoader + .load(initSegment) + .then(() => this._loadInitSegment(initSegment)) + : this._loadInitSegment(initSegment); + + return initPromise.catch((reason: LoadError | Error) => { + if (this.state === State.STOPPED || this.state === State.ERROR) { + throw reason; + } + if ('data' in reason) { + reason.data.frag = mediaFrag; + this.fragmentLoader.abort(); + this.handleFragLoadError(reason); + } + this.warn(reason); + this.resetFragmentLoading(mediaFrag); + throw reason; + }); + } } protected unhandledEncryptionError( @@ -983,12 +1015,15 @@ export default class BaseStreamController this.nextLoadPosition = part.start + part.duration; this.state = State.FRAG_LOADING; let result: Promise; - if (keyLoadingPromise) { - result = keyLoadingPromise - .then((keyLoadedData) => { + const initDataPromise = this.loadInitSegmentIfNeeded(frag); + if (keyLoadingPromise || initDataPromise) { + const useKeyLoader = !!keyLoadingPromise; + result = Promise.all([keyLoadingPromise, initDataPromise]) + .then(([keyLoadedData]) => { if ( - !keyLoadedData || - this.fragContextChanged(keyLoadedData.frag) + useKeyLoader && + (!keyLoadedData || + this.fragContextChanged(keyLoadedData.frag)) ) { return null; } @@ -1057,6 +1092,9 @@ export default class BaseStreamController // Load key before streaming fragment data const dataOnProgress = this.config.progressive && frag.type !== PlaylistLevelType.SUBTITLE; + + const initDataPromise = this.loadInitSegmentIfNeeded(frag); + let result: Promise; if (dataOnProgress && keyLoadingPromise) { result = keyLoadingPromise @@ -1068,6 +1106,7 @@ export default class BaseStreamController frag, this.iframesOnly, progressCallback, + initDataPromise, ); }) .catch((error) => this.handleFragLoadError(error)); @@ -1079,8 +1118,10 @@ export default class BaseStreamController frag, this.iframesOnly, dataOnProgress ? progressCallback : undefined, + dataOnProgress ? initDataPromise : undefined, ), keyLoadingPromise, + initDataPromise, ]) .then(([fragLoadedData]) => { if (!dataOnProgress && progressCallback) { @@ -1096,7 +1137,10 @@ export default class BaseStreamController new Error(`frag load aborted, context changed in FRAG_LOADING`), ); } - return result; + + return Promise.all([result, initDataPromise]).then( + ([fragData]) => fragData, + ); } private doFragPartsLoad( @@ -1431,7 +1475,7 @@ export default class BaseStreamController protected getNextFragment( pos: number, levelDetails: LevelDetails, - ): Fragment | null { + ): MediaFragment | null { const fragments = levelDetails.fragments; const fragLen = fragments.length; @@ -1523,7 +1567,7 @@ export default class BaseStreamController levelDetails, ); } - return this.mapToInitFragWhenRequired(programFrag); + return programFrag; } protected isLoopLoading(frag: Fragment, targetBufferTime: number): boolean { @@ -1546,13 +1590,13 @@ export default class BaseStreamController } protected getNextFragmentLoopLoading( - frag: Fragment, + frag: MediaFragment, levelDetails: LevelDetails, bufferInfo: BufferInfo, playlistType: PlaylistLevelType, maxBufLen: number, - ): Fragment | null { - let nextFragment: Fragment | null = null; + ): MediaFragment | null { + let nextFragment: MediaFragment | null = null; if (frag.gap) { nextFragment = this.getNextFragment(this.nextLoadPosition, levelDetails); if (nextFragment && !nextFragment.gap && bufferInfo.nextStart) { @@ -1656,17 +1700,6 @@ export default class BaseStreamController return frag; } - protected mapToInitFragWhenRequired( - frag: T, - ): T | Fragment { - // If an initSegment is present, it must be buffered first - if (frag?.initSegment && !frag.initSegment.data && !this.bitrateTest) { - return frag.initSegment; - } - - return frag; - } - getNextPart( partList: Part[], frag: Fragment, diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index d48a388bdcf..a0338adf022 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -374,13 +374,11 @@ export default class StreamController return; } - frag = this.mapToInitFragWhenRequired(frag); - this.loadFragment(frag, levelInfo, targetBufferTime); } protected loadFragment( - frag: Fragment, + frag: MediaFragment, level: Level, targetBufferTime: number, ) { @@ -390,9 +388,7 @@ export default class StreamController fragState === FragmentState.NOT_LOADED || fragState === FragmentState.PARTIAL ) { - if (!isMediaFragment(frag)) { - this._loadInitSegment(frag, level); - } else if (this.bitrateTest) { + if (this.bitrateTest) { this.log( `Fragment ${frag.sn} of level ${frag.level} is being downloaded to test bitrate and will not be buffered`, ); diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts index 424fd1c99a5..89d0aa9b74e 100644 --- a/src/controller/subtitle-stream-controller.ts +++ b/src/controller/subtitle-stream-controller.ts @@ -2,11 +2,7 @@ import BaseStreamController, { State } from './base-stream-controller'; import { FragmentState } from './fragment-tracker'; import { ErrorDetails, ErrorTypes } from '../errors'; import { Events } from '../events'; -import { - type Fragment, - isMediaFragment, - type MediaFragment, -} from '../loader/fragment'; +import { isMediaFragment, type MediaFragment } from '../loader/fragment'; import { Level } from '../types/level'; import { PlaylistLevelType } from '../types/loader'; import { BufferHelper } from '../utils/buffer-helper'; @@ -448,7 +444,7 @@ export class SubtitleStreamController } protected loadFragment( - frag: Fragment, + frag: MediaFragment, level: Level, targetBufferTime: number, ) { @@ -458,11 +454,7 @@ export class SubtitleStreamController fragState === FragmentState.NOT_LOADED || fragState === FragmentState.PARTIAL ) { - if (!isMediaFragment(frag)) { - this._loadInitSegment(frag, level); - } else { - super.loadFragment(frag, level, targetBufferTime); - } + super.loadFragment(frag, level, targetBufferTime); } } diff --git a/src/loader/fragment-loader.ts b/src/loader/fragment-loader.ts index 2f5bca474b9..fc4eb65c551 100644 --- a/src/loader/fragment-loader.ts +++ b/src/loader/fragment-loader.ts @@ -45,6 +45,7 @@ export default class FragmentLoader { frag: Fragment, isIFrame?: boolean, onProgress?: FragmentLoadProgressCallback, + progressGate?: Promise, ): Promise { const url = frag.url; if (!url) { @@ -159,8 +160,10 @@ export default class FragmentLoader { }, }; if (onProgress) { + const onProgressGated = this.gateProgress(onProgress, progressGate); + callbacks.onProgress = (stats, context, data, networkDetails) => - onProgress({ + onProgressGated({ frag, part: null, payload: data as ArrayBuffer, @@ -171,6 +174,38 @@ export default class FragmentLoader { }); } + /* + When a progressGate is provided, buffer incoming chunks until the + gate promise resolves, then flush them in arrival order. This + ensuring the transmuxer receives init data before the first chunk. + Which is important when progressively loading media fragments. + */ + private gateProgress( + onProgress: FragmentLoadProgressCallback, + progressGate?: Promise, + ): FragmentLoadProgressCallback { + let handleChunk = onProgress; + if (progressGate) { + const pendingChunks: FragLoadedData[] = []; + let gateDone = false; + progressGate + .then(() => { + gateDone = true; + pendingChunks.forEach((chunk) => onProgress(chunk)); + pendingChunks.length = 0; + }) + .catch(() => {}); + handleChunk = (data) => { + if (gateDone) { + onProgress(data); + } else { + pendingChunks.push(data as FragLoadedData); + } + }; + } + return handleChunk; + } + public loadPart( frag: Fragment, part: Part, diff --git a/tests/unit/controller/base-stream-controller.ts b/tests/unit/controller/base-stream-controller.ts index 065f92791e0..e0290b321cb 100644 --- a/tests/unit/controller/base-stream-controller.ts +++ b/tests/unit/controller/base-stream-controller.ts @@ -4,10 +4,13 @@ import sinonChai from 'sinon-chai'; import { hlsDefaultConfig } from '../../../src/config'; import { State } from '../../../src/controller/base-stream-controller'; import BaseStreamControllerImpl from '../../../src/controller/stream-controller'; +import { ErrorDetails, ErrorTypes } from '../../../src/errors'; import Hls from '../../../src/hls'; import { Fragment } from '../../../src/loader/fragment'; +import { LoadError } from '../../../src/loader/fragment-loader'; import KeyLoader from '../../../src/loader/key-loader'; import { LevelDetails } from '../../../src/loader/level-details'; +import { LevelKey } from '../../../src/loader/level-key'; import { PlaylistLevelType } from '../../../src/types/loader'; import { BufferHelper } from '../../../src/utils/buffer-helper'; import { TimeRangesMock } from '../../mocks/time-ranges.mock'; @@ -876,6 +879,127 @@ describe('BaseStreamController', function () { }); }); + describe('init segment loading', function () { + function createMediaFragWithInitSegment(encrypted: boolean = false) { + const initSegment = new Fragment(PlaylistLevelType.MAIN, 'init.mp4'); + initSegment.sn = 'initSegment'; + initSegment.relurl = 'init.mp4'; + if (encrypted) { + initSegment.levelkeys = { + identity: new LevelKey( + 'AES-128', + 'https://example.com/key.bin', + 'identity', + ), + }; + } + + const mediaFrag = new Fragment( + PlaylistLevelType.MAIN, + 'segment.ts', + ) as MediaFragment; + mediaFrag.sn = 0; + mediaFrag.initSegment = initSegment; + return { mediaFrag, initSegment }; + } + + it('loadInitSegmentIfNeeded loads the init fragment key before fetching init data', function () { + const { mediaFrag, initSegment } = createMediaFragWithInitSegment(true); + const callOrder: string[] = []; + + const keyLoadStub = sinon + .stub((baseStreamController as any).keyLoader, 'load') + .callsFake((frag) => { + callOrder.push('keyLoad'); + expect(frag).to.equal(initSegment); + return Promise.resolve({ frag, keyInfo: {} }); + }); + const loadInitStub = sinon + .stub(baseStreamController as any, '_loadInitSegment') + .callsFake(() => { + callOrder.push('loadInit'); + return Promise.resolve(); + }); + + const result = (baseStreamController as any).loadInitSegmentIfNeeded( + mediaFrag, + ); + + expect(result).to.be.a('promise'); + return result! + .then(() => { + expect(keyLoadStub).to.have.been.calledOnceWith(initSegment); + expect(loadInitStub).to.have.been.calledOnceWith(initSegment); + expect(callOrder).to.deep.equal(['keyLoad', 'loadInit']); + }) + .finally(() => { + keyLoadStub.restore(); + loadInitStub.restore(); + }); + }); + + it('loadInitSegmentIfNeeded handles init segment load errors', function () { + const { mediaFrag, initSegment } = createMediaFragWithInitSegment(false); + baseStreamController.state = State.FRAG_LOADING; + const loadError = new LoadError({ + type: ErrorTypes.NETWORK_ERROR, + details: ErrorDetails.FRAG_LOAD_ERROR, + fatal: false, + error: new Error('init load failed'), + frag: initSegment, + networkDetails: null, + }); + + sinon + .stub(baseStreamController as any, '_loadInitSegment') + .rejects(loadError); + const abortStub = sinon.stub( + (baseStreamController as any).fragmentLoader, + 'abort', + ); + const handleFragLoadErrorStub = sinon.stub( + baseStreamController as any, + 'handleFragLoadError', + ); + const resetFragmentLoadingStub = sinon.stub( + baseStreamController as any, + 'resetFragmentLoading', + ); + const warnStub = sinon.stub(baseStreamController as any, 'warn'); + + const result = (baseStreamController as any).loadInitSegmentIfNeeded( + mediaFrag, + ); + + expect(result).to.be.a('promise'); + return result! + .then(() => { + throw new Error('Expected init segment load to reject'); + }) + .catch((error) => { + expect(error).to.equal(loadError); + expect(loadError.data.frag).to.equal(mediaFrag); + expect(abortStub).to.have.been.calledOnce; + expect(handleFragLoadErrorStub).to.have.been.calledOnceWith( + loadError, + ); + expect(handleFragLoadErrorStub.firstCall.args[0].data.frag).to.equal( + mediaFrag, + ); + expect(resetFragmentLoadingStub).to.have.been.calledOnceWith( + mediaFrag, + ); + expect(warnStub).to.have.been.calledOnceWith(loadError); + }) + .finally(() => { + abortStub.restore(); + handleFragLoadErrorStub.restore(); + resetFragmentLoadingStub.restore(); + warnStub.restore(); + }); + }); + }); + describe('backtrackFragment and couldBacktrack properties', function () { it('should return undefined for backtrackFragment by default', function () { expect((baseStreamController as any).backtrackFragment).to.be.undefined; diff --git a/tests/unit/loader/fragment-loader.ts b/tests/unit/loader/fragment-loader.ts index dbb796595ea..1dea228254a 100644 --- a/tests/unit/loader/fragment-loader.ts +++ b/tests/unit/loader/fragment-loader.ts @@ -140,6 +140,106 @@ describe('FragmentLoader tests', function () { }); }); + it('buffers progress callbacks until progressGate resolves', function () { + response = { data: new Uint8Array(4) }; + const onProgress = sinon.spy(); + let resolveGate: () => void; + const progressGate = new Promise((res) => { + resolveGate = res; + }); + const fragmentLoaderPrivates = fragmentLoader as any; + const chunk1 = new ArrayBuffer(4); + const chunk2 = new ArrayBuffer(8); + + const loadPromise = fragmentLoader.load( + frag, + false, + onProgress, + progressGate, + ); + + fragmentLoaderPrivates.loader.callbacks.onProgress( + stats, + context, + chunk1, + networkDetails, + ); + fragmentLoaderPrivates.loader.callbacks.onProgress( + stats, + context, + chunk2, + networkDetails, + ); + expect(onProgress).to.not.have.been.called; + + resolveGate!(); + + return Promise.resolve() + .then(() => { + expect(onProgress).to.have.been.calledTwice; + expect(onProgress.firstCall).to.have.been.calledWith({ + frag, + part: null, + payload: chunk1, + networkDetails, + }); + expect(onProgress.secondCall).to.have.been.calledWith({ + frag, + part: null, + payload: chunk2, + networkDetails, + }); + fragmentLoaderPrivates.loader.callbacks.onSuccess( + response, + stats, + context, + networkDetails, + ); + }) + .then(() => loadPromise) + .then((data) => { + expect(data.payload).to.equal(response.data); + }); + }); + + it('forwards progress callbacks immediately after progressGate resolves', function () { + response = { data: new Uint8Array(4) }; + const onProgress = sinon.spy(); + const progressGate = Promise.resolve(); + const fragmentLoaderPrivates = fragmentLoader as any; + const chunk = new ArrayBuffer(4); + + const loadPromise = fragmentLoader.load( + frag, + false, + onProgress, + progressGate, + ); + + return progressGate + .then(() => { + fragmentLoaderPrivates.loader.callbacks.onProgress( + stats, + context, + chunk, + networkDetails, + ); + expect(onProgress).to.have.been.calledOnceWith({ + frag, + part: null, + payload: chunk, + networkDetails, + }); + fragmentLoaderPrivates.loader.callbacks.onSuccess( + response, + stats, + context, + networkDetails, + ); + }) + .then(() => loadPromise); + }); + it('handles fragment load timeouts', function () { const fragmentLoaderPrivates = fragmentLoader as any; return new Promise((resolve, reject) => {