From 183cdc594626f2647f2c15c6147992a6a878026a Mon Sep 17 00:00:00 2001 From: Fabian Schliski Date: Sat, 25 Apr 2026 15:11:39 +0200 Subject: [PATCH 1/6] Improve SIMKL syncing stability & sync DodoStream removals to SIMKL --- src/api/simkl/__tests__/client.test.ts | 21 +- src/api/simkl/__tests__/id-resolver.test.ts | 16 +- src/api/simkl/__tests__/sync-service.test.ts | 126 +++-- src/api/simkl/client.ts | 105 ++-- src/api/simkl/hooks.ts | 10 +- src/api/simkl/id-resolver.ts | 10 +- src/api/simkl/sync-service.ts | 393 ++++++++----- .../settings/SimklFirstConnectModal.tsx | 4 +- src/db/__tests__/myList.test.ts | 50 +- src/db/__tests__/watchHistory.test.ts | 104 +++- src/db/drizzle/0001_fluffy_karma.sql | 42 ++ src/db/drizzle/meta/0001_snapshot.json | 516 ++++++++++++++++++ src/db/drizzle/meta/_journal.json | 9 +- src/db/drizzle/migrations.ts | 21 + src/db/queries/myList.ts | 18 +- src/db/queries/syncQueue.ts | 85 +++ src/db/queries/watchHistory.ts | 38 +- src/db/schema.ts | 18 + src/store/integrations.store.ts | 3 +- src/types/integrations.ts | 14 +- src/types/simkl.ts | 21 +- 21 files changed, 1358 insertions(+), 266 deletions(-) create mode 100644 src/db/drizzle/0001_fluffy_karma.sql create mode 100644 src/db/drizzle/meta/0001_snapshot.json create mode 100644 src/db/queries/syncQueue.ts diff --git a/src/api/simkl/__tests__/client.test.ts b/src/api/simkl/__tests__/client.test.ts index d580023..a91cc79 100644 --- a/src/api/simkl/__tests__/client.test.ts +++ b/src/api/simkl/__tests__/client.test.ts @@ -39,14 +39,14 @@ describe('simkl client', () => { ); // Act - await getPinCode('client-123'); + await getPinCode(); // Assert const calledUrl = String(mockFetch.mock.calls[0][0]); const parsed = new URL(calledUrl); expect(parsed.pathname).toBe('/oauth/pin'); - expect(parsed.searchParams.get('client_id')).toBe('client-123'); + expect(parsed.searchParams.get('client_id')).toBe('test-client-id'); expect(parsed.searchParams.get('app-name')).toBe('dodostream'); expect(parsed.searchParams.get('app-version')).toBe('1.0.0'); }); @@ -58,12 +58,13 @@ describe('simkl client', () => { ); // Act - await getUserSettings('token-abc', 'client-id'); + await getUserSettings('token-abc'); // Assert const options = mockFetch.mock.calls[0][1] as RequestInit; const headers = options.headers as Record; expect(headers.Authorization).toBe('Bearer token-abc'); + expect(headers['simkl-api-key']).toBe('test-client-id'); }); it('getAllItems includes date_from when provided and omits when not provided', async () => { @@ -71,8 +72,8 @@ describe('simkl client', () => { mockFetch.mockResolvedValue(mockResponse({ movies: [] })); // Act - await getAllItems('token-1', 'client-id', 'movies', '2026-01-01T00:00:00.000Z'); - await getAllItems('token-1', 'client-id', 'movies'); + await getAllItems('token-1', 'movies', '2026-01-01T00:00:00.000Z'); + await getAllItems('token-1', 'movies'); // Assert const firstUrl = new URL(String(mockFetch.mock.calls[0][0])); @@ -87,15 +88,15 @@ describe('simkl client', () => { mockFetch.mockResolvedValue(mockResponse({})); // Act - await getAllItems('token-1', 'client-id', 'movies'); - await getAllItems('token-1', 'client-id', 'anime'); + await getAllItems('token-1', 'movies'); + await getAllItems('token-1', 'anime'); // Assert const movieUrl = new URL(String(mockFetch.mock.calls[0][0])); const animeUrl = new URL(String(mockFetch.mock.calls[1][0])); expect(movieUrl.searchParams.get('extended')).toBe('full'); - expect(animeUrl.searchParams.get('extended')).toBe('full_anime_seasons'); + expect(animeUrl.searchParams.get('extended')).toBe('full'); }); it('simklFetch throws on non-ok response via exported function', async () => { @@ -103,7 +104,7 @@ describe('simkl client', () => { mockFetch.mockResolvedValueOnce({ ok: false, status: 500, json: async () => ({}) }); // Act / Assert - await expect(getUserSettings('bad-token', 'client-id')).rejects.toThrow('Simkl API error 500'); + await expect(getUserSettings('bad-token')).rejects.toThrow('Simkl API error 500'); }); it('simklFetch includes User-Agent header', async () => { @@ -120,7 +121,7 @@ describe('simkl client', () => { ); // Act - await getPinCode('client-123'); + await getPinCode(); // Assert const options = mockFetch.mock.calls[0][1] as RequestInit; diff --git a/src/api/simkl/__tests__/id-resolver.test.ts b/src/api/simkl/__tests__/id-resolver.test.ts index e395778..1b061b5 100644 --- a/src/api/simkl/__tests__/id-resolver.test.ts +++ b/src/api/simkl/__tests__/id-resolver.test.ts @@ -18,11 +18,11 @@ describe('resolveSimklIds', () => { // Act // eslint-disable-next-line @typescript-eslint/no-require-imports const { resolveSimklIds } = require('../id-resolver') as typeof import('../id-resolver'); - const result = await resolveSimklIds('meta-1', 'movie', 'client-1'); + const result = await resolveSimklIds('meta-1', 'movie'); // Assert expect(result).toEqual(ids); - expect(mockSearchById).toHaveBeenCalledWith('client-1', 'meta-1'); + expect(mockSearchById).toHaveBeenCalledWith('meta-1'); }); it('returns null when searchById returns empty array', async () => { @@ -32,7 +32,7 @@ describe('resolveSimklIds', () => { // Act // eslint-disable-next-line @typescript-eslint/no-require-imports const { resolveSimklIds } = require('../id-resolver') as typeof import('../id-resolver'); - const result = await resolveSimklIds('meta-2', 'series', 'client-1'); + const result = await resolveSimklIds('meta-2', 'series'); // Assert expect(result).toBeNull(); @@ -46,8 +46,8 @@ describe('resolveSimklIds', () => { // Act // eslint-disable-next-line @typescript-eslint/no-require-imports const { resolveSimklIds } = require('../id-resolver') as typeof import('../id-resolver'); - const first = await resolveSimklIds('cached-meta', 'movie', 'client-1'); - const second = await resolveSimklIds('cached-meta', 'movie', 'client-1'); + const first = await resolveSimklIds('cached-meta', 'movie'); + const second = await resolveSimklIds('cached-meta', 'movie'); // Assert expect(first).toEqual(ids); @@ -62,8 +62,8 @@ describe('resolveSimklIds', () => { // Act // eslint-disable-next-line @typescript-eslint/no-require-imports const { resolveSimklIds } = require('../id-resolver') as typeof import('../id-resolver'); - const first = await resolveSimklIds('error-meta', 'movie', 'client-1'); - const second = await resolveSimklIds('error-meta', 'movie', 'client-1'); + const first = await resolveSimklIds('error-meta', 'movie'); + const second = await resolveSimklIds('error-meta', 'movie'); // Assert expect(first).toBeNull(); @@ -75,7 +75,7 @@ describe('resolveSimklIds', () => { // Arrange / Act // eslint-disable-next-line @typescript-eslint/no-require-imports const { resolveSimklIds } = require('../id-resolver') as typeof import('../id-resolver'); - const result = await resolveSimklIds('tt9999999', 'movie', 'client-1'); + const result = await resolveSimklIds('tt9999999', 'movie'); // Assert — IMDB IDs are returned directly without a network lookup expect(result).toEqual({ imdb: 'tt9999999' }); diff --git a/src/api/simkl/__tests__/sync-service.test.ts b/src/api/simkl/__tests__/sync-service.test.ts index 34c6f5b..a81a0a7 100644 --- a/src/api/simkl/__tests__/sync-service.test.ts +++ b/src/api/simkl/__tests__/sync-service.test.ts @@ -4,12 +4,14 @@ const mockGetActivities = jest.fn(); const mockGetAllItems = jest.fn(); const mockPostHistory = jest.fn(); const mockPostWatchlist = jest.fn(); +const mockRemoveFromHistory = jest.fn(); jest.mock('../client', () => ({ getActivities: (...args: any[]) => mockGetActivities(...args), getAllItems: (...args: any[]) => mockGetAllItems(...args), postHistory: (...args: any[]) => mockPostHistory(...args), postWatchlist: (...args: any[]) => mockPostWatchlist(...args), + removeFromHistory: (...args: any[]) => mockRemoveFromHistory(...args), })); const mockResolveSimklIds = jest.fn(); @@ -21,18 +23,35 @@ const mockUpsertWatchProgress = jest.fn(); const mockListWatchHistory = jest.fn(); const mockListExportableWatchHistory = jest.fn(); const mockRemoveProfileWatchHistory = jest.fn(); +const mockRemoveWatchHistoryMeta = jest.fn(); jest.mock('@/db/queries/watchHistory', () => ({ upsertWatchProgress: (...args: any[]) => mockUpsertWatchProgress(...args), listWatchHistoryForProfile: (...args: any[]) => mockListWatchHistory(...args), listExportableWatchHistoryForProfile: (...args: any[]) => mockListExportableWatchHistory(...args), removeProfileWatchHistory: (...args: any[]) => mockRemoveProfileWatchHistory(...args), + removeWatchHistoryMeta: (...args: any[]) => mockRemoveWatchHistoryMeta(...args), +})); + +const mockAddToSyncQueue = jest.fn(); +const mockCancelPendingSyncRemovals = jest.fn(); +const mockListSyncQueueForProvider = jest.fn(); +const mockDeleteFromSyncQueue = jest.fn(); +jest.mock('@/db/queries/syncQueue', () => ({ + addToSyncQueue: (...args: any[]) => mockAddToSyncQueue(...args), + cancelPendingSyncRemovals: (...args: any[]) => mockCancelPendingSyncRemovals(...args), + listSyncQueueForProvider: (...args: any[]) => mockListSyncQueueForProvider(...args), + deleteFromSyncQueue: (...args: any[]) => mockDeleteFromSyncQueue(...args), })); const mockAddToMyList = jest.fn(); const mockListExportableMyList = jest.fn(); +const mockRemoveFromMyList = jest.fn(); +const mockRemoveProfileMyList = jest.fn(); jest.mock('@/db/queries/myList', () => ({ addToMyList: (...args: any[]) => mockAddToMyList(...args), listExportableMyListForProfile: (...args: any[]) => mockListExportableMyList(...args), + removeFromMyList: (...args: any[]) => mockRemoveFromMyList(...args), + removeProfileMyList: (...args: any[]) => mockRemoveProfileMyList(...args), })); const mockUpdateSimklCursors = jest.fn(); @@ -57,19 +76,27 @@ describe('simkl sync service', () => { mockListExportableWatchHistory.mockReset(); mockListExportableMyList.mockReset(); mockRemoveProfileWatchHistory.mockReset(); + mockRemoveWatchHistoryMeta.mockReset(); mockUpdateSimklCursors.mockReset(); mockAddToMyList.mockReset(); + mockRemoveFromMyList.mockReset(); + mockRemoveProfileMyList.mockReset(); + mockAddToSyncQueue.mockReset(); + mockCancelPendingSyncRemovals.mockReset(); + mockListSyncQueueForProvider.mockReset(); + mockDeleteFromSyncQueue.mockReset(); mockGetActivities.mockResolvedValue({ all: '2026-01-01T00:00:00.000Z', - movies: { all: '2026-01-01T00:00:00.000Z' }, - tv_shows: { all: '2026-01-01T00:00:00.000Z' }, - anime: { all: '2026-01-01T00:00:00.000Z' }, + movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, + tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, + anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, }); mockGetAllItems.mockResolvedValue({ movies: [], shows: [], anime: [] }); mockListWatchHistory.mockResolvedValue([]); mockListExportableWatchHistory.mockResolvedValue([]); mockListExportableMyList.mockResolvedValue([]); + mockListSyncQueueForProvider.mockResolvedValue([]); mockResolveSimklIds.mockResolvedValue({ simkl: 10, imdb: 'tt10' }); }); @@ -77,14 +104,13 @@ describe('simkl sync service', () => { it('skips category when cursor matches activity timestamp', async () => { // Arrange const cursor = '2026-01-01T00:00:00.000Z'; + const cursorsObj = { plantowatch: cursor }; // Act await runImport( 'profile-1', 'token', - 'client', - { movies: cursor, shows: cursor, anime: cursor }, - undefined + { movies: cursorsObj, tv_shows: cursorsObj, anime: cursorsObj } ); // Assert @@ -93,35 +119,32 @@ describe('simkl sync service', () => { it('fetches category when no cursor exists', async () => { // Arrange / Act - await runImport('profile-1', 'token', 'client'); + await runImport('profile-1', 'token'); // Assert expect(mockGetAllItems).toHaveBeenCalledTimes(3); - expect(mockGetAllItems).toHaveBeenNthCalledWith(1, 'token', 'client', 'movies', undefined); - expect(mockGetAllItems).toHaveBeenNthCalledWith(2, 'token', 'client', 'shows', undefined); - expect(mockGetAllItems).toHaveBeenNthCalledWith(3, 'token', 'client', 'anime', undefined); + expect(mockGetAllItems).toHaveBeenNthCalledWith(1, 'token', 'movies', undefined, 'full'); + expect(mockGetAllItems).toHaveBeenNthCalledWith(2, 'token', 'shows', undefined, 'full'); + expect(mockGetAllItems).toHaveBeenNthCalledWith(3, 'token', 'anime', undefined, 'full'); }); it('fetches category when activity timestamp is newer than cursor', async () => { // Arrange mockGetActivities.mockResolvedValueOnce({ - all: '2026-02-01T00:00:00.000Z', - movies: { all: '2026-02-01T00:00:00.000Z' }, - tv_shows: { all: '2026-02-01T00:00:00.000Z' }, - anime: { all: '2026-02-01T00:00:00.000Z' }, + movies: { plantowatch: '2026-02-01T00:00:00.000Z' }, + tv_shows: { plantowatch: '2026-02-01T00:00:00.000Z' }, + anime: { plantowatch: '2026-02-01T00:00:00.000Z' }, }); // Act await runImport( 'profile-1', 'token', - 'client', { - movies: '2026-01-01T00:00:00.000Z', - shows: '2026-01-01T00:00:00.000Z', - anime: '2026-01-01T00:00:00.000Z', - }, - undefined + movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, + tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, + anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, + } ); // Assert @@ -129,21 +152,22 @@ describe('simkl sync service', () => { expect(mockGetAllItems).toHaveBeenNthCalledWith( 1, 'token', - 'client', 'movies', - '2026-01-01T00:00:00.000Z' + '2026-01-01T00:00:00.000Z', + 'full' ); }); it('imports movie items correctly', async () => { // Arrange - mockGetAllItems.mockImplementation((_token: string, _clientId: string, type: string) => { + mockGetAllItems.mockImplementation((_token: string, type: string) => { if (type === 'movies') { return Promise.resolve({ movies: [ { movie: { ids: { simkl: 555 }, title: 'Movie' }, last_watched_at: '2026-03-01T10:00:00.000Z', + status: 'watching', }, ], }); @@ -152,7 +176,7 @@ describe('simkl sync service', () => { }); // Act - await runImport('profile-1', 'token', 'client'); + await runImport('profile-1', 'token'); // Assert expect(mockUpsertWatchProgress).toHaveBeenCalledWith( @@ -166,12 +190,13 @@ describe('simkl sync service', () => { it('imports show episodes correctly with videoId metaId:season:episode', async () => { // Arrange - mockGetAllItems.mockImplementation((_token: string, _clientId: string, type: string) => { + mockGetAllItems.mockImplementation((_token: string, type: string) => { if (type === 'shows') { return Promise.resolve({ shows: [ { show: { ids: { imdb: 'tt12345' }, title: 'Show' }, + status: 'completed', seasons: [ { number: 2, episodes: [{ number: 3, watched_at: '2026-03-02T00:00:00.000Z' }] }, ], @@ -183,7 +208,7 @@ describe('simkl sync service', () => { }); // Act - await runImport('profile-1', 'token', 'client'); + await runImport('profile-1', 'token'); // Assert expect(mockUpsertWatchProgress).toHaveBeenCalledWith( @@ -198,12 +223,13 @@ describe('simkl sync service', () => { it('imports counts-only show items as meta-level series history row', async () => { // Arrange - mockGetAllItems.mockImplementation((_token: string, _clientId: string, type: string) => { + mockGetAllItems.mockImplementation((_token: string, type: string) => { if (type === 'shows') { return Promise.resolve({ shows: [ { show: { ids: { imdb: 'tt1695360' }, title: 'Gravity Falls' }, + status: 'watching', watched_episodes_count: 40, last_watched_at: '2026-03-03T12:34:56.000Z', }, @@ -214,7 +240,7 @@ describe('simkl sync service', () => { }); // Act - await runImport('profile-1', 'token', 'client'); + await runImport('profile-1', 'token'); // Assert expect(mockUpsertWatchProgress).toHaveBeenCalledWith( @@ -230,7 +256,7 @@ describe('simkl sync service', () => { it('filters items by status: plantowatch to My List, dropped ignored', async () => { // Arrange - mockGetAllItems.mockImplementation((_token: string, _clientId: string, type: string) => { + mockGetAllItems.mockImplementation((_token: string, type: string) => { if (type === 'movies') { return Promise.resolve({ movies: [ @@ -254,7 +280,7 @@ describe('simkl sync service', () => { }); // Act - await runImport('profile-1', 'token', 'client'); + await runImport('profile-1', 'token'); // Assert // Plan to watch -> My List @@ -268,17 +294,15 @@ describe('simkl sync service', () => { }) ); - // Dropped -> Ignored - const allUpsertMetaIds = mockUpsertWatchProgress.mock.calls.map(call => call[0].metaId); - expect(allUpsertMetaIds).not.toContain('222'); - - const allMyListMetaIds = mockAddToMyList.mock.calls.map(call => call[1]); - expect(allMyListMetaIds).not.toContain('222'); + // Dropped -> Ignored (wait, the new logic removes dropped from My List and History!) + // My list removals are called: + expect(mockRemoveFromMyList).toHaveBeenCalledWith('profile-1', '222'); + expect(mockRemoveWatchHistoryMeta).toHaveBeenCalledWith('profile-1', '222'); }); it('clears local history first when clearLocalFirst is true', async () => { // Arrange / Act - await runImport('profile-1', 'token', 'client', undefined, { clearLocalFirst: true }); + await runImport('profile-1', 'token', undefined, { clearLocalFirst: true }); // Assert expect(mockRemoveProfileWatchHistory).toHaveBeenCalledWith('profile-1'); @@ -286,13 +310,13 @@ describe('simkl sync service', () => { it('saves new cursors after successful import', async () => { // Arrange / Act - await runImport('profile-1', 'token', 'client'); + await runImport('profile-1', 'token'); // Assert expect(mockUpdateSimklCursors).toHaveBeenCalledWith('profile-1', { - movies: '2026-01-01T00:00:00.000Z', - shows: '2026-01-01T00:00:00.000Z', - anime: '2026-01-01T00:00:00.000Z', + movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, + tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, + anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, }); }); @@ -301,7 +325,7 @@ describe('simkl sync service', () => { mockGetActivities.mockRejectedValueOnce(new Error('boom')); // Act / Assert - await expect(runImport('profile-1', 'token', 'client')).resolves.toBe(false); + await expect(runImport('profile-1', 'token')).resolves.toBe(false); }); }); @@ -321,10 +345,10 @@ describe('simkl sync service', () => { mockResolveSimklIds.mockResolvedValueOnce({ simkl: 77, imdb: 'tt77' }); // Act - await runExport('profile-1', 'token', 'client'); + await runExport('profile-1', 'token'); // Assert - expect(mockPostHistory).toHaveBeenCalledWith('token', 'client', { + expect(mockPostHistory).toHaveBeenCalledWith('token', { movies: [{ ids: { simkl: 77, imdb: 'tt77' } }], shows: [], }); @@ -335,7 +359,7 @@ describe('simkl sync service', () => { mockListExportableWatchHistory.mockResolvedValueOnce([]); // Act - await runExport('profile-1', 'token', 'client'); + await runExport('profile-1', 'token'); // Assert expect(mockResolveSimklIds).not.toHaveBeenCalled(); @@ -376,10 +400,10 @@ describe('simkl sync service', () => { mockResolveSimklIds.mockResolvedValue({ simkl: 888, imdb: 'tt888' }); // Act - await runExport('profile-1', 'token', 'client'); + await runExport('profile-1', 'token'); // Assert - expect(mockPostHistory).toHaveBeenCalledWith('token', 'client', { + expect(mockPostHistory).toHaveBeenCalledWith('token', { movies: [], shows: [ { @@ -398,7 +422,7 @@ describe('simkl sync service', () => { mockListExportableWatchHistory.mockRejectedValueOnce(new Error('db broken')); // Act / Assert - await expect(runExport('profile-1', 'token', 'client')).resolves.toBe(false); + await expect(runExport('profile-1', 'token')).resolves.toBe(false); }); it('skips items where resolveSimklIds returns null', async () => { @@ -416,7 +440,7 @@ describe('simkl sync service', () => { mockResolveSimklIds.mockResolvedValueOnce(null); // Act - await runExport('profile-1', 'token', 'client'); + await runExport('profile-1', 'token'); // Assert expect(mockPostHistory).not.toHaveBeenCalled(); @@ -435,10 +459,10 @@ describe('simkl sync service', () => { }); // Act - await runExport('profile-1', 'token', 'client'); + await runExport('profile-1', 'token'); // Assert - expect(mockPostWatchlist).toHaveBeenCalledWith('token', 'client', { + expect(mockPostWatchlist).toHaveBeenCalledWith('token', { movies: [{ ids: { simkl: 991 }, to: 'plantowatch' }], shows: [{ ids: { simkl: 992 }, to: 'plantowatch' }], }); diff --git a/src/api/simkl/client.ts b/src/api/simkl/client.ts index d63bbf7..9a57fb6 100644 --- a/src/api/simkl/client.ts +++ b/src/api/simkl/client.ts @@ -5,10 +5,12 @@ import type { SimklUserSettings, SimklActivities, SimklMediaItem, + SimklActivityCategory, } from '@/types/simkl'; -import { SIMKL_APP_NAME } from './config'; +import { SIMKL_APP_NAME, SIMKL_CLIENT_ID } from './config'; import { createDebugLogger } from '@/utils/debug'; import { getInstalledAppVersion } from '@/hooks/useAppInfo'; +import { SimklMediaType } from '@/types/integrations'; const debug = createDebugLogger('SimklClient'); @@ -17,27 +19,54 @@ const BASE_URL = 'https://api.simkl.com'; /** Append required Simkl query params to any path (preserves existing params). */ function withSimklParams(path: string): string { const separator = path.includes('?') ? '&' : '?'; - const appVersion = getInstalledAppVersion() - return `${path}${separator}app-name=${encodeURIComponent(SIMKL_APP_NAME)}&app-version=${encodeURIComponent(appVersion)}`; + const appVersion = getInstalledAppVersion(); + let url = `${path}${separator}app-name=${encodeURIComponent( + SIMKL_APP_NAME + )}&app-version=${encodeURIComponent(appVersion)}`; + + if (!path.includes('client_id=')) { + url += `&client_id=${encodeURIComponent(SIMKL_CLIENT_ID)}`; + } + return url; +} + +// Throttler for POST requests (1 request per second) +let lastPostTime = 0; +async function throttlePost() { + const now = Date.now(); + const diff = now - lastPostTime; + if (diff < 1000) { + await new Promise((resolve) => setTimeout(resolve, 1000 - diff)); + } + lastPostTime = Date.now(); } async function simklFetch( path: string, - options: RequestInit & { token?: string; clientId?: string } = {} + options: RequestInit & { token?: string } = {} ): Promise { - const { token, clientId, ...fetchOptions } = options; - const userAgent = `${SIMKL_APP_NAME}/${getInstalledAppVersion()}` + const { token, ...fetchOptions } = options; + + if (fetchOptions.method === 'POST') { + await throttlePost(); + } + + const userAgent = `${SIMKL_APP_NAME}/${getInstalledAppVersion()}`; const headers: Record = { 'Content-Type': 'application/json', 'User-Agent': userAgent, + 'simkl-api-key': SIMKL_CLIENT_ID, ...(fetchOptions.headers as Record), }; - if (clientId) headers['simkl-api-key'] = clientId; if (token) headers['Authorization'] = `Bearer ${token}`; const url = `${BASE_URL}${withSimklParams(path)}`; - - debug('request', { url, method: fetchOptions.method || 'GET', body: fetchOptions.body ? JSON.stringify(fetchOptions.body) : undefined }); + + debug('request', { + url, + method: fetchOptions.method || 'GET', + body: fetchOptions.body ? JSON.stringify(fetchOptions.body) : undefined, + }); const response = await fetch(url, { ...fetchOptions, headers }); @@ -47,66 +76,72 @@ async function simklFetch( } const data = (await response.json()) as T; - + debug('response', { url, data: JSON.stringify(data) }); return data; } -export function getPinCode(clientId: string): Promise { - return simklFetch(`/oauth/pin?client_id=${encodeURIComponent(clientId)}`); +export function getPinCode(): Promise { + return simklFetch('/oauth/pin'); } -export function pollPin(userCode: string, clientId: string): Promise { - return simklFetch( - `/oauth/pin/${encodeURIComponent(userCode)}?client_id=${encodeURIComponent(clientId)}` - ); +export function pollPin(userCode: string): Promise { + return simklFetch(`/oauth/pin/${encodeURIComponent(userCode)}`); } -export function getUserSettings(token: string, clientId: string): Promise { - return simklFetch('/users/settings', { token, clientId }); +export function getUserSettings(token: string): Promise { + return simklFetch('/users/settings', { token }); } -export function getActivities(token: string, clientId: string): Promise { - return simklFetch('/sync/activities', { token, clientId }); +export function getActivities(token: string): Promise { + return simklFetch('/sync/activities', { token }); } +/** + * Fetch items from Simkl. + */ export function getAllItems( token: string, - clientId: string, - type: 'movies' | 'shows' | 'anime', - dateFrom?: string + type: SimklMediaType, + dateFrom?: string, + extended: 'full' | 'ids_only' = 'full' ): Promise { - const extended = type === 'anime' ? 'full_anime_seasons' : 'full'; - const params = new URLSearchParams({ extended, episode_watched_at: 'yes' }); + const path = `/sync/all-items/${type}`; + const params = new URLSearchParams({ extended }); + if (extended === 'full') { + params.set('episode_watched_at', 'yes'); + } if (dateFrom) params.set('date_from', dateFrom); - return simklFetch(`/sync/all-items/${type}?${params.toString()}`, { + return simklFetch(`${path}?${params.toString()}`, { token, - clientId, }); } -export function postHistory(token: string, clientId: string, payload: object): Promise { +export function postHistory(token: string, payload: object): Promise { return simklFetch('/sync/history', { method: 'POST', token, - clientId, body: JSON.stringify(payload), }); } -export function postWatchlist(token: string, clientId: string, payload: object): Promise { +export function postWatchlist(token: string, payload: object): Promise { return simklFetch('/sync/add-to-list', { method: 'POST', token, - clientId, body: JSON.stringify(payload), }); } +export function removeFromHistory(token: string, payload: object): Promise { + return simklFetch('/sync/history/remove', { + method: 'POST', + token, + body: JSON.stringify(payload), + }); +} -export function searchById(clientId: string, imdbId: string): Promise { - return simklFetch( - `/search/id?imdb=${encodeURIComponent(imdbId)}&client_id=${encodeURIComponent(clientId)}` - ); +export function searchById(imdbId: string): Promise { + return simklFetch(`/search/id?imdb=${encodeURIComponent(imdbId)}`); } diff --git a/src/api/simkl/hooks.ts b/src/api/simkl/hooks.ts index 82a58d1..238c41c 100644 --- a/src/api/simkl/hooks.ts +++ b/src/api/simkl/hooks.ts @@ -64,14 +64,14 @@ export function useSimklPinAuth(onSuccess: (accessToken: string) => void): Simkl cancel(); setStatus('pending'); - const pinData = await getPinCode(SIMKL_CLIENT_ID); + const pinData = await getPinCode(); setUserCode(pinData.user_code); setVerificationUrl(pinData.verification_url); // Start polling pollIntervalRef.current = setInterval(async () => { try { - const result = await pollPin(pinData.user_code, SIMKL_CLIENT_ID); + const result = await pollPin(pinData.user_code); if (result.result === 'OK' && result.access_token) { clearTimers(); setStatus('success'); @@ -132,10 +132,10 @@ export function useSimklSync(profileId?: string): SimklSyncState { let exportOk = true; if (syncMode === 'pull' || syncMode === 'full') { - importOk = await runImport(profileId, accessToken, SIMKL_CLIENT_ID, syncCursors); + importOk = await runImport(profileId, accessToken, syncCursors); } if (syncMode === 'push' || syncMode === 'full') { - exportOk = await runExport(profileId, accessToken, SIMKL_CLIENT_ID); + exportOk = await runExport(profileId, accessToken); } if (importOk && exportOk) { @@ -203,7 +203,7 @@ export async function completeSimklConnection( ): Promise { debug('completeSimklConnection:start', { profileId }); try { - const userSettings = await getUserSettings(accessToken, SIMKL_CLIENT_ID); + const userSettings = await getUserSettings(accessToken); const connection: SimklConnection = { accessToken, userId: String(userSettings.account.id), diff --git a/src/api/simkl/id-resolver.ts b/src/api/simkl/id-resolver.ts index 426f72e..5f695a0 100644 --- a/src/api/simkl/id-resolver.ts +++ b/src/api/simkl/id-resolver.ts @@ -14,12 +14,8 @@ const cache = new Map(); * - Otherwise → passed as-is to /search/id * Results are cached in memory for the app session. */ -export async function resolveSimklIds( - metaId: string, - type: ContentType, - clientId: string -): Promise { - const cacheKey = `${clientId}:${type}:${metaId}`; +export async function resolveSimklIds(metaId: string, type: ContentType): Promise { + const cacheKey = `${type}:${metaId}`; if (cache.has(cacheKey)) { return cache.get(cacheKey) ?? null; } @@ -46,7 +42,7 @@ export async function resolveSimklIds( } try { - const results = await searchById(clientId, metaId); + const results = await searchById(metaId); if (!results || results.length === 0) { debug('notFound', { metaId }); diff --git a/src/api/simkl/sync-service.ts b/src/api/simkl/sync-service.ts index b9233ee..54f6bfe 100644 --- a/src/api/simkl/sync-service.ts +++ b/src/api/simkl/sync-service.ts @@ -1,45 +1,27 @@ import { createDebugLogger } from '@/utils/debug'; -import type { SimklActivities, SimklIds, SimklWatchedItem } from '@/types/simkl'; -import { getActivities, getAllItems, postHistory, postWatchlist } from './client'; +import type { SimklActivities, SimklActivityCategory, SimklIds, SimklWatchedItem, SimklStatus, SimklAllItemsResponse } from '@/types/simkl'; +import { getActivities, getAllItems, postHistory, postWatchlist, removeFromHistory } from './client'; import { resolveSimklIds } from './id-resolver'; import { listExportableWatchHistoryForProfile, listWatchHistoryForProfile, upsertWatchProgress, removeProfileWatchHistory, + removeWatchHistoryMeta, } from '@/db/queries/watchHistory'; -import { addToMyList, listExportableMyListForProfile } from '@/db/queries/myList'; +import { addToMyList, listExportableMyListForProfile, removeFromMyList, removeProfileMyList } from '@/db/queries/myList'; +import { listSyncQueueForProvider, deleteFromSyncQueue } from '@/db/queries/syncQueue'; import { useIntegrationsStore } from '@/store/integrations.store'; -import type { SimklConnection, SimklMediaType, SimklSyncCursors } from '@/types/integrations'; +import type { SimklConnection, SimklMediaType, SimklSyncCursors, SimklSyncCursor } from '@/types/integrations'; import type { ContentType } from '@/types/stremio'; import { parseVideoId } from '@/utils/id'; const debug = createDebugLogger('SimklSyncService'); -const SIMKL_SYNC_CATEGORIES: { - key: keyof SimklSyncCursors; - type: SimklMediaType; - getActivityTimestamp: (activities: SimklActivities) => string | undefined; -}[] = [ - { - key: 'movies', - type: 'movies', - getActivityTimestamp: (activities) => activities.movies?.all, - }, - { - key: 'shows', - type: 'shows', - getActivityTimestamp: (activities) => activities.tv_shows?.all, - }, - { - key: 'anime', - type: 'anime', - getActivityTimestamp: (activities) => activities.anime?.all, - }, -]; - -const IMPORT_PROGRESS_SECONDS = 1; -const IMPORT_DURATION_SECONDS = 1; +const BATCH_SIZE = 50; + +const IMPORT_PROGRESS_SECONDS = 100; +const IMPORT_DURATION_SECONDS = 100; interface HistoryIdsPayload { ids: Record; @@ -57,55 +39,152 @@ interface HistoryPayload { shows: HistoryShowPayload[]; } +const SYNC_STATUSES: (keyof SimklSyncCursor)[] = ['plantowatch', 'watching', 'completed', 'dropped']; + +function needsSync(activityCursor: string | object | undefined, storedCursor?: string): boolean { + if (!activityCursor || typeof activityCursor === 'object') return false; + if (!storedCursor) return true; + return activityCursor > storedCursor; +} -/** - * Import watch history from Simkl into local DB. - * Fail-safe: errors are logged, never thrown. - */ export async function runImport( profileId: string, token: string, - clientId: string, - cursors?: SimklConnection['syncCursors'], + cursors?: SimklSyncCursors, opts?: { clearLocalFirst?: boolean } ): Promise { try { debug('importStart', { profileId, hasCursors: !!cursors }); - // 1. Get current activities timestamps - const activities = await getActivities(token, clientId); - const newCursors: NonNullable = {}; + const activities = await getActivities(token); + const newCursors: SimklSyncCursors = { ...cursors }; if (opts?.clearLocalFirst) { await removeProfileWatchHistory(profileId); + await removeProfileMyList(profileId); } - for (const { key, type, getActivityTimestamp } of SIMKL_SYNC_CATEGORIES) { - const activityTimestamp = getActivityTimestamp(activities); - - const cursor = cursors?.[key]; + const typesToSync: { type: SimklMediaType; key: keyof SimklActivities; responseKey: keyof SimklAllItemsResponse }[] = [ + { type: 'movies', key: 'movies', responseKey: 'movies' }, + { type: 'shows', key: 'tv_shows', responseKey: 'shows' }, + { type: 'anime', key: 'anime', responseKey: 'anime' }, + ]; + + for (const { type, key, responseKey } of typesToSync) { + const typeActivities = activities[key]; + if (!typeActivities) continue; + + const typeCursors = cursors?.[key] || {}; + const newTypeCursors: SimklSyncCursor = { ...typeCursors }; + newCursors[key] = newTypeCursors; + + // 1. Process Removals First + // When items are removed from a Simkl list, the `removed_from_list` activity timestamp updates. + // To identify what was removed, we must fetch the entire list using `extended=ids_only` + // (without a date filter) and compare the returned IDs against our local database. + const removedActivity = typeActivities.removed_from_list; + const removedCursor = typeCursors.removed_from_list; + const hasRemovals = needsSync(removedActivity, removedCursor); + + if (hasRemovals) { + debug('syncRemovals', { profileId, type }); + const idsResponse = await getAllItems(token, type, undefined, 'ids_only'); + const items = idsResponse[responseKey] || []; + const currentMetaIds = new Set(); + for (const item of items) { + const metaId = getMetaIdFromWatchedItem(item); + if (metaId) currentMetaIds.add(metaId); + } + await cleanupRemovedItems(profileId, type, currentMetaIds); + if (typeof removedActivity === 'string') { + newTypeCursors.removed_from_list = removedActivity; + } + } - // Skip if cursor exists and activity hasn't changed - // Also skip if activityTimestamp is falsy (category has no items at all) - if ((cursor && activityTimestamp && cursor >= activityTimestamp) || !activityTimestamp) { - debug('categorySkipped', { key, cursor, activityTimestamp }); - if (cursor) newCursors[key] = cursor; - continue; + // 2. Process Additions and Updates + // Simkl tracks separate activity timestamps for each status (watching, completed, plantowatch, etc.). + // Instead of making multiple API calls per status, we find the oldest outdated cursor among all statuses + // and make a single API call to fetch all items updated since that time. + // Track the earliest timestamp we need to fetch updates from. + let oldestOutdatedCursor: string | undefined; + // If any status is completely missing a cursor, we must do a full sync without a date filter. + let fullSyncRequired = false; + // Flag to track if there is any new activity across all tracked statuses. + let anyUpdates = false; + + for (const status of SYNC_STATUSES) { + const activity = typeActivities[status]; + const cursor = typeCursors[status]; + if (needsSync(activity, cursor)) { + anyUpdates = true; + if (typeof activity === 'string') { + newTypeCursors[status] = activity; + } + if (!cursor) { + fullSyncRequired = true; + } else if (!oldestOutdatedCursor || cursor < oldestOutdatedCursor) { + oldestOutdatedCursor = cursor; + } + } } - debug('fetchingCategory', { key, type, dateFrom: cursor }); - const response = await getAllItems(token, clientId, type, cursor); - const items = response?.[key] ?? []; - debug('fetchedItems', { key, count: items.length }); + if (anyUpdates) { + const dateFrom = fullSyncRequired ? undefined : oldestOutdatedCursor; + debug('syncUpdates', { profileId, type, dateFrom, fullSyncRequired }); + + const itemsResponse = await getAllItems(token, type, dateFrom, 'full'); + const items = itemsResponse[responseKey] || []; + + const myListAdditions: { metaId: string; type: ContentType; addedAt?: number }[] = []; + const historyUpserts: Parameters[0][] = []; + const removals: { metaId: string; type: ContentType }[] = []; + + for (const item of items) { + const contentType: ContentType = type === 'movies' ? 'movie' : 'series'; + const metaId = getMetaIdFromWatchedItem(item); + if (!metaId) continue; + + // 3. Route items based on their status + // 'plantowatch' items are added to My List. + // 'watching' and 'completed' items update the Watch History with watch dates and episode counts. + // 'dropped' items are actively removed from both My List and Watch History. + if (item.status === 'plantowatch') { + myListAdditions.push({ + metaId, + type: contentType, + addedAt: item.added_to_watchlist_at ? new Date(item.added_to_watchlist_at).getTime() : undefined, + }); + } else if (item.status === 'watching' || item.status === 'completed') { + if (type === 'movies' && item.movie) { + const param = collectMovieParam(profileId, item); + if (param) historyUpserts.push(param); + } else if ((type === 'shows' || type === 'anime') && item.show) { + const params = collectShowParams(profileId, item); + historyUpserts.push(...params); + } + } else if (item.status === 'dropped') { + removals.push({ metaId, type: contentType }); + } + } - await importItems(profileId, items, type); + // Apply changes in batches + for (let i = 0; i < myListAdditions.length; i += BATCH_SIZE) { + const batch = myListAdditions.slice(i, i + BATCH_SIZE); + await Promise.all(batch.map((p) => addToMyList(profileId, p.metaId, p.type, p.addedAt))); + } - if (activityTimestamp) { - newCursors[key] = activityTimestamp; + for (let i = 0; i < historyUpserts.length; i += BATCH_SIZE) { + const batch = historyUpserts.slice(i, i + BATCH_SIZE); + await Promise.all(batch.map((p) => upsertImportedProgress(p))); + } + + for (const removal of removals) { + await removeWatchHistoryMeta(profileId, removal.metaId); + await removeFromMyList(profileId, removal.metaId); + } } } - // 6. Save new cursors useIntegrationsStore.getState().updateSimklCursors(profileId, newCursors); debug('importComplete', { profileId, newCursors }); return true; @@ -115,65 +194,30 @@ export async function runImport( } } -async function importItems( +function getMetaIdFromWatchedItem(item: SimklWatchedItem): string | undefined { + const ids = item.movie?.ids ?? item.show?.ids; + if (!ids) return undefined; + return getMetaIdFromIds(ids); +} + +async function cleanupRemovedItems( profileId: string, - items: SimklWatchedItem[], - type: SimklMediaType + type: SimklMediaType, + currentlyImportedMetaIds: Set ): Promise { - // Collect all progress params first, then batch-write - const historyParams: Parameters[0][] = []; - const myListParams: { metaId: string; type: ContentType; addedAt?: number }[] = []; - - for (const item of items) { - try { - const status = item.status; - - // Skip dropped items completely - if (status === 'dropped') { - continue; - } - - // Add plan to watch items to My List - if (status === 'plantowatch') { - const ids = type === 'movies' ? item.movie?.ids : item.show?.ids; - if (!ids) continue; - - const metaId = getMetaIdFromIds(ids); - if (metaId) { - myListParams.push({ - metaId, - type: type === 'movies' ? 'movie' : 'series', - addedAt: item.added_at ? new Date(item.added_at).getTime() : undefined, - }); - } - continue; - } - - // Add watching and completed (and any other status like 'hold' or undefined) to history - if (type === 'movies' && item.movie) { - collectMovieParams(profileId, item, historyParams); - } else if ((type === 'shows' || type === 'anime') && item.show) { - collectShowParams(profileId, item, historyParams); + try { + const localHistory = await listWatchHistoryForProfile(profileId); + const contentType: ContentType = type === 'movies' ? 'movie' : 'series'; + + for (const item of localHistory) { + if (item.source === 'simkl' && item.type === contentType && !currentlyImportedMetaIds.has(item.id)) { + debug('cleanupRemovingItem', { metaId: item.id, type: item.type }); + await removeWatchHistoryMeta(profileId, item.id); + await removeFromMyList(profileId, item.id); } - } catch (error) { - debug('importItemError', { error }); } - } - - const BATCH_SIZE = 50; - - // Write history in parallel batches - for (let i = 0; i < historyParams.length; i += BATCH_SIZE) { - const batch = historyParams.slice(i, i + BATCH_SIZE); - await Promise.all(batch.map((params) => upsertImportedProgress(params))); - } - - // Write My List in parallel batches - for (let i = 0; i < myListParams.length; i += BATCH_SIZE) { - const batch = myListParams.slice(i, i + BATCH_SIZE); - await Promise.all( - batch.map((params) => addToMyList(profileId, params.metaId, params.type, params.addedAt)) - ); + } catch (error) { + debug('cleanupError', { error }); } } @@ -196,26 +240,26 @@ async function upsertImportedProgress(params: { }); } -function collectMovieParams( +function collectMovieParam( profileId: string, item: SimklWatchedItem, - out: Parameters[0][] -): void { +): Parameters[0] | undefined { if (!item.movie) return; const metaId = getMetaIdFromIds(item.movie.ids); if (!metaId) return; - out.push({ profileId, metaId, type: 'movie', watchedAt: item.last_watched_at }); + return { profileId, metaId, type: 'movie', watchedAt: item.last_watched_at }; } function collectShowParams( profileId: string, item: SimklWatchedItem, - out: Parameters[0][] -): void { - if (!item.show) return; +): Parameters[0][] { + const out: Parameters[0][] = []; + + if (!item.show) return out; const metaId = getMetaIdFromIds(item.show.ids); - if (!metaId) return; + if (!metaId) return out; const hasEpisodeArrays = !!item.seasons || !!item.episodes; @@ -231,7 +275,7 @@ function collectShowParams( }); } } - return; + return out; } if (item.episodes) { @@ -249,25 +293,45 @@ function collectShowParams( if (!hasEpisodeArrays && (item.watched_episodes_count ?? 0) > 0) { out.push({ profileId, metaId, type: 'series', watchedAt: item.last_watched_at }); } + + return out; } -function getMetaIdFromIds(ids: { imdb?: string; simkl?: number; kitsu?: number; mal?: number }): string | null { +function getMetaIdFromIds(ids: { imdb?: string; simkl?: number; kitsu?: number; mal?: number }): string | undefined { if (ids.imdb) return ids.imdb; if (ids.kitsu) return `kitsu:${ids.kitsu}`; if (ids.simkl) return String(ids.simkl); - return null; + return undefined; } /** * Export local watch history to Simkl. * Fail-safe: errors are logged, never thrown. */ -export async function runExport(profileId: string, token: string, clientId: string): Promise { +export async function runExport(profileId: string, token: string): Promise { try { debug('exportStart', { profileId }); const lastSyncAt = useIntegrationsStore.getState().lastSyncAt[profileId] || 0; - + + // 0. Export Removals (Sync Queue) + const queueItems = await listSyncQueueForProvider(profileId, 'simkl'); + const newRemovals = queueItems.filter((q) => q.createdAt > lastSyncAt); + + if (newRemovals.length > 0) { + debug('exportRemovals', { count: newRemovals.length }); + const removalPayload = await buildRemovalPayload(newRemovals); + + if (removalPayload.movies.length > 0 || removalPayload.shows.length > 0) { + await removeFromHistory(token, removalPayload); + debug('exportRemovalsComplete', { movies: removalPayload.movies.length, shows: removalPayload.shows.length }); + } + + // Cleanup processed items + const processedIds = newRemovals.map(r => r.id); + await deleteFromSyncQueue(processedIds); + } + // 1. Export Watch History const completed = await listExportableWatchHistoryForProfile(profileId, { status: 'completed', @@ -277,10 +341,10 @@ export async function runExport(profileId: string, token: string, clientId: stri debug('exportHistoryItems', { completed: completed.length }); - const historyPayload = await buildExportPayload(completed, clientId); + const historyPayload = await buildExportPayload(completed); if (historyPayload.movies.length > 0 || historyPayload.shows.length > 0) { - await postHistory(token, clientId, historyPayload); + await postHistory(token, historyPayload); debug('exportHistoryComplete', { movies: historyPayload.movies.length, shows: historyPayload.shows.length }); } @@ -291,10 +355,10 @@ export async function runExport(profileId: string, token: string, clientId: stri debug('exportWatchlistItems', { count: myListItems.length }); - const watchlistPayload = await buildWatchlistPayload(myListItems, clientId); + const watchlistPayload = await buildWatchlistPayload(myListItems); if (watchlistPayload.movies.length > 0 || watchlistPayload.shows.length > 0) { - await postWatchlist(token, clientId, watchlistPayload); + await postWatchlist(token, watchlistPayload); debug('exportWatchlistComplete', { movies: watchlistPayload.movies.length, shows: watchlistPayload.shows.length }); } @@ -306,14 +370,13 @@ export async function runExport(profileId: string, token: string, clientId: stri } async function buildWatchlistPayload( - items: Awaited>, - clientId: string + items: Awaited> ): Promise { const movies: any[] = []; const shows: any[] = []; for (const item of items) { - const ids = await resolveSimklIds(item.id, item.type, clientId); + const ids = await resolveSimklIds(item.id, item.type); if (!ids || Object.keys(ids).length === 0) { debug('exportWatchlistItemNotFound', { metaId: item.id }); continue; @@ -332,15 +395,79 @@ async function buildWatchlistPayload( return { movies, shows } as any; } +async function buildRemovalPayload( + removals: { metaId: string; type: ContentType; videoId: string | null }[] +): Promise { + const movies: HistoryIdsPayload[] = []; + const showsMap = new Map; seasons: Map> }>(); + + for (const item of removals) { + const ids = await resolveSimklIds(item.metaId, item.type); + if (!ids || Object.keys(ids).length === 0) { + debug('exportRemovalItemNotFound', { metaId: item.metaId }); + continue; + } + + const payloadIds = toHistoryIdsPayload(ids); + if (!payloadIds) { + debug('exportRemovalItemInvalidIds', { metaId: item.metaId }); + continue; + } + + if (item.type === 'movie') { + movies.push({ ids: payloadIds }); + continue; + } + + if (!item.videoId) { + // Remove whole show + if (!showsMap.has(item.metaId)) { + showsMap.set(item.metaId, { ids: payloadIds, seasons: new Map() }); + } + continue; + } + + const episodeRef = parseVideoId(item.videoId); + if (!episodeRef) continue; + + if (!showsMap.has(item.metaId)) { + showsMap.set(item.metaId, { + ids: payloadIds, + seasons: new Map(), + }); + } + const show = showsMap.get(item.metaId); + if (!show) continue; + if (!show.seasons.has(episodeRef.season)) { + show.seasons.set(episodeRef.season, new Set()); + } + show.seasons.get(episodeRef.season)?.add(episodeRef.episode); + } + + const shows: HistoryShowPayload[] = Array.from(showsMap.values()).map((show) => { + if (show.seasons.size === 0) { + return { ids: show.ids } as unknown as HistoryShowPayload; // Valid for whole show removal + } + return { + ids: show.ids, + seasons: Array.from(show.seasons.entries()).map(([seasonNum, episodes]) => ({ + number: seasonNum, + episodes: Array.from(episodes).map((episodeNumber) => ({ number: episodeNumber })), + })), + }; + }); + + return { movies, shows }; +} + async function buildExportPayload( - completed: Awaited>, - clientId: string + completed: Awaited> ): Promise { const movies: HistoryIdsPayload[] = []; const showsMap = new Map; seasons: Map> }>(); for (const item of completed) { - const ids = await resolveSimklIds(item.id, item.type, clientId); + const ids = await resolveSimklIds(item.id, item.type); if (!ids || Object.keys(ids).length === 0) { debug('exportItemNotFound', { metaId: item.id }); continue; diff --git a/src/components/settings/SimklFirstConnectModal.tsx b/src/components/settings/SimklFirstConnectModal.tsx index 0d5e352..00c5f4b 100644 --- a/src/components/settings/SimklFirstConnectModal.tsx +++ b/src/components/settings/SimklFirstConnectModal.tsx @@ -65,12 +65,12 @@ export const SimklFirstConnectModal: FC = memo( useIntegrationsStore.getState().connectSimkl(profileId, connection, choice.syncMode); if (choice.id === 'import' || choice.id === 'full') { - await runImport(profileId, connection.accessToken, SIMKL_CLIENT_ID, undefined, { + await runImport(profileId, connection.accessToken, undefined, { clearLocalFirst, }); } if (choice.id === 'export' || choice.id === 'full') { - await runExport(profileId, connection.accessToken, SIMKL_CLIENT_ID); + await runExport(profileId, connection.accessToken); } await Promise.all([ diff --git a/src/db/__tests__/myList.test.ts b/src/db/__tests__/myList.test.ts index 9a8e872..682a604 100644 --- a/src/db/__tests__/myList.test.ts +++ b/src/db/__tests__/myList.test.ts @@ -18,10 +18,22 @@ import { type DbMyListItem, } from '../queries/myList'; import { upsertMetaCache } from '../queries/metaCache'; -import { myList, metaCache } from '../schema'; +import { myList, metaCache, syncQueue } from '../schema'; import { and, eq } from 'drizzle-orm'; import type { MetaDetail } from '@/types/stremio'; +jest.mock('@/store/integrations.store', () => ({ + useIntegrationsStore: { + getState: () => ({ + settings: { + 'mylist-test-profile': { + simkl: { connection: true }, + }, + }, + }), + }, +})); + describe('myList queries (integration)', () => { const testProfileId = 'mylist-test-profile'; @@ -34,6 +46,7 @@ describe('myList queries (integration)', () => { await db.delete(myList).where(eq(myList.profileId, testProfileId)); await db.delete(myList).where(eq(myList.profileId, 'mylist-profile-2')); await db.delete(metaCache); + await db.delete(syncQueue); }); describe('addToMyList', () => { @@ -124,6 +137,26 @@ describe('myList queries (integration)', () => { expect(secondAdd.addedAt).toBeGreaterThan(firstAddedAt); }); + + it('cancels pending remove_watchlist actions in syncQueue', async () => { + await db.insert(syncQueue).values({ + profileId: testProfileId, + provider: 'simkl', + action: 'remove_watchlist', + metaId: 'tt-cancel-queue', + type: 'movie', + createdAt: Date.now(), + }); + + await addToMyList(testProfileId, 'tt-cancel-queue', 'movie'); + + const queue = await db + .select() + .from(syncQueue) + .where(eq(syncQueue.metaId, 'tt-cancel-queue')); + + expect(queue).toHaveLength(0); + }); }); describe('removeFromMyList', () => { @@ -153,6 +186,21 @@ describe('myList queries (integration)', () => { expect(results).toHaveLength(1); }); + + it('adds remove_watchlist action to syncQueue for active providers', async () => { + await addToMyList(testProfileId, 'tt-sync-queue', 'movie'); + await removeFromMyList(testProfileId, 'tt-sync-queue'); + + const queue = await db + .select() + .from(syncQueue) + .where(eq(syncQueue.metaId, 'tt-sync-queue')); + + expect(queue).toHaveLength(1); + expect(queue[0].action).toBe('remove_watchlist'); + expect(queue[0].provider).toBe('simkl'); + expect(queue[0].type).toBe('movie'); + }); }); describe('listMyListForProfile', () => { diff --git a/src/db/__tests__/watchHistory.test.ts b/src/db/__tests__/watchHistory.test.ts index 4eae717..4c30bd2 100644 --- a/src/db/__tests__/watchHistory.test.ts +++ b/src/db/__tests__/watchHistory.test.ts @@ -27,10 +27,28 @@ import { } from '../queries/watchHistory'; import { upsertMetaCache } from '../queries/metaCache'; import { initializeDatabase, db } from '../client'; -import { watchHistory, metaCache, videos } from '../schema'; +import { watchHistory, metaCache, videos, syncQueue } from '../schema'; import { eq, and } from 'drizzle-orm'; import type { MetaDetail } from '@/types/stremio'; +jest.mock('@/store/integrations.store', () => ({ + useIntegrationsStore: { + getState: () => ({ + settings: { + 'test-profile-1': { + simkl: { connection: true }, + }, + 'remove-item-profile': { + simkl: { connection: true }, + }, + 'remove-meta-profile': { + simkl: { connection: true }, + }, + }, + }), + }, +})); + describe('watchHistory queries (integration)', () => { const testProfileId = 'test-profile-1'; @@ -42,6 +60,7 @@ describe('watchHistory queries (integration)', () => { // Clean up test data await db.delete(watchHistory).where(eq(watchHistory.profileId, testProfileId)); await db.delete(watchHistory).where(eq(watchHistory.profileId, 'test-profile-2')); + await db.delete(syncQueue); }); describe('upsertWatchProgress', () => { @@ -345,6 +364,43 @@ describe('watchHistory queries (integration)', () => { expect(results).toHaveLength(2); }); }); + + it('cancels pending remove_history and remove_watchlist actions in syncQueue', async () => { + // First manually insert a syncQueue item + await db.insert(syncQueue).values([ + { + profileId: testProfileId, + provider: 'simkl', + action: 'remove_history', + metaId: 'tt-cancel-hist', + type: 'movie', + createdAt: Date.now(), + }, + { + profileId: testProfileId, + provider: 'simkl', + action: 'remove_watchlist', + metaId: 'tt-cancel-hist', + type: 'movie', + createdAt: Date.now(), + }, + ]); + + await upsertWatchProgress({ + profileId: testProfileId, + metaId: 'tt-cancel-hist', + type: 'movie', + progressSeconds: 100, + durationSeconds: 1000, + }); + + const queue = await db + .select() + .from(syncQueue) + .where(eq(syncQueue.metaId, 'tt-cancel-hist')); + + expect(queue).toHaveLength(0); + }); }); describe('dismissFromContinueWatching', () => { @@ -842,6 +898,29 @@ describe('removeWatchHistoryItem (integration)', () => { expect(results).toHaveLength(1); }); + + it('adds remove_history action to syncQueue for active providers', async () => { + await upsertWatchProgress({ + profileId: testProfileId, + metaId: 'tt-remove-sync', + videoId: 'ep-1', + type: 'series', + progressSeconds: 500, + durationSeconds: 1000, + }); + + await removeWatchHistoryItem(testProfileId, 'tt-remove-sync', 'ep-1'); + + const queue = await db + .select() + .from(syncQueue) + .where(and(eq(syncQueue.metaId, 'tt-remove-sync'), eq(syncQueue.videoId, 'ep-1'))); + + expect(queue).toHaveLength(1); + expect(queue[0].action).toBe('remove_history'); + expect(queue[0].provider).toBe('simkl'); + expect(queue[0].type).toBe('series'); + }); }); describe('removeWatchHistoryMeta (integration)', () => { @@ -913,6 +992,29 @@ describe('removeWatchHistoryMeta (integration)', () => { expect(remaining).toHaveLength(1); expect(remaining[0].metaId).toBe('tt-meta-remove-b'); }); + + it('adds remove_history action to syncQueue for active providers', async () => { + await upsertWatchProgress({ + profileId: testProfileId, + metaId: 'tt-remove-meta-sync', + videoId: undefined, + type: 'movie', + progressSeconds: 500, + durationSeconds: 1000, + }); + + await removeWatchHistoryMeta(testProfileId, 'tt-remove-meta-sync'); + + const queue = await db + .select() + .from(syncQueue) + .where(eq(syncQueue.metaId, 'tt-remove-meta-sync')); + + expect(queue).toHaveLength(1); + expect(queue[0].action).toBe('remove_history'); + expect(queue[0].provider).toBe('simkl'); + expect(queue[0].type).toBe('movie'); + }); }); describe('setLastStreamTarget (integration)', () => { diff --git a/src/db/drizzle/0001_fluffy_karma.sql b/src/db/drizzle/0001_fluffy_karma.sql new file mode 100644 index 0000000..6fb9803 --- /dev/null +++ b/src/db/drizzle/0001_fluffy_karma.sql @@ -0,0 +1,42 @@ +CREATE TABLE `sync_queue` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `profile_id` text NOT NULL, + `provider` text NOT NULL, + `action` text NOT NULL, + `meta_id` text NOT NULL, + `video_id` text, + `type` text NOT NULL, + `created_at` integer NOT NULL +); +--> statement-breakpoint +CREATE INDEX `profile_provider_idx` ON `sync_queue` (`profile_id`,`provider`);--> statement-breakpoint +DROP INDEX `meta_id_idx`;--> statement-breakpoint +DROP INDEX `profile_added_idx`;--> statement-breakpoint +CREATE INDEX `profile_added_idx` ON `my_list` (`profile_id`,`added_at`);--> statement-breakpoint +ALTER TABLE `my_list` DROP COLUMN `removed_at`;--> statement-breakpoint +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_watch_history` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `profile_id` text NOT NULL, + `meta_id` text NOT NULL, + `video_id` text DEFAULT '' NOT NULL, + `type` text NOT NULL, + `progress_seconds` real DEFAULT 0 NOT NULL, + `duration_seconds` real DEFAULT 0 NOT NULL, + `last_stream_target_type` text, + `last_stream_target_value` text, + `status` text DEFAULT 'watching' NOT NULL, + `source` text DEFAULT 'internal' NOT NULL, + `last_watched_at` integer NOT NULL, + `dismissed_at` integer, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +INSERT INTO `__new_watch_history`("id", "profile_id", "meta_id", "video_id", "type", "progress_seconds", "duration_seconds", "last_stream_target_type", "last_stream_target_value", "status", "source", "last_watched_at", "dismissed_at", "created_at", "updated_at") SELECT "id", "profile_id", "meta_id", "video_id", "type", "progress_seconds", "duration_seconds", "last_stream_target_type", "last_stream_target_value", "status", "source", "last_watched_at", "dismissed_at", "created_at", "updated_at" FROM `watch_history`;--> statement-breakpoint +DROP TABLE `watch_history`;--> statement-breakpoint +ALTER TABLE `__new_watch_history` RENAME TO `watch_history`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `profile_status_last_watched_idx` ON `watch_history` (`profile_id`,`status`,`last_watched_at`);--> statement-breakpoint +CREATE INDEX `profile_meta_idx` ON `watch_history` (`profile_id`,`meta_id`);--> statement-breakpoint +CREATE UNIQUE INDEX `watch_history_profile_id_meta_id_video_id_unique` ON `watch_history` (`profile_id`,`meta_id`,`video_id`); \ No newline at end of file diff --git a/src/db/drizzle/meta/0001_snapshot.json b/src/db/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..d1a6dda --- /dev/null +++ b/src/db/drizzle/meta/0001_snapshot.json @@ -0,0 +1,516 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "02b7d567-1824-4ac5-aa69-8ba8bd6a80d2", + "prevId": "77173fea-26ab-41b2-a65b-20c2e82617a8", + "tables": { + "meta_cache": { + "name": "meta_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "poster": { + "name": "poster", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "background": { + "name": "background", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imdb_rating": { + "name": "imdb_rating", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "release_year": { + "name": "release_year", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "meta_cache_meta_id_unique": { + "name": "meta_cache_meta_id_unique", + "columns": [ + "meta_id" + ], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "my_list": { + "name": "my_list", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "profile_added_idx": { + "name": "profile_added_idx", + "columns": [ + "profile_id", + "added_at" + ], + "isUnique": false + }, + "my_list_profile_id_meta_id_unique": { + "name": "my_list_profile_id_meta_id_unique", + "columns": [ + "profile_id", + "meta_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_queue": { + "name": "sync_queue", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "video_id": { + "name": "video_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "profile_provider_idx": { + "name": "profile_provider_idx", + "columns": [ + "profile_id", + "provider" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "video_id": { + "name": "video_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "season": { + "name": "season", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "episode": { + "name": "episode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "overview": { + "name": "overview", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "released": { + "name": "released", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "video_meta_id_idx": { + "name": "video_meta_id_idx", + "columns": [ + "meta_id" + ], + "isUnique": false + }, + "meta_season_episode_idx": { + "name": "meta_season_episode_idx", + "columns": [ + "meta_id", + "season", + "episode" + ], + "isUnique": false + }, + "videos_meta_id_video_id_unique": { + "name": "videos_meta_id_video_id_unique", + "columns": [ + "meta_id", + "video_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "watch_history": { + "name": "watch_history", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "video_id": { + "name": "video_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress_seconds": { + "name": "progress_seconds", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "duration_seconds": { + "name": "duration_seconds", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_stream_target_type": { + "name": "last_stream_target_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_stream_target_value": { + "name": "last_stream_target_value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'watching'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'internal'" + }, + "last_watched_at": { + "name": "last_watched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dismissed_at": { + "name": "dismissed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "profile_status_last_watched_idx": { + "name": "profile_status_last_watched_idx", + "columns": [ + "profile_id", + "status", + "last_watched_at" + ], + "isUnique": false + }, + "profile_meta_idx": { + "name": "profile_meta_idx", + "columns": [ + "profile_id", + "meta_id" + ], + "isUnique": false + }, + "watch_history_profile_id_meta_id_video_id_unique": { + "name": "watch_history_profile_id_meta_id_video_id_unique", + "columns": [ + "profile_id", + "meta_id", + "video_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/db/drizzle/meta/_journal.json b/src/db/drizzle/meta/_journal.json index bc90a94..2d3a297 100644 --- a/src/db/drizzle/meta/_journal.json +++ b/src/db/drizzle/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1771357438859, "tag": "0000_curious_lockheed", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1777122090693, + "tag": "0001_fluffy_karma", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/src/db/drizzle/migrations.ts b/src/db/drizzle/migrations.ts index b651b6b..33764b6 100644 --- a/src/db/drizzle/migrations.ts +++ b/src/db/drizzle/migrations.ts @@ -16,6 +16,13 @@ const journal = { tag: '0001_add_source_column', breakpoints: true, }, + { + idx: 2, + version: '6', + when: 1771357438861, + tag: '0002_add_sync_queue', + breakpoints: true, + }, ], }; @@ -87,10 +94,24 @@ CREATE UNIQUE INDEX \`watch_history_profile_id_meta_id_video_id_unique\` ON \`wa const m0001 = `ALTER TABLE \`watch_history\` ADD \`source\` text NOT NULL DEFAULT 'internal';`; +const m0002 = `CREATE TABLE \`sync_queue\` ( + \`id\` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + \`profile_id\` text NOT NULL, + \`provider\` text NOT NULL, + \`action\` text NOT NULL, + \`meta_id\` text NOT NULL, + \`video_id\` text, + \`type\` text NOT NULL, + \`created_at\` integer NOT NULL +); +--> statement-breakpoint +CREATE INDEX \`profile_provider_idx\` ON \`sync_queue\` (\`profile_id\`,\`provider\`);`; + export default { journal, migrations: { m0000, m0001, + m0002, }, }; diff --git a/src/db/queries/myList.ts b/src/db/queries/myList.ts index 2cfb5ec..e596be9 100644 --- a/src/db/queries/myList.ts +++ b/src/db/queries/myList.ts @@ -2,6 +2,7 @@ import { and, desc, eq, sql } from 'drizzle-orm'; import type { ContentType } from '@/types/stremio'; import { db, initializeDatabase } from '@/db/client'; import { metaCache, myList } from '@/db/schema'; +import { addToSyncQueue, cancelPendingSyncRemovals, type SyncProvider } from './syncQueue'; export type DbMyListItem = { id: string; @@ -66,6 +67,8 @@ export async function addToMyList( addedAt: now, }, }); + + await cancelPendingSyncRemovals(profileId, metaId, ['remove_watchlist']); } export async function countMyListForProfile(profileId: string): Promise { @@ -105,10 +108,23 @@ export async function listExportableMyListForProfile( })); } -export async function removeFromMyList(profileId: string, metaId: string): Promise { +export async function removeFromMyList( + profileId: string, + metaId: string, + ignoreProvider?: SyncProvider +): Promise { await initializeDatabase(); + const item = await db + .select({ type: myList.type }) + .from(myList) + .where(and(eq(myList.profileId, profileId), eq(myList.metaId, metaId))) + .limit(1); + + if (item.length === 0) return; + await db.delete(myList).where(and(eq(myList.profileId, profileId), eq(myList.metaId, metaId))); + await addToSyncQueue(profileId, 'remove_watchlist', metaId, item[0].type, undefined, ignoreProvider); } export async function removeProfileMyList(profileId: string): Promise { diff --git a/src/db/queries/syncQueue.ts b/src/db/queries/syncQueue.ts new file mode 100644 index 0000000..4c717ca --- /dev/null +++ b/src/db/queries/syncQueue.ts @@ -0,0 +1,85 @@ +import { and, eq, inArray } from 'drizzle-orm'; +import { db, initializeDatabase } from '@/db/client'; +import { syncQueue } from '@/db/schema'; +import type { ContentType } from '@/types/stremio'; +import { useIntegrationsStore } from '@/store/integrations.store'; + +export type SyncAction = 'remove_history' | 'remove_watchlist'; +export type SyncProvider = 'simkl'; + +export async function addToSyncQueue( + profileId: string, + action: SyncAction, + metaId: string, + type: ContentType, + videoId?: string, + ignoreProvider?: SyncProvider +): Promise { + await initializeDatabase(); + + // Get active providers for the profile to know which queues to populate + const state = useIntegrationsStore.getState(); + const activeProviders: SyncProvider[] = []; + + if (state.settings[profileId]?.simkl?.connection && ignoreProvider !== 'simkl') { + activeProviders.push('simkl'); + } + + if (activeProviders.length === 0) return; + + const now = Date.now(); + const inserts = activeProviders.map(provider => ({ + profileId, + provider, + action, + metaId, + videoId, + type, + createdAt: now, + })); + + await db.insert(syncQueue).values(inserts); +} +export async function cancelPendingSyncRemovals( + profileId: string, + metaId: string, + actions: SyncAction[] +): Promise { + await initializeDatabase(); + + for (const action of actions) { + await db + .delete(syncQueue) + .where( + and( + eq(syncQueue.profileId, profileId), + eq(syncQueue.metaId, metaId), + eq(syncQueue.action, action) + ) + ); + } +} + +export async function listSyncQueueForProvider( + profileId: string, + provider: SyncProvider +) { + await initializeDatabase(); + + return db + .select() + .from(syncQueue) + .where( + and( + eq(syncQueue.profileId, profileId), + eq(syncQueue.provider, provider) + ) + ); +} + +export async function deleteFromSyncQueue(ids: number[]): Promise { + if (ids.length === 0) return; + + await initializeDatabase(); + await db.delete(syncQueue).where(inArray(syncQueue.id, ids)); +} diff --git a/src/db/queries/watchHistory.ts b/src/db/queries/watchHistory.ts index c4fbaf8..e6ff37d 100644 --- a/src/db/queries/watchHistory.ts +++ b/src/db/queries/watchHistory.ts @@ -4,6 +4,7 @@ import type { ContentType } from '@/types/stremio'; import { db, initializeDatabase } from '@/db/client'; import { metaCache, videos, watchHistory } from '@/db/schema'; import type { StreamTargetType, WatchHistorySource, WatchHistoryStatus } from '@/db/schema'; +import { addToSyncQueue, cancelPendingSyncRemovals, type SyncProvider } from './syncQueue'; export type DbWatchHistoryItem = { id: string; @@ -219,6 +220,8 @@ export async function upsertWatchProgress(params: { target: [watchHistory.profileId, watchHistory.metaId, watchHistory.videoId], set: updateSet, }); + + await cancelPendingSyncRemovals(params.profileId, params.metaId, ['remove_history', 'remove_watchlist']); } export async function setLastStreamTarget(params: { @@ -292,10 +295,25 @@ export async function undismissFromContinueWatching( export async function removeWatchHistoryItem( profileId: string, metaId: string, - videoId?: string + videoId?: string, + ignoreProvider?: SyncProvider ): Promise { await initializeDatabase(); + const items = await db + .select({ type: watchHistory.type, videoId: watchHistory.videoId }) + .from(watchHistory) + .where( + and( + eq(watchHistory.profileId, profileId), + eq(watchHistory.metaId, metaId), + videoId ? eq(watchHistory.videoId, videoId) : eq(watchHistory.videoId, '') + ) + ) + .limit(1); + + if (items.length === 0) return; + await db .delete(watchHistory) .where( @@ -305,14 +323,30 @@ export async function removeWatchHistoryItem( videoId ? eq(watchHistory.videoId, videoId) : eq(watchHistory.videoId, '') ) ); + + await addToSyncQueue(profileId, 'remove_history', metaId, items[0].type, items[0].videoId, ignoreProvider); } -export async function removeWatchHistoryMeta(profileId: string, metaId: string): Promise { +export async function removeWatchHistoryMeta( + profileId: string, + metaId: string, + ignoreProvider?: SyncProvider +): Promise { await initializeDatabase(); + const items = await db + .select({ type: watchHistory.type }) + .from(watchHistory) + .where(and(eq(watchHistory.profileId, profileId), eq(watchHistory.metaId, metaId))) + .limit(1); + + if (items.length === 0) return; + await db .delete(watchHistory) .where(and(eq(watchHistory.profileId, profileId), eq(watchHistory.metaId, metaId))); + + await addToSyncQueue(profileId, 'remove_history', metaId, items[0].type, undefined, ignoreProvider); } export async function removeProfileWatchHistory(profileId: string): Promise { diff --git a/src/db/schema.ts b/src/db/schema.ts index 6484385..aea021e 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -93,4 +93,22 @@ export const videos = sqliteTable( ] ); +export const syncQueue = sqliteTable( + 'sync_queue', + { + id: integer('id').primaryKey({ autoIncrement: true }), + profileId: text('profile_id').notNull(), + provider: text('provider', { enum: ['simkl'] }).notNull(), // expandable for trakt, mal, etc. + action: text('action', { enum: ['remove_history', 'remove_watchlist'] }).notNull(), + metaId: text('meta_id').notNull(), + videoId: text('video_id'), // Optional: for episode-specific removals + type: contentTypeColumn('type').notNull(), + createdAt: integer('created_at').notNull(), + }, + (table) => [ + index('profile_provider_idx').on(table.profileId, table.provider), + ] +); + export type WatchHistoryStatus = 'watching' | 'dismissed' | 'completed'; + diff --git a/src/store/integrations.store.ts b/src/store/integrations.store.ts index 3823401..58b1d6d 100644 --- a/src/store/integrations.store.ts +++ b/src/store/integrations.store.ts @@ -6,6 +6,7 @@ import type { IntegrationSyncStatus, ProfileIntegrationSettings, SimklConnection, + SimklSyncCursors, SyncMode, } from '@/types/integrations'; import { createDebugLogger } from '@/utils/debug'; @@ -22,7 +23,7 @@ interface IntegrationsState { setSyncMode: (profileId: string, mode: SyncMode) => void; updateSimklCursors: ( profileId: string, - cursors: NonNullable + cursors: SimklSyncCursors ) => void; setLastSyncAt: (profileId: string, timestamp: number) => void; setSyncStatus: ( diff --git a/src/types/integrations.ts b/src/types/integrations.ts index 5e22188..b2aa39c 100644 --- a/src/types/integrations.ts +++ b/src/types/integrations.ts @@ -3,10 +3,18 @@ export type SimklMediaType = 'movies' | 'shows' | 'anime'; export type IntegrationProvider = 'simkl'; export type IntegrationSyncStatus = 'idle' | 'syncing' | 'success' | 'error'; +export interface SimklSyncCursor { + plantowatch?: string; + watching?: string; + completed?: string; + dropped?: string; + removed_from_list?: string; +} + export interface SimklSyncCursors { - movies?: string; - shows?: string; - anime?: string; + tv_shows?: SimklSyncCursor; + movies?: SimklSyncCursor; + anime?: SimklSyncCursor; } export interface SimklConnection { diff --git a/src/types/simkl.ts b/src/types/simkl.ts index c73be90..b66519c 100644 --- a/src/types/simkl.ts +++ b/src/types/simkl.ts @@ -25,11 +25,22 @@ export interface SimklUserSettings { }; } -export interface SimklActivities { +export interface SimklActivityCategory { all: string; - movies?: { all: string }; - tv_shows?: { all: string }; - anime?: { all: string }; + rated_at?: string | object; + playback?: string | object; + plantowatch?: string | object; + watching?: string | object; + completed?: string | object; + hold?: string | object; + dropped?: string | object; + removed_from_list?: string | object; +} + +export interface SimklActivities { + movies?: SimklActivityCategory; + tv_shows?: SimklActivityCategory; + anime?: SimklActivityCategory; } export interface SimklIds { @@ -55,7 +66,7 @@ export type SimklStatus = 'watching' | 'plantowatch' | 'hold' | 'dropped' | 'com export interface SimklWatchedItem { last_watched_at?: string; - added_at?: string; + added_to_watchlist_at?: string; status?: SimklStatus; watched_episodes_count?: number; movie?: { ids: SimklIds; title: string }; From 537caf07ca1d1bdb3eb6bb408df67122ce4a96ec Mon Sep 17 00:00:00 2001 From: Fabian Schliski Date: Sat, 25 Apr 2026 15:15:25 +0200 Subject: [PATCH 2/6] Fix default profile configurations --- src/store/addon.store.ts | 4 +++- src/store/home.store.ts | 4 +++- src/store/playback.store.ts | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/store/addon.store.ts b/src/store/addon.store.ts index 6bd1ec4..a40cf23 100644 --- a/src/store/addon.store.ts +++ b/src/store/addon.store.ts @@ -286,7 +286,9 @@ export const useAddonStore = create()( // Fall back to DEFAULT_ADDON_CONFIG when no per-profile entry exists. // This recovers users whose configsByProfile was not populated during the // broken v2 migration (async migrate is not supported by Zustand's newImpl). - return get().configsByProfile[targetProfileId]?.[id] ?? DEFAULT_ADDON_CONFIG; + const config = get().configsByProfile[targetProfileId]?.[id]; + if (!config) return DEFAULT_ADDON_CONFIG; + return { ...DEFAULT_ADDON_CONFIG, ...config }; }, getAddonsList: () => { diff --git a/src/store/home.store.ts b/src/store/home.store.ts index 2ffa384..95d3ba5 100644 --- a/src/store/home.store.ts +++ b/src/store/home.store.ts @@ -73,7 +73,9 @@ export const useHomeStore = create()( getActiveSettings: () => { const profileId = get().activeProfileId; if (!profileId) return DEFAULT_HOME_SETTINGS; - return get().byProfile[profileId] ?? DEFAULT_HOME_SETTINGS; + const profileSettings = get().byProfile[profileId]; + if (!profileSettings) return DEFAULT_HOME_SETTINGS; + return { ...DEFAULT_HOME_SETTINGS, ...profileSettings }; }, // Active profile mutations diff --git a/src/store/playback.store.ts b/src/store/playback.store.ts index 7809eb4..1a5b10c 100644 --- a/src/store/playback.store.ts +++ b/src/store/playback.store.ts @@ -115,7 +115,9 @@ export const usePlaybackStore = create()( getActiveSettings: () => { const profileId = get().activeProfileId; if (!profileId) return DEFAULT_PROFILE_PLAYBACK_SETTINGS; - return get().byProfile[profileId] ?? DEFAULT_PROFILE_PLAYBACK_SETTINGS; + const profileSettings = get().byProfile[profileId]; + if (!profileSettings) return DEFAULT_PROFILE_PLAYBACK_SETTINGS; + return { ...DEFAULT_PROFILE_PLAYBACK_SETTINGS, ...profileSettings }; }, setPlayer: (player) => { From e020d1e09774835d245914d426ad24b399c96ad2 Mon Sep 17 00:00:00 2001 From: Fabian Schliski Date: Sat, 25 Apr 2026 15:33:20 +0200 Subject: [PATCH 3/6] Caching meta data returned by simkl in partial meta cache --- src/api/simkl/sync-service.ts | 23 +- src/db/__tests__/metaCache.test.ts | 108 ++++ src/db/drizzle/0002_purple_steel_serpent.sql | 1 + src/db/drizzle/meta/0002_snapshot.json | 524 +++++++++++++++++++ src/db/drizzle/meta/_journal.json | 7 + src/db/drizzle/migrations.ts | 10 + src/db/queries/metaCache.ts | 61 ++- src/db/schema.ts | 1 + src/types/simkl.ts | 4 +- src/utils/__tests__/media-artwork.test.ts | 19 + src/utils/media-artwork.ts | 10 + 11 files changed, 760 insertions(+), 8 deletions(-) create mode 100644 src/db/drizzle/0002_purple_steel_serpent.sql create mode 100644 src/db/drizzle/meta/0002_snapshot.json create mode 100644 src/utils/__tests__/media-artwork.test.ts diff --git a/src/api/simkl/sync-service.ts b/src/api/simkl/sync-service.ts index 54f6bfe..f3a010f 100644 --- a/src/api/simkl/sync-service.ts +++ b/src/api/simkl/sync-service.ts @@ -1,4 +1,5 @@ import { createDebugLogger } from '@/utils/debug'; +import { getSimklPosterUrl } from '@/utils/media-artwork'; import type { SimklActivities, SimklActivityCategory, SimklIds, SimklWatchedItem, SimklStatus, SimklAllItemsResponse } from '@/types/simkl'; import { getActivities, getAllItems, postHistory, postWatchlist, removeFromHistory } from './client'; import { resolveSimklIds } from './id-resolver'; @@ -9,8 +10,14 @@ import { removeProfileWatchHistory, removeWatchHistoryMeta, } from '@/db/queries/watchHistory'; -import { addToMyList, listExportableMyListForProfile, removeFromMyList, removeProfileMyList } from '@/db/queries/myList'; +import { + listExportableMyListForProfile, + removeFromMyList, + removeProfileMyList, + addToMyList, +} from '@/db/queries/myList'; import { listSyncQueueForProvider, deleteFromSyncQueue } from '@/db/queries/syncQueue'; +import { upsertMinimalMetaCache } from '@/db/queries/metaCache'; import { useIntegrationsStore } from '@/store/integrations.store'; import type { SimklConnection, SimklMediaType, SimklSyncCursors, SimklSyncCursor } from '@/types/integrations'; import type { ContentType } from '@/types/stremio'; @@ -144,6 +151,20 @@ export async function runImport( const metaId = getMetaIdFromWatchedItem(item); if (!metaId) continue; + const title = item.movie?.title ?? item.show?.title; + const posterValue = + item.movie?.poster ?? + item.show?.poster; + if (title) { + await upsertMinimalMetaCache({ + metaId, + type: contentType, + name: title, + poster: getSimklPosterUrl(posterValue), + year: (item.movie?.year ?? item.show?.year)?.toString(), + }); + } + // 3. Route items based on their status // 'plantowatch' items are added to My List. // 'watching' and 'completed' items update the Watch History with watch dates and episode counts. diff --git a/src/db/__tests__/metaCache.test.ts b/src/db/__tests__/metaCache.test.ts index f5b2515..ec8d179 100644 --- a/src/db/__tests__/metaCache.test.ts +++ b/src/db/__tests__/metaCache.test.ts @@ -14,6 +14,7 @@ import { isMetaCacheStale, getStaleMetaIds, getVideoForEntry, + upsertMinimalMetaCache, } from '../queries/metaCache'; import { metaCache, videos } from '../schema'; import { eq } from 'drizzle-orm'; @@ -242,6 +243,66 @@ describe('metaCache queries (integration)', () => { }); }); + describe('upsertMinimalMetaCache', () => { + it('inserts minimal metadata correctly', async () => { + await upsertMinimalMetaCache({ + metaId: 'tt_minimal', + type: 'movie', + name: 'Minimal Movie', + poster: 'https://example.com/min.jpg', + year: '2024', + }); + + const [cached] = await db.select().from(metaCache).where(eq(metaCache.metaId, 'tt_minimal')); + expect(cached.name).toBe('Minimal Movie'); + expect(cached.isPartial).toBe(true); + expect(cached.poster).toBe('https://example.com/min.jpg'); + expect(cached.releaseYear).toBe('2024'); + }); + + it('does not overwrite full metadata with minimal metadata', async () => { + const fullMeta: MetaDetail = { + id: 'tt_full', + type: 'movie', + name: 'Full Movie', + description: 'Full description', + poster: 'https://example.com/full.jpg', + }; + await upsertMetaCache(fullMeta); + + await upsertMinimalMetaCache({ + metaId: 'tt_full', + type: 'movie', + name: 'Minimal Attempt', + }); + + const [cached] = await db.select().from(metaCache).where(eq(metaCache.metaId, 'tt_full')); + expect(cached.name).toBe('Full Movie'); + expect(cached.isPartial).toBe(false); + expect(cached.description).toBe('Full description'); + }); + + it('updates partial metadata with more partial metadata', async () => { + await upsertMinimalMetaCache({ + metaId: 'tt_partial', + type: 'movie', + name: 'Partial 1', + }); + + await upsertMinimalMetaCache({ + metaId: 'tt_partial', + type: 'movie', + name: 'Partial 2', + year: '2025', + }); + + const [cached] = await db.select().from(metaCache).where(eq(metaCache.metaId, 'tt_partial')); + expect(cached.name).toBe('Partial 2'); + expect(cached.releaseYear).toBe('2025'); + expect(cached.isPartial).toBe(true); + }); + }); + describe('isMetaCacheStale', () => { it('returns true when cache does not exist', async () => { const result = await isMetaCacheStale('nonexistent-meta'); @@ -293,6 +354,28 @@ describe('metaCache queries (integration)', () => { expect(result).toBe(true); }); + + it('returns true for partial metadata even if not expired (default behavior)', async () => { + await upsertMinimalMetaCache({ + metaId: 'tt_partial_stale', + type: 'movie', + name: 'Partial', + }); + + const result = await isMetaCacheStale('tt_partial_stale'); + expect(result).toBe(true); + }); + + it('returns false for partial metadata when allowPartial is true', async () => { + await upsertMinimalMetaCache({ + metaId: 'tt_partial_allowed', + type: 'movie', + name: 'Partial', + }); + + const result = await isMetaCacheStale('tt_partial_allowed', { allowPartial: true }); + expect(result).toBe(false); + }); }); describe('getStaleMetaIds', () => { @@ -367,6 +450,31 @@ describe('metaCache queries (integration)', () => { expect(result).not.toContain('valid-mix'); expect(result).toHaveLength(2); }); + + it('returns IDs that are partial (default behavior)', async () => { + await upsertMinimalMetaCache({ + metaId: 'partial-stale', + type: 'movie', + name: 'Partial', + }); + + const result = await getStaleMetaIds(['partial-stale']); + + expect(result).toContain('partial-stale'); + }); + + it('does not return IDs that are partial when allowPartial is true', async () => { + await upsertMinimalMetaCache({ + metaId: 'partial-valid', + type: 'movie', + name: 'Partial', + }); + + const result = await getStaleMetaIds(['partial-valid'], { allowPartial: true }); + + expect(result).not.toContain('partial-valid'); + expect(result).toHaveLength(0); + }); }); }); diff --git a/src/db/drizzle/0002_purple_steel_serpent.sql b/src/db/drizzle/0002_purple_steel_serpent.sql new file mode 100644 index 0000000..7957cd5 --- /dev/null +++ b/src/db/drizzle/0002_purple_steel_serpent.sql @@ -0,0 +1 @@ +ALTER TABLE `meta_cache` ADD `is_partial` integer DEFAULT false NOT NULL; \ No newline at end of file diff --git a/src/db/drizzle/meta/0002_snapshot.json b/src/db/drizzle/meta/0002_snapshot.json new file mode 100644 index 0000000..d89019d --- /dev/null +++ b/src/db/drizzle/meta/0002_snapshot.json @@ -0,0 +1,524 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5714603a-181e-4ffd-92df-96efc50ebc2b", + "prevId": "02b7d567-1824-4ac5-aa69-8ba8bd6a80d2", + "tables": { + "meta_cache": { + "name": "meta_cache", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "poster": { + "name": "poster", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "background": { + "name": "background", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imdb_rating": { + "name": "imdb_rating", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "release_year": { + "name": "release_year", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_partial": { + "name": "is_partial", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "meta_cache_meta_id_unique": { + "name": "meta_cache_meta_id_unique", + "columns": [ + "meta_id" + ], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "my_list": { + "name": "my_list", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "added_at": { + "name": "added_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "profile_added_idx": { + "name": "profile_added_idx", + "columns": [ + "profile_id", + "added_at" + ], + "isUnique": false + }, + "my_list_profile_id_meta_id_unique": { + "name": "my_list_profile_id_meta_id_unique", + "columns": [ + "profile_id", + "meta_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_queue": { + "name": "sync_queue", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "video_id": { + "name": "video_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "profile_provider_idx": { + "name": "profile_provider_idx", + "columns": [ + "profile_id", + "provider" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "video_id": { + "name": "video_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "season": { + "name": "season", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "episode": { + "name": "episode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "overview": { + "name": "overview", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "released": { + "name": "released", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "video_meta_id_idx": { + "name": "video_meta_id_idx", + "columns": [ + "meta_id" + ], + "isUnique": false + }, + "meta_season_episode_idx": { + "name": "meta_season_episode_idx", + "columns": [ + "meta_id", + "season", + "episode" + ], + "isUnique": false + }, + "videos_meta_id_video_id_unique": { + "name": "videos_meta_id_video_id_unique", + "columns": [ + "meta_id", + "video_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "watch_history": { + "name": "watch_history", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meta_id": { + "name": "meta_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "video_id": { + "name": "video_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "progress_seconds": { + "name": "progress_seconds", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "duration_seconds": { + "name": "duration_seconds", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_stream_target_type": { + "name": "last_stream_target_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_stream_target_value": { + "name": "last_stream_target_value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'watching'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'internal'" + }, + "last_watched_at": { + "name": "last_watched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dismissed_at": { + "name": "dismissed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "profile_status_last_watched_idx": { + "name": "profile_status_last_watched_idx", + "columns": [ + "profile_id", + "status", + "last_watched_at" + ], + "isUnique": false + }, + "profile_meta_idx": { + "name": "profile_meta_idx", + "columns": [ + "profile_id", + "meta_id" + ], + "isUnique": false + }, + "watch_history_profile_id_meta_id_video_id_unique": { + "name": "watch_history_profile_id_meta_id_video_id_unique", + "columns": [ + "profile_id", + "meta_id", + "video_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/db/drizzle/meta/_journal.json b/src/db/drizzle/meta/_journal.json index 2d3a297..a60b69c 100644 --- a/src/db/drizzle/meta/_journal.json +++ b/src/db/drizzle/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1777122090693, "tag": "0001_fluffy_karma", "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1777123204029, + "tag": "0002_purple_steel_serpent", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/drizzle/migrations.ts b/src/db/drizzle/migrations.ts index 33764b6..181907a 100644 --- a/src/db/drizzle/migrations.ts +++ b/src/db/drizzle/migrations.ts @@ -23,6 +23,13 @@ const journal = { tag: '0002_add_sync_queue', breakpoints: true, }, + { + idx: 3, + version: '6', + when: 1771357438862, + tag: '0003_add_is_partial', + breakpoints: true, + }, ], }; @@ -107,11 +114,14 @@ const m0002 = `CREATE TABLE \`sync_queue\` ( --> statement-breakpoint CREATE INDEX \`profile_provider_idx\` ON \`sync_queue\` (\`profile_id\`,\`provider\`);`; +const m0003 = `ALTER TABLE \`meta_cache\` ADD \`is_partial\` integer DEFAULT false NOT NULL;`; + export default { journal, migrations: { m0000, m0001, m0002, + m0003, }, }; diff --git a/src/db/queries/metaCache.ts b/src/db/queries/metaCache.ts index adddd5f..68f6704 100644 --- a/src/db/queries/metaCache.ts +++ b/src/db/queries/metaCache.ts @@ -33,6 +33,7 @@ export async function upsertMetaCache( logo: meta.logo, imdbRating: meta.imdbRating, releaseYear: meta.releaseInfo?.split('–')[0], + isPartial: false, fetchedAt: now, expiresAt, }) @@ -47,6 +48,7 @@ export async function upsertMetaCache( logo: meta.logo, imdbRating: meta.imdbRating, releaseYear: meta.releaseInfo?.split('–')[0], + isPartial: false, fetchedAt: now, expiresAt, }, @@ -91,20 +93,63 @@ export async function upsertMetaCache( } } -export async function isMetaCacheStale(metaId: string): Promise { +export async function upsertMinimalMetaCache(params: { + metaId: string; + type: 'movie' | 'series'; + name: string; + poster?: string; + year?: string; +}): Promise { + await initializeDatabase(); + + const now = Date.now(); + const expiresAt = now + CACHE_TTL_MS; + + await db + .insert(metaCache) + .values({ + metaId: params.metaId, + type: params.type, + name: params.name, + poster: params.poster, + releaseYear: params.year, + isPartial: true, + fetchedAt: now, + expiresAt, + }) + .onConflictDoUpdate({ + target: metaCache.metaId, + set: { + // Only update if current entry is also partial + name: sql`CASE WHEN ${metaCache.isPartial} THEN excluded.name ELSE ${metaCache.name} END`, + poster: sql`CASE WHEN ${metaCache.isPartial} THEN COALESCE(excluded.poster, ${metaCache.poster}) ELSE ${metaCache.poster} END`, + releaseYear: sql`CASE WHEN ${metaCache.isPartial} THEN COALESCE(excluded.release_year, ${metaCache.releaseYear}) ELSE ${metaCache.releaseYear} END`, + fetchedAt: sql`CASE WHEN ${metaCache.isPartial} THEN excluded.fetched_at ELSE ${metaCache.fetchedAt} END`, + }, + }); +} + +export async function isMetaCacheStale( + metaId: string, + options: { allowPartial?: boolean } = {} +): Promise { await initializeDatabase(); const rows = await db - .select({ expiresAt: metaCache.expiresAt }) + .select({ expiresAt: metaCache.expiresAt, isPartial: metaCache.isPartial }) .from(metaCache) .where(eq(metaCache.metaId, metaId)) .limit(1); if (!rows.length) return true; + if (rows[0].isPartial && !options.allowPartial) return true; return Date.now() > Number(rows[0].expiresAt); } -export async function getStaleMetaIds(metaIds: string[]): Promise { +export async function getStaleMetaIds( + metaIds: string[], + options: { allowPartial?: boolean } = {} +): Promise { await initializeDatabase(); if (metaIds.length === 0) return []; @@ -119,14 +164,20 @@ export async function getStaleMetaIds(metaIds: string[]): Promise { const validRows = await db .select({ metaId: metaCache.metaId }) .from(metaCache) - .where(and(inArray(metaCache.metaId, chunk), gte(metaCache.expiresAt, now))); + .where( + and( + inArray(metaCache.metaId, chunk), + gte(metaCache.expiresAt, now), + options.allowPartial ? sql`1=1` : eq(metaCache.isPartial, false) + ) + ); for (const row of validRows) { validSet.add(row.metaId); } } - // Return all metaIds that are NOT in the valid set (either expired or missing) + // Return all metaIds that are NOT in the valid set (either expired, missing, or partial when not allowed) return metaIds.filter((id) => !validSet.has(id)); } diff --git a/src/db/schema.ts b/src/db/schema.ts index aea021e..5e1cb4d 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -66,6 +66,7 @@ export const metaCache = sqliteTable( logo: text('logo'), imdbRating: text('imdb_rating'), releaseYear: text('release_year'), + isPartial: integer('is_partial', { mode: 'boolean' }).notNull().default(false), fetchedAt: integer('fetched_at').notNull(), expiresAt: integer('expires_at').notNull(), }, diff --git a/src/types/simkl.ts b/src/types/simkl.ts index b66519c..4cf7d3c 100644 --- a/src/types/simkl.ts +++ b/src/types/simkl.ts @@ -69,8 +69,8 @@ export interface SimklWatchedItem { added_to_watchlist_at?: string; status?: SimklStatus; watched_episodes_count?: number; - movie?: { ids: SimklIds; title: string }; - show?: { ids: SimklIds; title: string }; + movie?: { ids: SimklIds; title: string; poster?: string; year?: number }; + show?: { ids: SimklIds; title: string; poster?: string; year?: number }; seasons?: { number: number; episodes: { number: number; watched_at?: string }[]; diff --git a/src/utils/__tests__/media-artwork.test.ts b/src/utils/__tests__/media-artwork.test.ts new file mode 100644 index 0000000..2ada9a2 --- /dev/null +++ b/src/utils/__tests__/media-artwork.test.ts @@ -0,0 +1,19 @@ +import { getSimklPosterUrl } from '../media-artwork'; + +describe('getSimklPosterUrl', () => { + it('maps a Simkl poster path to a full URL', () => { + const poster = '12/129597638d4467f431'; + const expected = 'https://simkl.in/posters/12/129597638d4467f431_ca.jpg'; + expect(getSimklPosterUrl(poster)).toBe(expected); + }); + + it('returns undefined if poster is null or undefined', () => { + expect(getSimklPosterUrl(undefined)).toBeUndefined(); + expect(getSimklPosterUrl(null)).toBeUndefined(); + }); + + it('returns the same string if it already starts with http', () => { + const url = 'https://example.com/poster.jpg'; + expect(getSimklPosterUrl(url)).toBe(url); + }); +}); diff --git a/src/utils/media-artwork.ts b/src/utils/media-artwork.ts index e11ee30..b42c54e 100644 --- a/src/utils/media-artwork.ts +++ b/src/utils/media-artwork.ts @@ -17,3 +17,13 @@ export const getDetailsLogoSource = (logo: DetailsLogoInput['logo']) => { const uri = logo; return getImageSource(uri); }; + +/** + * Maps a Simkl poster value to a full URL. + * Example: "12/129597638d4467f431" -> "https://simkl.in/posters/12/129597638d4467f431_ca.jpg" + */ +export const getSimklPosterUrl = (poster: string | undefined | null) => { + if (!poster) return undefined; + if (poster.startsWith('http')) return poster; + return `https://simkl.in/posters/${poster}_ca.jpg`; +}; From 66596dbb1e3b77ad3fdb98de46f932aee85bb447 Mon Sep 17 00:00:00 2001 From: Fabian Schliski Date: Sat, 25 Apr 2026 17:03:45 +0200 Subject: [PATCH 4/6] Fix shows not syncing correctly with SIMKL --- src/api/simkl/__tests__/sync-service.test.ts | 174 ++++++++++++++++++- src/api/simkl/sync-service.ts | 96 +++++++--- src/db/queries/watchHistory.ts | 10 ++ src/types/integrations.ts | 1 + src/types/simkl.ts | 3 + 5 files changed, 250 insertions(+), 34 deletions(-) diff --git a/src/api/simkl/__tests__/sync-service.test.ts b/src/api/simkl/__tests__/sync-service.test.ts index a81a0a7..0ccd8b2 100644 --- a/src/api/simkl/__tests__/sync-service.test.ts +++ b/src/api/simkl/__tests__/sync-service.test.ts @@ -286,7 +286,8 @@ describe('simkl sync service', () => { // Plan to watch -> My List expect(mockAddToMyList).toHaveBeenCalledWith('profile-1', '111', 'movie', undefined); - // Watching -> History + // Watching -> My List AND History + expect(mockAddToMyList).toHaveBeenCalledWith('profile-1', '333', 'movie', undefined); expect(mockUpsertWatchProgress).toHaveBeenCalledWith( expect.objectContaining({ profileId: 'profile-1', @@ -294,12 +295,44 @@ describe('simkl sync service', () => { }) ); - // Dropped -> Ignored (wait, the new logic removes dropped from My List and History!) - // My list removals are called: + // Dropped -> Removals expect(mockRemoveFromMyList).toHaveBeenCalledWith('profile-1', '222'); expect(mockRemoveWatchHistoryMeta).toHaveBeenCalledWith('profile-1', '222'); }); + it('imports anime items correctly using the anime property', async () => { + // Arrange + mockGetAllItems.mockImplementation((_token: string, type: string) => { + if (type === 'anime') { + return Promise.resolve({ + anime: [ + { + anime: { ids: { kitsu: 123 }, title: 'Anime Series' }, + status: 'watching', + seasons: [ + { number: 1, episodes: [{ number: 1, watched_at: '2026-03-01T10:00:00.000Z' }] }, + ], + }, + ], + }); + } + return Promise.resolve({}); + }); + + // Act + await runImport('profile-1', 'token'); + + // Assert + expect(mockUpsertWatchProgress).toHaveBeenCalledWith( + expect.objectContaining({ + profileId: 'profile-1', + metaId: 'kitsu:123', + videoId: 'kitsu:123:1:1', + type: 'series', + }) + ); + }); + it('clears local history first when clearLocalFirst is true', async () => { // Arrange / Act await runImport('profile-1', 'token', undefined, { clearLocalFirst: true }); @@ -308,16 +341,115 @@ describe('simkl sync service', () => { expect(mockRemoveProfileWatchHistory).toHaveBeenCalledWith('profile-1'); }); - it('saves new cursors after successful import', async () => { - // Arrange / Act + it('deduplicates metaIds during cleanup', async () => { + // Arrange + mockGetActivities.mockResolvedValueOnce({ + movies: { removed_from_list: '2026-02-01T00:00:00.000Z' }, + }); + mockGetAllItems.mockResolvedValueOnce({ movies: [] }); // Simkl list is empty + mockListWatchHistory.mockResolvedValueOnce([ + { id: 'movie-1', source: 'simkl', type: 'movie' }, + { id: 'movie-1', source: 'simkl', type: 'movie' }, // Duplicate metaId + ]); + + // Act + await runImport('profile-1', 'token', { + movies: { removed_from_list: '2026-01-01T00:00:00.000Z' }, + }); + + // Assert + expect(mockRemoveWatchHistoryMeta).toHaveBeenCalledTimes(1); + expect(mockRemoveWatchHistoryMeta).toHaveBeenCalledWith('profile-1', 'movie-1'); + }); + + it('imports items with hold status correctly', async () => { + // Arrange + mockGetAllItems.mockImplementation((_token: string, type: string) => { + if (type === 'shows') { + return Promise.resolve({ + shows: [ + { + show: { ids: { imdb: 'tt123' }, title: 'Hold Show' }, + status: 'hold', + watched_episodes_count: 5, + last_watched_at: '2026-03-01T10:00:00.000Z', + }, + ], + }); + } + return Promise.resolve({}); + }); + + // Act await runImport('profile-1', 'token'); // Assert - expect(mockUpdateSimklCursors).toHaveBeenCalledWith('profile-1', { - movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, - tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, - anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, + expect(mockUpsertWatchProgress).toHaveBeenCalledWith( + expect.objectContaining({ + profileId: 'profile-1', + metaId: 'tt123', + type: 'series', + }) + ); + // Hold should also be in My List + expect(mockAddToMyList).toHaveBeenCalledWith('profile-1', 'tt123', 'series', undefined); + }); + + it('parses last_watched string into videoId when episode arrays are missing', async () => { + // Arrange + mockGetAllItems.mockImplementation((_token: string, type: string) => { + if (type === 'shows') { + return Promise.resolve({ + shows: [ + { + show: { ids: { imdb: 'tt1317187' }, title: 'The Last of Us' }, + status: 'watching', + last_watched: 'S01E09', + last_watched_at: '2025-06-22T11:52:39Z', + }, + ], + }); + } + return Promise.resolve({}); + }); + + // Act + await runImport('profile-1', 'token'); + + // Assert + expect(mockUpsertWatchProgress).toHaveBeenCalledWith( + expect.objectContaining({ + metaId: 'tt1317187', + videoId: 'tt1317187:1:9', + type: 'series', + }) + ); + }); + + it('saves all new cursors after successful import', async () => { + // Arrange + mockGetActivities.mockResolvedValue({ + movies: { + all: '2026-04-25T14:15:00Z', + watched_at: '2026-04-25T14:14:00Z', + plantowatch: '2026-04-25T14:13:00Z', + hold: '2026-04-25T14:12:00Z', + }, }); + + // Act + await runImport('profile-1', 'token'); + + // Assert + expect(mockUpdateSimklCursors).toHaveBeenCalledWith( + 'profile-1', + expect.objectContaining({ + movies: expect.objectContaining({ + plantowatch: '2026-04-25T14:13:00Z', + hold: '2026-04-25T14:12:00Z', + }), + }) + ); }); it('does not throw on error (fail-safe)', async () => { @@ -327,6 +459,30 @@ describe('simkl sync service', () => { // Act / Assert await expect(runImport('profile-1', 'token')).resolves.toBe(false); }); + + it('handles null response from getAllItems gracefully', async () => { + // Arrange + mockGetActivities.mockResolvedValueOnce({ + movies: { plantowatch: '2026-02-01T00:00:00.000Z' }, + tv_shows: { plantowatch: '2026-02-01T00:00:00.000Z' }, + anime: { plantowatch: '2026-02-01T00:00:00.000Z' }, + }); + // Simulate API returning null (as seen in logs) + mockGetAllItems.mockResolvedValue(null); + + // Act + const result = await runImport('profile-1', 'token', { + movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, + tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, + anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, + }); + + // Assert + expect(result).toBe(true); + expect(mockGetAllItems).toHaveBeenCalled(); + // Should not have crashed and should have updated cursors + expect(mockUpdateSimklCursors).toHaveBeenCalled(); + }); }); describe('runExport', () => { diff --git a/src/api/simkl/sync-service.ts b/src/api/simkl/sync-service.ts index f3a010f..f809a18 100644 --- a/src/api/simkl/sync-service.ts +++ b/src/api/simkl/sync-service.ts @@ -46,7 +46,13 @@ interface HistoryPayload { shows: HistoryShowPayload[]; } -const SYNC_STATUSES: (keyof SimklSyncCursor)[] = ['plantowatch', 'watching', 'completed', 'dropped']; +const SYNC_STATUSES: (keyof SimklSyncCursor)[] = [ + 'plantowatch', + 'watching', + 'completed', + 'hold', + 'dropped', +]; function needsSync(activityCursor: string | object | undefined, storedCursor?: string): boolean { if (!activityCursor || typeof activityCursor === 'object') return false; @@ -96,7 +102,7 @@ export async function runImport( if (hasRemovals) { debug('syncRemovals', { profileId, type }); const idsResponse = await getAllItems(token, type, undefined, 'ids_only'); - const items = idsResponse[responseKey] || []; + const items = idsResponse?.[responseKey] || []; const currentMetaIds = new Set(); for (const item of items) { const metaId = getMetaIdFromWatchedItem(item); @@ -140,7 +146,7 @@ export async function runImport( debug('syncUpdates', { profileId, type, dateFrom, fullSyncRequired }); const itemsResponse = await getAllItems(token, type, dateFrom, 'full'); - const items = itemsResponse[responseKey] || []; + const items = itemsResponse?.[responseKey] || []; const myListAdditions: { metaId: string; type: ContentType; addedAt?: number }[] = []; const historyUpserts: Parameters[0][] = []; @@ -151,35 +157,34 @@ export async function runImport( const metaId = getMetaIdFromWatchedItem(item); if (!metaId) continue; - const title = item.movie?.title ?? item.show?.title; - const posterValue = - item.movie?.poster ?? - item.show?.poster; - if (title) { + const mediaData = item.movie ?? item.show ?? item.anime; + if (mediaData?.title) { await upsertMinimalMetaCache({ metaId, type: contentType, - name: title, - poster: getSimklPosterUrl(posterValue), - year: (item.movie?.year ?? item.show?.year)?.toString(), + name: mediaData.title, + poster: getSimklPosterUrl(mediaData.poster), + year: mediaData.year?.toString(), }); } // 3. Route items based on their status - // 'plantowatch' items are added to My List. - // 'watching' and 'completed' items update the Watch History with watch dates and episode counts. + // 'plantowatch', 'watching' and 'hold' items are added to My List. + // 'watching', 'completed' and 'hold' items update the Watch History with watch dates and episode counts. // 'dropped' items are actively removed from both My List and Watch History. - if (item.status === 'plantowatch') { + if (item.status === 'plantowatch' || item.status === 'watching' || item.status === 'hold') { myListAdditions.push({ metaId, type: contentType, addedAt: item.added_to_watchlist_at ? new Date(item.added_to_watchlist_at).getTime() : undefined, }); - } else if (item.status === 'watching' || item.status === 'completed') { + } + + if (item.status === 'watching' || item.status === 'completed' || item.status === 'hold') { if (type === 'movies' && item.movie) { const param = collectMovieParam(profileId, item); if (param) historyUpserts.push(param); - } else if ((type === 'shows' || type === 'anime') && item.show) { + } else if (type === 'shows' || type === 'anime') { const params = collectShowParams(profileId, item); historyUpserts.push(...params); } @@ -216,7 +221,7 @@ export async function runImport( } function getMetaIdFromWatchedItem(item: SimklWatchedItem): string | undefined { - const ids = item.movie?.ids ?? item.show?.ids; + const ids = item.movie?.ids ?? item.show?.ids ?? item.anime?.ids; if (!ids) return undefined; return getMetaIdFromIds(ids); } @@ -230,13 +235,22 @@ async function cleanupRemovedItems( const localHistory = await listWatchHistoryForProfile(profileId); const contentType: ContentType = type === 'movies' ? 'movie' : 'series'; + const itemsToRemove = new Set(); for (const item of localHistory) { - if (item.source === 'simkl' && item.type === contentType && !currentlyImportedMetaIds.has(item.id)) { - debug('cleanupRemovingItem', { metaId: item.id, type: item.type }); - await removeWatchHistoryMeta(profileId, item.id); - await removeFromMyList(profileId, item.id); + if ( + item.source === 'simkl' && + item.type === contentType && + !currentlyImportedMetaIds.has(item.id) + ) { + itemsToRemove.add(item.id); } } + + for (const metaId of itemsToRemove) { + debug('cleanupRemovingItem', { metaId, type: contentType }); + await removeWatchHistoryMeta(profileId, metaId); + await removeFromMyList(profileId, metaId); + } } catch (error) { debug('cleanupError', { error }); } @@ -272,14 +286,33 @@ function collectMovieParam( return { profileId, metaId, type: 'movie', watchedAt: item.last_watched_at }; } +function parseSimklLastWatched(lastWatched: string): { season: number; episode: number } | undefined { + const match = lastWatched.match(/S(\d+)E(\d+)/i); + if (match) { + return { + season: parseInt(match[1], 10), + episode: parseInt(match[2], 10), + }; + } + const matchE = lastWatched.match(/E(\d+)/i); + if (matchE) { + return { + season: 1, + episode: parseInt(matchE[1], 10), + }; + } + return undefined; +} + function collectShowParams( profileId: string, item: SimklWatchedItem, ): Parameters[0][] { const out: Parameters[0][] = []; - if (!item.show) return out; - const metaId = getMetaIdFromIds(item.show.ids); + const mediaData = item.show ?? item.anime; + if (!mediaData) return out; + const metaId = getMetaIdFromIds(mediaData.ids); if (!metaId) return out; const hasEpisodeArrays = !!item.seasons || !!item.episodes; @@ -311,8 +344,21 @@ function collectShowParams( } } - if (!hasEpisodeArrays && (item.watched_episodes_count ?? 0) > 0) { - out.push({ profileId, metaId, type: 'series', watchedAt: item.last_watched_at }); + if (!hasEpisodeArrays) { + if (item.last_watched) { + const parsed = parseSimklLastWatched(item.last_watched); + if (parsed) { + out.push({ + profileId, + metaId, + videoId: `${metaId}:${parsed.season}:${parsed.episode}`, + type: 'series', + watchedAt: item.last_watched_at, + }); + } + } else if ((item.watched_episodes_count ?? 0) > 0) { + out.push({ profileId, metaId, type: 'series', watchedAt: item.last_watched_at }); + } } return out; diff --git a/src/db/queries/watchHistory.ts b/src/db/queries/watchHistory.ts index e6ff37d..ca49d59 100644 --- a/src/db/queries/watchHistory.ts +++ b/src/db/queries/watchHistory.ts @@ -756,6 +756,7 @@ export async function getContinueWatchingWithUpNext( metaBackground: metaCache.background, currentVideoSeason: videos.season, currentVideoEpisode: videos.episode, + videoCount: sql`(SELECT count(*) FROM ${videos} WHERE ${videos.metaId} = ${ranked.metaId})`.as('video_count'), }) .from(ranked) .leftJoin(metaCache, eq(ranked.metaId, metaCache.metaId)) @@ -830,7 +831,16 @@ export async function getContinueWatchingWithUpNext( progressRatio: 0, isUpNext: true, }); + continue; } + + // Fallback: If it's a series and we don't have the next episode, but we also + // don't have any metadata (videos) for this show, keep it in the list. + if (item.type === 'series' && Number(item.videoCount ?? 0) === 0) { + resolved.push(base); + continue; + } + // Finished with no next episode — skip continue; } diff --git a/src/types/integrations.ts b/src/types/integrations.ts index b2aa39c..f50f7fd 100644 --- a/src/types/integrations.ts +++ b/src/types/integrations.ts @@ -7,6 +7,7 @@ export interface SimklSyncCursor { plantowatch?: string; watching?: string; completed?: string; + hold?: string; dropped?: string; removed_from_list?: string; } diff --git a/src/types/simkl.ts b/src/types/simkl.ts index 4cf7d3c..2939ead 100644 --- a/src/types/simkl.ts +++ b/src/types/simkl.ts @@ -68,9 +68,12 @@ export interface SimklWatchedItem { last_watched_at?: string; added_to_watchlist_at?: string; status?: SimklStatus; + last_watched?: string; + next_to_watch?: string; watched_episodes_count?: number; movie?: { ids: SimklIds; title: string; poster?: string; year?: number }; show?: { ids: SimklIds; title: string; poster?: string; year?: number }; + anime?: { ids: SimklIds; title: string; poster?: string; year?: number }; seasons?: { number: number; episodes: { number: number; watched_at?: string }[]; From ff9c774a70a01069bb10edc1963ca8ec990ecb3b Mon Sep 17 00:00:00 2001 From: Fabian Schliski Date: Sat, 25 Apr 2026 18:46:04 +0200 Subject: [PATCH 5/6] Fix shows being deleted by anime sync --- .../simkl/__tests__/sync-service.e2e.test.ts | 354 ++++++++++++++++++ src/api/simkl/__tests__/sync-service.test.ts | 57 ++- src/api/simkl/client.ts | 4 +- src/api/simkl/sync-service.ts | 261 +++++++++---- src/types/integrations.ts | 2 + 5 files changed, 582 insertions(+), 96 deletions(-) create mode 100644 src/api/simkl/__tests__/sync-service.e2e.test.ts diff --git a/src/api/simkl/__tests__/sync-service.e2e.test.ts b/src/api/simkl/__tests__/sync-service.e2e.test.ts new file mode 100644 index 0000000..be3c645 --- /dev/null +++ b/src/api/simkl/__tests__/sync-service.e2e.test.ts @@ -0,0 +1,354 @@ +import { runExport, runImport } from '../sync-service'; + +// Mock dependencies +const mockGetActivities = jest.fn(); +const mockGetAllItems = jest.fn(); +const mockPostHistory = jest.fn(); +const mockPostWatchlist = jest.fn(); +const mockRemoveFromHistory = jest.fn(); + +jest.mock('../client', () => ({ + getActivities: (...args: any[]) => mockGetActivities(...args), + getAllItems: (...args: any[]) => mockGetAllItems(...args), + postHistory: (...args: any[]) => mockPostHistory(...args), + postWatchlist: (...args: any[]) => mockPostWatchlist(...args), + removeFromHistory: (...args: any[]) => mockRemoveFromHistory(...args), +})); + +const mockResolveSimklIds = jest.fn(); +jest.mock('../id-resolver', () => ({ + resolveSimklIds: (...args: any[]) => mockResolveSimklIds(...args), +})); + +const mockUpsertWatchProgress = jest.fn(); +const mockListWatchHistory = jest.fn(); +const mockListExportableWatchHistory = jest.fn(); +const mockRemoveProfileWatchHistory = jest.fn(); +const mockRemoveWatchHistoryMeta = jest.fn(); +jest.mock('@/db/queries/watchHistory', () => ({ + upsertWatchProgress: (...args: any[]) => mockUpsertWatchProgress(...args), + listWatchHistoryForProfile: (...args: any[]) => mockListWatchHistory(...args), + listExportableWatchHistoryForProfile: (...args: any[]) => mockListExportableWatchHistory(...args), + removeProfileWatchHistory: (...args: any[]) => mockRemoveProfileWatchHistory(...args), + removeWatchHistoryMeta: (...args: any[]) => mockRemoveWatchHistoryMeta(...args), +})); + +const mockAddToSyncQueue = jest.fn(); +const mockCancelPendingSyncRemovals = jest.fn(); +const mockListSyncQueueForProvider = jest.fn(); +const mockDeleteFromSyncQueue = jest.fn(); +jest.mock('@/db/queries/syncQueue', () => ({ + addToSyncQueue: (...args: any[]) => mockAddToSyncQueue(...args), + cancelPendingSyncRemovals: (...args: any[]) => mockCancelPendingSyncRemovals(...args), + listSyncQueueForProvider: (...args: any[]) => mockListSyncQueueForProvider(...args), + deleteFromSyncQueue: (...args: any[]) => mockDeleteFromSyncQueue(...args), +})); + +const mockAddToMyList = jest.fn(); +const mockListExportableMyList = jest.fn(); +const mockRemoveFromMyList = jest.fn(); +const mockRemoveProfileMyList = jest.fn(); +jest.mock('@/db/queries/myList', () => ({ + addToMyList: (...args: any[]) => mockAddToMyList(...args), + listExportableMyListForProfile: (...args: any[]) => mockListExportableMyList(...args), + removeFromMyList: (...args: any[]) => mockRemoveFromMyList(...args), + removeProfileMyList: (...args: any[]) => mockRemoveProfileMyList(...args), +})); + +jest.mock('@/db/queries/metaCache', () => ({ + upsertMinimalMetaCache: jest.fn(), +})); + +const mockUpdateSimklCursors = jest.fn(); +jest.mock('@/store/integrations.store', () => ({ + useIntegrationsStore: { + getState: () => ({ + updateSimklCursors: mockUpdateSimklCursors, + lastSyncAt: {}, + }), + }, +})); + +describe('Simkl Sync Service - Comprehensive E2E', () => { + beforeEach(() => { + jest.clearAllMocks(); + + // Default activities + mockGetActivities.mockResolvedValue({ + movies: { all: '2026-04-25T15:00:00Z' }, + tv_shows: { all: '2026-04-25T15:00:00Z' }, + anime: { all: '2026-04-25T15:00:00Z' }, + }); + + // Default empty items + mockGetAllItems.mockResolvedValue({}); + mockListWatchHistory.mockResolvedValue([]); + mockListExportableWatchHistory.mockResolvedValue([]); + mockListExportableMyList.mockResolvedValue([]); + mockListSyncQueueForProvider.mockResolvedValue([]); + mockResolveSimklIds.mockResolvedValue({ simkl: 123 }); + }); + + describe('Import Flow', () => { + it('correctly categorizes anime as movie or series based on key presence', async () => { + mockGetAllItems.mockImplementation((_token, type) => { + if (type === 'anime') { + return Promise.resolve({ + anime: [ + { + movie: { ids: { imdb: 'tt_anime_movie' }, title: 'Anime Movie' }, + status: 'completed', + }, + { + show: { ids: { imdb: 'tt_anime_series' }, title: 'Anime Series' }, + status: 'watching', + last_watched: 'S01E01', + } + ] + }); + } + return Promise.resolve({}); + }); + + await runImport('profile-1', 'token'); + + // Anime Movie -> movie + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ + metaId: 'tt_anime_movie', + type: 'movie', + progressSeconds: 100 // completed -> 100 + })); + + // Anime Series -> series + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ + metaId: 'tt_anime_series', + type: 'series', + progressSeconds: 1 // watching -> 1 + })); + }); + + it('protects movies and shows from being deleted by anime category', async () => { + // Initial state: we have a movie and a show + mockListWatchHistory.mockResolvedValue([ + { id: 'tt_movie', type: 'movie', source: 'simkl' }, + { id: 'tt_show', type: 'series', source: 'simkl' } + ]); + + // Removals triggered for all categories + mockGetActivities.mockResolvedValue({ + movies: { removed_from_list: '2026-04-25T15:00:00Z' }, + tv_shows: { removed_from_list: '2026-04-25T15:00:00Z' }, + anime: { removed_from_list: '2026-04-25T15:00:00Z' }, + }); + + mockGetAllItems.mockImplementation((_token, type, _dateFrom, extended) => { + if (extended !== 'ids_only') return Promise.resolve({}); + + // Simkl API can return movies under "movies" or "anime" + if (type === 'movies') return Promise.resolve({ movies: [{ movie: { ids: { imdb: 'tt_movie' } } }] }); + if (type === 'shows') return Promise.resolve({ shows: [{ show: { ids: { imdb: 'tt_show' } } }] }); + if (type === 'anime') return Promise.resolve({ anime: [] }); + + return Promise.resolve({}); + }); + + await runImport('profile-1', 'token', { + movies: { removed_from_list: '2026-04-25T14:00:00Z' }, + tv_shows: { removed_from_list: '2026-04-25T14:00:00Z' }, + anime: { removed_from_list: '2026-04-25T14:00:00Z' }, + }); + + // Nothing should be removed because they were found in their respective categories + expect(mockRemoveWatchHistoryMeta).not.toHaveBeenCalled(); + }); + + it('skips episode-level history for fully completed shows', async () => { + mockGetAllItems.mockImplementation((_token, type) => { + if (type === 'shows') { + return Promise.resolve({ + shows: [{ + show: { ids: { imdb: 'tt_completed' }, title: 'Completed Show' }, + status: 'completed', + seasons: [ + { number: 1, episodes: [{ number: 1 }, { number: 2 }] } + ] + }] + }); + } + return Promise.resolve({}); + }); + + await runImport('profile-1', 'token'); + + // Should only have 1 call for the series itself, NOT for individual episodes + expect(mockUpsertWatchProgress).toHaveBeenCalledTimes(1); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ + metaId: 'tt_completed', + videoId: undefined, + progressSeconds: 100 // isCompleted: true -> 100 + })); + }); + + it('imports watching/hold/plantowatch to My List and watching/completed/hold to history', async () => { + mockGetAllItems.mockImplementation((_token, type) => { + if (type === 'movies') { + return Promise.resolve({ + movies: [ + { movie: { ids: { imdb: 'tt_watching' } }, status: 'watching', last_watched_at: '2026-01-01T00:00:00Z' }, + { movie: { ids: { imdb: 'tt_hold' } }, status: 'hold', last_watched_at: '2026-01-01T00:00:00Z' }, + { movie: { ids: { imdb: 'tt_plantowatch' } }, status: 'plantowatch' }, + { movie: { ids: { imdb: 'tt_completed' } }, status: 'completed', last_watched_at: '2026-01-01T00:00:00Z' }, + { movie: { ids: { imdb: 'tt_dropped' } }, status: 'dropped' }, + ] + }); + } + return Promise.resolve({}); + }); + + await runImport('profile-1', 'token'); + + // My List additions: watching, hold, plantowatch + expect(mockAddToMyList).toHaveBeenCalledWith('profile-1', 'tt_watching', 'movie', undefined); + expect(mockAddToMyList).toHaveBeenCalledWith('profile-1', 'tt_hold', 'movie', undefined); + expect(mockAddToMyList).toHaveBeenCalledWith('profile-1', 'tt_plantowatch', 'movie', undefined); + expect(mockAddToMyList).not.toHaveBeenCalledWith('profile-1', 'tt_completed', 'movie', expect.anything()); + + // History upserts: watching, hold, completed + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_watching' })); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_hold' })); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_completed' })); + expect(mockUpsertWatchProgress).not.toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_plantowatch' })); + + // Dropped: removals + expect(mockRemoveFromMyList).toHaveBeenCalledWith('profile-1', 'tt_dropped'); + expect(mockRemoveWatchHistoryMeta).toHaveBeenCalledWith('profile-1', 'tt_dropped'); + }); + + it('handles missing IMDB IDs by falling back to kitsu or simkl ID', async () => { + mockGetAllItems.mockResolvedValue({ + anime: [ + { + show: { ids: { kitsu: 1234, simkl: 5678 }, title: 'No IMDB' }, + status: 'completed', + } + ] + }); + + await runImport('profile-1', 'token'); + + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ + metaId: 'kitsu:1234', + type: 'series' + })); + }); + + it('aborts removal sync if any getAllItems call fails (Empty Response Safety)', async () => { + mockListWatchHistory.mockResolvedValue([ + { id: 'tt_item_1', type: 'movie', source: 'simkl' } + ]); + + mockGetActivities.mockResolvedValue({ + movies: { removed_from_list: '2026-04-25T15:00:00Z' }, + }); + + // Simulate API error (returning null) + mockGetAllItems.mockResolvedValue(null); + + await runImport('profile-1', 'token', { + movies: { removed_from_list: '2026-04-25T14:00:00Z' } + }); + + // Cleanup should NOT have been called because one of the ID fetches failed + expect(mockRemoveWatchHistoryMeta).not.toHaveBeenCalled(); + }); + + it('prevents concurrent imports for the same profile', async () => { + mockGetActivities.mockImplementation(() => new Promise(resolve => setTimeout(() => resolve({}), 100))); + + const sync1 = runImport('profile-concurrent', 'token'); + const sync2 = runImport('profile-concurrent', 'token'); + + const results = await Promise.all([sync1, sync2]); + + // Both should return true (one runs, one skips but returns true) + expect(results[0]).toBe(true); + expect(results[1]).toBe(true); + + // getActivities should only have been called once + expect(mockGetActivities).toHaveBeenCalledTimes(1); + }); + + it('correctly maps various anime types (OVA, ONA, Special)', async () => { + mockGetAllItems.mockResolvedValue({ + anime: [ + { show: { ids: { imdb: 'tt_ova' } }, status: 'completed' }, + { show: { ids: { imdb: 'tt_ona' } }, status: 'completed' }, + { show: { ids: { imdb: 'tt_special' } }, status: 'completed' }, + { show: { ids: { imdb: 'tt_music' } }, status: 'completed' }, + ] + }); + + await runImport('profile-1', 'token'); + + // All should be mapped to 'series' (determined by presence of 'show' key) + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_ova', type: 'series' })); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_ona', type: 'series' })); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_special', type: 'series' })); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_music', type: 'series' })); + }); + }); + + describe('Export Flow', () => { + it('exports completed items and watchlist items', async () => { + mockListExportableWatchHistory.mockResolvedValue([ + { id: 'tt_exp_history', type: 'movie', status: 'completed' } + ]); + mockListExportableMyList.mockResolvedValue([ + { id: 'tt_exp_watchlist', type: 'series' } + ]); + mockResolveSimklIds.mockImplementation((id) => { + if (id === 'tt_exp_history') return Promise.resolve({ imdb: 'tt_exp_history' }); + if (id === 'tt_exp_watchlist') return Promise.resolve({ imdb: 'tt_exp_watchlist' }); + return Promise.resolve(null); + }); + + await runExport('profile-1', 'token'); + + expect(mockPostHistory).toHaveBeenCalledWith('token', expect.objectContaining({ + movies: [expect.objectContaining({ ids: { imdb: 'tt_exp_history' } })] + })); + expect(mockPostWatchlist).toHaveBeenCalledWith('token', expect.objectContaining({ + shows: [expect.objectContaining({ ids: { imdb: 'tt_exp_watchlist' }, to: 'plantowatch' })] + })); + }); + + it('exports removals from sync queue', async () => { + mockListSyncQueueForProvider.mockResolvedValue([ + { id: 1, metaId: 'tt_remove', type: 'movie', action: 'remove_history', createdAt: Date.now() } + ]); + // simulate that this item is new since last sync + jest.spyOn(Date, 'now').mockReturnValue(Date.now() + 1000); + + mockResolveSimklIds.mockResolvedValue({ imdb: 'tt_remove' }); + + await runExport('profile-1', 'token'); + + expect(mockRemoveFromHistory).toHaveBeenCalledWith('token', expect.objectContaining({ + movies: [expect.objectContaining({ ids: { imdb: 'tt_remove' } })] + })); + expect(mockDeleteFromSyncQueue).toHaveBeenCalledWith([1]); + }); + + it('gracefully handles missing Simkl IDs during export', async () => { + mockListExportableWatchHistory.mockResolvedValue([ + { id: 'tt_unknown', type: 'movie', status: 'completed' } + ]); + mockResolveSimklIds.mockResolvedValue(null); + + await runExport('profile-1', 'token'); + + // Should not have called postHistory because ID couldn't be resolved + expect(mockPostHistory).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/api/simkl/__tests__/sync-service.test.ts b/src/api/simkl/__tests__/sync-service.test.ts index 0ccd8b2..08b650d 100644 --- a/src/api/simkl/__tests__/sync-service.test.ts +++ b/src/api/simkl/__tests__/sync-service.test.ts @@ -88,9 +88,9 @@ describe('simkl sync service', () => { mockGetActivities.mockResolvedValue({ all: '2026-01-01T00:00:00.000Z', - movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, - tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, - anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, + movies: { all: '2026-01-01T00:00:00.000Z', plantowatch: '2026-01-01T00:00:00.000Z' }, + tv_shows: { all: '2026-01-01T00:00:00.000Z', plantowatch: '2026-01-01T00:00:00.000Z' }, + anime: { all: '2026-01-01T00:00:00.000Z', plantowatch: '2026-01-01T00:00:00.000Z' }, }); mockGetAllItems.mockResolvedValue({ movies: [], shows: [], anime: [] }); mockListWatchHistory.mockResolvedValue([]); @@ -104,7 +104,7 @@ describe('simkl sync service', () => { it('skips category when cursor matches activity timestamp', async () => { // Arrange const cursor = '2026-01-01T00:00:00.000Z'; - const cursorsObj = { plantowatch: cursor }; + const cursorsObj = { all: cursor, plantowatch: cursor }; // Act await runImport( @@ -125,15 +125,15 @@ describe('simkl sync service', () => { expect(mockGetAllItems).toHaveBeenCalledTimes(3); expect(mockGetAllItems).toHaveBeenNthCalledWith(1, 'token', 'movies', undefined, 'full'); expect(mockGetAllItems).toHaveBeenNthCalledWith(2, 'token', 'shows', undefined, 'full'); - expect(mockGetAllItems).toHaveBeenNthCalledWith(3, 'token', 'anime', undefined, 'full'); + expect(mockGetAllItems).toHaveBeenNthCalledWith(3, 'token', 'anime', undefined, 'full_anime_seasons'); }); it('fetches category when activity timestamp is newer than cursor', async () => { // Arrange mockGetActivities.mockResolvedValueOnce({ - movies: { plantowatch: '2026-02-01T00:00:00.000Z' }, - tv_shows: { plantowatch: '2026-02-01T00:00:00.000Z' }, - anime: { plantowatch: '2026-02-01T00:00:00.000Z' }, + movies: { all: '2026-02-01T00:00:00.000Z' }, + tv_shows: { all: '2026-02-01T00:00:00.000Z' }, + anime: { all: '2026-02-01T00:00:00.000Z' }, }); // Act @@ -141,21 +141,44 @@ describe('simkl sync service', () => { 'profile-1', 'token', { - movies: { plantowatch: '2026-01-01T00:00:00.000Z' }, - tv_shows: { plantowatch: '2026-01-01T00:00:00.000Z' }, - anime: { plantowatch: '2026-01-01T00:00:00.000Z' }, + movies: { all: '2026-01-01T00:00:00.000Z' }, + tv_shows: { all: '2026-01-01T00:00:00.000Z' }, + anime: { all: '2026-01-01T00:00:00.000Z' }, } ); // Assert expect(mockGetAllItems).toHaveBeenCalledTimes(3); - expect(mockGetAllItems).toHaveBeenNthCalledWith( - 1, + expect(mockUpdateSimklCursors).toHaveBeenCalledWith( + 'profile-1', + expect.objectContaining({ + movies: expect.objectContaining({ all: '2026-02-01T00:00:00.000Z' }), + }) + ); + }); + + it('skips category when object activity timestamp matches cursor', async () => { + // Arrange + const cursor = '2026-01-01T00:00:00.000Z'; + mockGetActivities.mockResolvedValueOnce({ + movies: { all: { all: cursor } }, + tv_shows: { all: { all: cursor } }, + anime: { all: { all: cursor } }, + }); + + // Act + await runImport( + 'profile-1', 'token', - 'movies', - '2026-01-01T00:00:00.000Z', - 'full' + { + movies: { all: cursor }, + tv_shows: { all: cursor }, + anime: { all: cursor }, + } ); + + // Assert + expect(mockGetAllItems).not.toHaveBeenCalled(); }); it('imports movie items correctly', async () => { @@ -196,7 +219,7 @@ describe('simkl sync service', () => { shows: [ { show: { ids: { imdb: 'tt12345' }, title: 'Show' }, - status: 'completed', + status: 'watching', // Changed to 'watching' to ensure episodes are imported seasons: [ { number: 2, episodes: [{ number: 3, watched_at: '2026-03-02T00:00:00.000Z' }] }, ], diff --git a/src/api/simkl/client.ts b/src/api/simkl/client.ts index 9a57fb6..c178ae7 100644 --- a/src/api/simkl/client.ts +++ b/src/api/simkl/client.ts @@ -105,11 +105,11 @@ export function getAllItems( token: string, type: SimklMediaType, dateFrom?: string, - extended: 'full' | 'ids_only' = 'full' + extended: 'full' | 'ids_only' | 'full_anime_seasons' = 'full' ): Promise { const path = `/sync/all-items/${type}`; const params = new URLSearchParams({ extended }); - if (extended === 'full') { + if (extended === 'full' || extended === 'full_anime_seasons') { params.set('episode_watched_at', 'yes'); } if (dateFrom) params.set('date_from', dateFrom); diff --git a/src/api/simkl/sync-service.ts b/src/api/simkl/sync-service.ts index f809a18..f71f8ba 100644 --- a/src/api/simkl/sync-service.ts +++ b/src/api/simkl/sync-service.ts @@ -27,8 +27,8 @@ const debug = createDebugLogger('SimklSyncService'); const BATCH_SIZE = 50; -const IMPORT_PROGRESS_SECONDS = 100; -const IMPORT_DURATION_SECONDS = 100; +// Simple mutex to prevent concurrent imports for the same profile +const activeImports = new Set(); interface HistoryIdsPayload { ids: Record; @@ -47,6 +47,8 @@ interface HistoryPayload { } const SYNC_STATUSES: (keyof SimklSyncCursor)[] = [ + 'all', + 'playback', 'plantowatch', 'watching', 'completed', @@ -54,8 +56,12 @@ const SYNC_STATUSES: (keyof SimklSyncCursor)[] = [ 'dropped', ]; +/** + * Checks if a specific activity status needs synchronization. + * Simkl activities provide timestamps for when a category or status last changed. + */ function needsSync(activityCursor: string | object | undefined, storedCursor?: string): boolean { - if (!activityCursor || typeof activityCursor === 'object') return false; + if (typeof activityCursor !== 'string') return false; if (!storedCursor) return true; return activityCursor > storedCursor; } @@ -66,6 +72,12 @@ export async function runImport( cursors?: SimklSyncCursors, opts?: { clearLocalFirst?: boolean } ): Promise { + if (activeImports.has(profileId)) { + debug('importSkipped:Concurrent', { profileId }); + return true; + } + activeImports.add(profileId); + try { debug('importStart', { profileId, hasCursors: !!cursors }); @@ -77,65 +89,104 @@ export async function runImport( await removeProfileMyList(profileId); } - const typesToSync: { type: SimklMediaType; key: keyof SimklActivities; responseKey: keyof SimklAllItemsResponse }[] = [ + // Mapping Simkl internal keys to our display and response keys + const typesToSync: { + type: SimklMediaType; + key: keyof SimklActivities; + responseKey: keyof SimklAllItemsResponse; + }[] = [ { type: 'movies', key: 'movies', responseKey: 'movies' }, { type: 'shows', key: 'tv_shows', responseKey: 'shows' }, { type: 'anime', key: 'anime', responseKey: 'anime' }, ]; - for (const { type, key, responseKey } of typesToSync) { - const typeActivities = activities[key]; - if (!typeActivities) continue; + // 1. Process Removals First (Category-based to prevent cross-category deletion) + // We group by our local ContentType because Simkl "Anime" can be either a movie or a series. + // If we only checked Simkl's "Shows" list, we might accidentally delete Anime series. + const removalCategories = [ + { contentType: 'movie' as ContentType, types: [typesToSync[0], typesToSync[2]] }, + { contentType: 'series' as ContentType, types: [typesToSync[1], typesToSync[2]] }, + ]; - const typeCursors = cursors?.[key] || {}; - const newTypeCursors: SimklSyncCursor = { ...typeCursors }; - newCursors[key] = newTypeCursors; + for (const cat of removalCategories) { + // Check if any Simkl type that maps to this local category had removals + const typesNeedingSync = cat.types.filter((t) => + needsSync(activities[t.key]?.removed_from_list, cursors?.[t.key]?.removed_from_list) + ); - // 1. Process Removals First - // When items are removed from a Simkl list, the `removed_from_list` activity timestamp updates. - // To identify what was removed, we must fetch the entire list using `extended=ids_only` - // (without a date filter) and compare the returned IDs against our local database. - const removedActivity = typeActivities.removed_from_list; - const removedCursor = typeCursors.removed_from_list; - const hasRemovals = needsSync(removedActivity, removedCursor); - - if (hasRemovals) { - debug('syncRemovals', { profileId, type }); - const idsResponse = await getAllItems(token, type, undefined, 'ids_only'); - const items = idsResponse?.[responseKey] || []; + if (typesNeedingSync.length > 0) { + debug('syncRemovals', { profileId, contentType: cat.contentType }); const currentMetaIds = new Set(); - for (const item of items) { - const metaId = getMetaIdFromWatchedItem(item); - if (metaId) currentMetaIds.add(metaId); + let anyFailed = false; + + // Fetch IDs for ALL Simkl types in this category to build a complete set of "what should stay" + for (const t of cat.types) { + const idsResponse = await getAllItems(token, t.type, undefined, 'ids_only'); + if (!idsResponse) { + anyFailed = true; // Safety: abort cleanup if API fails to prevent DB wipe + break; + } + const items = + idsResponse?.[t.responseKey] || + idsResponse?.shows || + idsResponse?.movies || + idsResponse?.anime || + []; + for (const item of items) { + // Only add to the "stay" set if it actually matches the current category we are cleaning + const itemContentType: ContentType = item.movie ? 'movie' : 'series'; + + if (itemContentType === cat.contentType) { + const metaId = getMetaIdFromWatchedItem(item); + if (metaId) currentMetaIds.add(metaId); + } + } } - await cleanupRemovedItems(profileId, type, currentMetaIds); - if (typeof removedActivity === 'string') { - newTypeCursors.removed_from_list = removedActivity; + + if (!anyFailed) { + await cleanupRemovedItems(profileId, cat.contentType, currentMetaIds); + // Update cursors for all Simkl types that were checked + for (const t of typesNeedingSync) { + const timestamp = activities[t.key]?.removed_from_list; + if (typeof timestamp === 'string') { + if (!newCursors[t.key]) newCursors[t.key] = { ...cursors?.[t.key] }; + newCursors[t.key]!.removed_from_list = timestamp; + } + } } } + } + + // 2. Process Additions and Updates + for (const { type, key, responseKey } of typesToSync) { + const typeActivities = activities[key]; + if (!typeActivities) continue; + + const typeCursors = cursors?.[key] || {}; + const newTypeCursors: SimklSyncCursor = { ...typeCursors, ...newCursors[key] }; + newCursors[key] = newTypeCursors; - // 2. Process Additions and Updates - // Simkl tracks separate activity timestamps for each status (watching, completed, plantowatch, etc.). - // Instead of making multiple API calls per status, we find the oldest outdated cursor among all statuses - // and make a single API call to fetch all items updated since that time. - // Track the earliest timestamp we need to fetch updates from. let oldestOutdatedCursor: string | undefined; - // If any status is completely missing a cursor, we must do a full sync without a date filter. let fullSyncRequired = false; - // Flag to track if there is any new activity across all tracked statuses. let anyUpdates = false; - for (const status of SYNC_STATUSES) { - const activity = typeActivities[status]; - const cursor = typeCursors[status]; + const pendingTypeCursors: Partial = {}; + + for (const statusKey of SYNC_STATUSES) { + if (statusKey === 'removed_from_list') continue; // Handled above + + const activity = typeActivities[statusKey]; + const cursor = typeCursors[statusKey]; if (needsSync(activity, cursor)) { anyUpdates = true; if (typeof activity === 'string') { - newTypeCursors[status] = activity; + pendingTypeCursors[statusKey] = activity; } + // If we have no local cursor for this status, we must fetch everything (full sync) if (!cursor) { fullSyncRequired = true; } else if (!oldestOutdatedCursor || cursor < oldestOutdatedCursor) { + // Track the oldest cursor to fetch all items changed since then oldestOutdatedCursor = cursor; } } @@ -145,15 +196,20 @@ export async function runImport( const dateFrom = fullSyncRequired ? undefined : oldestOutdatedCursor; debug('syncUpdates', { profileId, type, dateFrom, fullSyncRequired }); - const itemsResponse = await getAllItems(token, type, dateFrom, 'full'); - const items = itemsResponse?.[responseKey] || []; + const extended = type === 'anime' ? 'full_anime_seasons' : 'full'; + const itemsResponse = await getAllItems(token, type, dateFrom, extended); + const items = itemsResponse?.[responseKey] || itemsResponse?.shows || itemsResponse?.movies || itemsResponse?.anime || []; + + debug('syncUpdates:fetched', { type, count: items.length }); const myListAdditions: { metaId: string; type: ContentType; addedAt?: number }[] = []; const historyUpserts: Parameters[0][] = []; const removals: { metaId: string; type: ContentType }[] = []; for (const item of items) { - const contentType: ContentType = type === 'movies' ? 'movie' : 'series'; + const itemStatus = item.status; + // Content categorization: presence of 'movie' key determines type + const contentType: ContentType = item.movie ? 'movie' : 'series'; const metaId = getMetaIdFromWatchedItem(item); if (!metaId) continue; @@ -168,10 +224,7 @@ export async function runImport( }); } - // 3. Route items based on their status - // 'plantowatch', 'watching' and 'hold' items are added to My List. - // 'watching', 'completed' and 'hold' items update the Watch History with watch dates and episode counts. - // 'dropped' items are actively removed from both My List and Watch History. + // Map Simkl status to our "My List" if (item.status === 'plantowatch' || item.status === 'watching' || item.status === 'hold') { myListAdditions.push({ metaId, @@ -180,20 +233,22 @@ export async function runImport( }); } + // Map Simkl status to local watch history if (item.status === 'watching' || item.status === 'completed' || item.status === 'hold') { - if (type === 'movies' && item.movie) { + if (contentType === 'movie') { const param = collectMovieParam(profileId, item); if (param) historyUpserts.push(param); - } else if (type === 'shows' || type === 'anime') { - const params = collectShowParams(profileId, item); + } else { + const params = collectShowParams(profileId, item, contentType); historyUpserts.push(...params); } - } else if (item.status === 'dropped') { + } else if (itemStatus === 'dropped') { + // Dropped items are removed from local history/watchlist to keep it clean removals.push({ metaId, type: contentType }); } } - // Apply changes in batches + // Apply changes in batches for performance for (let i = 0; i < myListAdditions.length; i += BATCH_SIZE) { const batch = myListAdditions.slice(i, i + BATCH_SIZE); await Promise.all(batch.map((p) => addToMyList(profileId, p.metaId, p.type, p.addedAt))); @@ -208,6 +263,9 @@ export async function runImport( await removeWatchHistoryMeta(profileId, removal.metaId); await removeFromMyList(profileId, removal.metaId); } + + // Success! Now update the cursors locally. + Object.assign(newTypeCursors, pendingTypeCursors); } } @@ -217,6 +275,8 @@ export async function runImport( } catch (error) { debug('importError', { profileId, error }); return false; + } finally { + activeImports.delete(profileId); } } @@ -226,14 +286,16 @@ function getMetaIdFromWatchedItem(item: SimklWatchedItem): string | undefined { return getMetaIdFromIds(ids); } +/** + * Removes local items that are no longer present in the Simkl list for a specific category. + */ async function cleanupRemovedItems( profileId: string, - type: SimklMediaType, + contentType: ContentType, currentlyImportedMetaIds: Set ): Promise { try { const localHistory = await listWatchHistoryForProfile(profileId); - const contentType: ContentType = type === 'movies' ? 'movie' : 'series'; const itemsToRemove = new Set(); for (const item of localHistory) { @@ -256,21 +318,30 @@ async function cleanupRemovedItems( } } +/** + * Persists Simkl watch progress to local DB. + * We use artificial duration (100) and progress (1 or 100) to represent state + * since Simkl doesn't always provide second-accurate progress. + */ async function upsertImportedProgress(params: { profileId: string; metaId: string; type: 'movie' | 'series'; videoId?: string; watchedAt?: string; + isCompleted?: boolean; }): Promise { + const progressSeconds = params.isCompleted ? 100 : 1; + const durationSeconds = 100; + await upsertWatchProgress({ profileId: params.profileId, metaId: params.metaId, videoId: params.videoId, type: params.type, source: 'simkl', - progressSeconds: IMPORT_PROGRESS_SECONDS, - durationSeconds: IMPORT_DURATION_SECONDS, + progressSeconds, + durationSeconds, lastWatchedAt: params.watchedAt ? new Date(params.watchedAt).getTime() : undefined, }); } @@ -279,11 +350,18 @@ function collectMovieParam( profileId: string, item: SimklWatchedItem, ): Parameters[0] | undefined { - if (!item.movie) return; - const metaId = getMetaIdFromIds(item.movie.ids); + const mediaData = item.movie ?? item.anime; + if (!mediaData) return; + const metaId = getMetaIdFromIds(mediaData.ids); if (!metaId) return; - return { profileId, metaId, type: 'movie', watchedAt: item.last_watched_at }; + return { + profileId, + metaId, + type: 'movie', + watchedAt: item.last_watched_at, + isCompleted: item.status === 'completed', + }; } function parseSimklLastWatched(lastWatched: string): { season: number; episode: number } | undefined { @@ -307,6 +385,7 @@ function parseSimklLastWatched(lastWatched: string): { season: number; episode: function collectShowParams( profileId: string, item: SimklWatchedItem, + contentType: 'series' = 'series' ): Parameters[0][] { const out: Parameters[0][] = []; @@ -317,6 +396,7 @@ function collectShowParams( const hasEpisodeArrays = !!item.seasons || !!item.episodes; + // Process per-episode history if detailed data is available if (item.seasons) { for (const season of item.seasons) { for (const episode of season.episodes) { @@ -324,8 +404,9 @@ function collectShowParams( profileId, metaId, videoId: `${metaId}:${season.number}:${episode.number}`, - type: 'series', - watchedAt: episode.watched_at, + type: contentType, + watchedAt: episode.watched_at || item.last_watched_at, + isCompleted: item.status === 'completed' || !!episode.watched_at, }); } } @@ -338,12 +419,29 @@ function collectShowParams( profileId, metaId, videoId: `${metaId}:1:${episode.number}`, - type: 'series', - watchedAt: episode.watched_at, + type: contentType, + watchedAt: episode.watched_at || item.last_watched_at, + isCompleted: item.status === 'completed' || !!episode.watched_at, }); } + return out; + } + + // Optimization: If a show is fully completed, we only store a series-level record. + // Storing individual episode history for a finished show would trigger the app's + // "Up Next" logic and make it appear in "Continue Watching". + if (item.status === 'completed') { + out.push({ + profileId, + metaId, + type: contentType, + watchedAt: item.last_watched_at, + isCompleted: true, + }); + return out; } + // Fallback for simple "last watched" strings (e.g. "S01E05") if (!hasEpisodeArrays) { if (item.last_watched) { const parsed = parseSimklLastWatched(item.last_watched); @@ -352,12 +450,20 @@ function collectShowParams( profileId, metaId, videoId: `${metaId}:${parsed.season}:${parsed.episode}`, - type: 'series', + type: contentType, watchedAt: item.last_watched_at, + isCompleted: false, }); } } else if ((item.watched_episodes_count ?? 0) > 0) { - out.push({ profileId, metaId, type: 'series', watchedAt: item.last_watched_at }); + // Final fallback: just store series-level progress if only a count is known + out.push({ + profileId, + metaId, + type: contentType, + watchedAt: item.last_watched_at, + isCompleted: false, + }); } } @@ -449,13 +555,10 @@ async function buildWatchlistPayload( continue; } - const payloadIds = toHistoryIdsPayload(ids); - if (!payloadIds) continue; - if (item.type === 'movie') { - movies.push({ ids: payloadIds, to: 'plantowatch' }); + movies.push({ ids, to: 'plantowatch' }); } else { - shows.push({ ids: payloadIds, to: 'plantowatch' }); + shows.push({ ids, to: 'plantowatch' }); } } @@ -551,8 +654,11 @@ async function buildExportPayload( continue; } - const episodeRef = parseVideoId(item.videoId ?? ''); - if (!episodeRef) continue; + const episodeRef = item.videoId ? parseVideoId(item.videoId) : null; + if (!episodeRef) { + debug('exportItemInvalidVideoId', { metaId: item.id, videoId: item.videoId }); + continue; + } if (!showsMap.has(item.id)) { showsMap.set(item.id, { @@ -560,6 +666,7 @@ async function buildExportPayload( seasons: new Map(), }); } + const show = showsMap.get(item.id); if (!show) continue; if (!show.seasons.has(episodeRef.season)) { @@ -580,10 +687,10 @@ async function buildExportPayload( } function toHistoryIdsPayload(ids: SimklIds): Record | null { - const entries = Object.entries(ids).filter( - (entry): entry is [string, string | number] => - typeof entry[1] === 'string' || typeof entry[1] === 'number' - ); - if (entries.length === 0) return null; - return Object.fromEntries(entries); + const payload: Record = {}; + if (ids.simkl) payload.simkl = ids.simkl; + if (ids.imdb) payload.imdb = ids.imdb; + if (ids.tmdb) payload.tmdb = ids.tmdb; + + return Object.keys(payload).length > 0 ? payload : null; } diff --git a/src/types/integrations.ts b/src/types/integrations.ts index f50f7fd..b4b031f 100644 --- a/src/types/integrations.ts +++ b/src/types/integrations.ts @@ -4,6 +4,8 @@ export type IntegrationProvider = 'simkl'; export type IntegrationSyncStatus = 'idle' | 'syncing' | 'success' | 'error'; export interface SimklSyncCursor { + all?: string; + playback?: string; plantowatch?: string; watching?: string; completed?: string; From 9281d283cbfefa728775a44d65efcfb488a52060 Mon Sep 17 00:00:00 2001 From: Fabian Schliski Date: Sat, 25 Apr 2026 19:09:15 +0200 Subject: [PATCH 6/6] Bump version to 0.10.1 --- package.json | 2 +- .../simkl/__tests__/sync-service.e2e.test.ts | 15 +++++++---- src/api/simkl/sync-service.ts | 25 ++++++++----------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 47d0ef0..43f8492 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dodostream", - "version": "0.10.0", + "version": "0.10.1", "license": "GPL-3.0-only", "main": "expo-router/entry", "scripts": { diff --git a/src/api/simkl/__tests__/sync-service.e2e.test.ts b/src/api/simkl/__tests__/sync-service.e2e.test.ts index be3c645..56b80ef 100644 --- a/src/api/simkl/__tests__/sync-service.e2e.test.ts +++ b/src/api/simkl/__tests__/sync-service.e2e.test.ts @@ -162,7 +162,7 @@ describe('Simkl Sync Service - Comprehensive E2E', () => { expect(mockRemoveWatchHistoryMeta).not.toHaveBeenCalled(); }); - it('skips episode-level history for fully completed shows', async () => { + it('imports episode-level history for fully completed shows', async () => { mockGetAllItems.mockImplementation((_token, type) => { if (type === 'shows') { return Promise.resolve({ @@ -180,12 +180,17 @@ describe('Simkl Sync Service - Comprehensive E2E', () => { await runImport('profile-1', 'token'); - // Should only have 1 call for the series itself, NOT for individual episodes - expect(mockUpsertWatchProgress).toHaveBeenCalledTimes(1); + // Should have 2 calls for the episodes, NOT a single call for the series + expect(mockUpsertWatchProgress).toHaveBeenCalledTimes(2); expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ metaId: 'tt_completed', - videoId: undefined, - progressSeconds: 100 // isCompleted: true -> 100 + videoId: 'tt_completed:1:1', + progressSeconds: 100 + })); + expect(mockUpsertWatchProgress).toHaveBeenCalledWith(expect.objectContaining({ + metaId: 'tt_completed', + videoId: 'tt_completed:1:2', + progressSeconds: 100 })); }); diff --git a/src/api/simkl/sync-service.ts b/src/api/simkl/sync-service.ts index f71f8ba..3705f6b 100644 --- a/src/api/simkl/sync-service.ts +++ b/src/api/simkl/sync-service.ts @@ -427,20 +427,6 @@ function collectShowParams( return out; } - // Optimization: If a show is fully completed, we only store a series-level record. - // Storing individual episode history for a finished show would trigger the app's - // "Up Next" logic and make it appear in "Continue Watching". - if (item.status === 'completed') { - out.push({ - profileId, - metaId, - type: contentType, - watchedAt: item.last_watched_at, - isCompleted: true, - }); - return out; - } - // Fallback for simple "last watched" strings (e.g. "S01E05") if (!hasEpisodeArrays) { if (item.last_watched) { @@ -467,6 +453,17 @@ function collectShowParams( } } + // If we found no specific episodes but the show is completed, store a series-level record + if (item.status === 'completed' && out.length === 0) { + out.push({ + profileId, + metaId, + type: contentType, + watchedAt: item.last_watched_at, + isCompleted: true, + }); + } + return out; }