diff --git a/README.md b/README.md index b78985dc..2b916a5c 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,12 @@ metadataProviders: # Datasource used for metadata retrieval. DATABASE mode will only work if MangaBaka database is installed # API or DATABASE mode: API + eHentai: + priority: 150 + enabled: false + preferredLanguages: + - "en" + - "ja" server: port: 8085 # or env:KOMF_SERVER_PORT diff --git a/komf-api-models/src/commonMain/kotlin/snd/komf/api/CommonTypes.kt b/komf-api-models/src/commonMain/kotlin/snd/komf/api/CommonTypes.kt index 66460b9c..1894d262 100644 --- a/komf-api-models/src/commonMain/kotlin/snd/komf/api/CommonTypes.kt +++ b/komf-api-models/src/commonMain/kotlin/snd/komf/api/CommonTypes.kt @@ -57,6 +57,7 @@ enum class KomfCoreProviders : KomfProviders { BANGUMI, BOOK_WALKER, COMIC_VINE, + EHENTAI, HENTAG, KODANSHA, MAL, diff --git a/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfig.kt b/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfig.kt index 663f9dda..106c2e9e 100644 --- a/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfig.kt +++ b/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfig.kt @@ -118,6 +118,7 @@ data class ProvidersConfigDto( val bangumi: ProviderConfigDto, val comicVine: ProviderConfigDto, val hentag: ProviderConfigDto, + val eHentai: EHentaiConfigDto, val mangaBaka: MangaBakaConfigDto, val webtoons: ProviderConfigDto, ) @@ -147,6 +148,21 @@ data class ProviderConfigDto( override val artistRoles: Collection, ) : ProviderConf +@Serializable +data class EHentaiConfigDto( + override val priority: Int, + override val enabled: Boolean, + override val seriesMetadata: SeriesMetadataConfigDto, + override val bookMetadata: BookMetadataConfigDto, + override val nameMatchingMode: KomfNameMatchingMode?, + override val mediaType: KomfMediaType, + + override val authorRoles: Collection, + override val artistRoles: Collection, + + val preferredLanguages: List, +) : ProviderConf + @Serializable data class AniListConfigDto( override val priority: Int, diff --git a/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfigUpdateRequest.kt b/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfigUpdateRequest.kt index 1ea45c9d..0ac439da 100644 --- a/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfigUpdateRequest.kt +++ b/komf-api-models/src/commonMain/kotlin/snd/komf/api/config/KomfConfigUpdateRequest.kt @@ -109,6 +109,7 @@ data class ProvidersConfigUpdateRequest( val bangumi: PatchValue = PatchValue.Unset, val comicVine: PatchValue = PatchValue.Unset, val hentag: PatchValue = PatchValue.Unset, + val eHentai: PatchValue = PatchValue.Unset, val mangaBaka: PatchValue = PatchValue.Unset, val webtoons: PatchValue = PatchValue.Unset, ) @@ -126,6 +127,21 @@ class ProviderConfigUpdateRequest( val artistRoles: PatchValue> = PatchValue.Unset, ) +@Serializable +class EHentaiConfigUpdateRequest( + val priority: PatchValue = PatchValue.Unset, + val enabled: PatchValue = PatchValue.Unset, + val seriesMetadata: PatchValue = PatchValue.Unset, + val bookMetadata: PatchValue = PatchValue.Unset, + val nameMatchingMode: PatchValue = PatchValue.Unset, + val mediaType: PatchValue = PatchValue.Unset, + + val authorRoles: PatchValue> = PatchValue.Unset, + val artistRoles: PatchValue> = PatchValue.Unset, + + val preferredLanguages: PatchValue> = PatchValue.Unset, +) + @Serializable class AniListConfigUpdateRequest( val priority: PatchValue = PatchValue.Unset, diff --git a/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigMapper.kt b/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigMapper.kt index 4ab7b364..ba97c109 100644 --- a/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigMapper.kt +++ b/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigMapper.kt @@ -6,6 +6,7 @@ import snd.komf.api.config.AniListConfigDto import snd.komf.api.config.AppriseConfigDto import snd.komf.api.config.BookMetadataConfigDto import snd.komf.api.config.DiscordConfigDto +import snd.komf.api.config.EHentaiConfigDto import snd.komf.api.config.EventListenerConfigDto import snd.komf.api.config.KavitaConfigDto import snd.komf.api.config.KomfConfig @@ -34,6 +35,7 @@ import snd.komf.notifications.apprise.AppriseConfig import snd.komf.notifications.discord.DiscordConfig import snd.komf.providers.AniListConfig import snd.komf.providers.BookMetadataConfig +import snd.komf.providers.EHentaiConfig import snd.komf.providers.MangaBakaConfig import snd.komf.providers.MangaDexConfig import snd.komf.providers.MetadataProvidersConfig @@ -174,6 +176,7 @@ class AppConfigMapper { bangumi = toDto(config.bangumi), comicVine = toDto(config.comicVine), hentag = toDto(config.hentag), + eHentai = toDto(config.eHentai), mangaBaka = toDto(config.mangaBaka), webtoons = toDto(config.webtoons), ) @@ -192,6 +195,20 @@ class AppConfigMapper { ) } + private fun toDto(config: EHentaiConfig): EHentaiConfigDto { + return EHentaiConfigDto( + nameMatchingMode = config.nameMatchingMode?.fromNameMatchingMode(), + priority = config.priority, + enabled = config.enabled, + mediaType = config.mediaType.fromMediaType(), + authorRoles = config.authorRoles.map { it.fromAuthorRole() }, + artistRoles = config.artistRoles.map { it.fromAuthorRole() }, + seriesMetadata = toDto(config.seriesMetadata), + bookMetadata = toDto(config.bookMetadata), + preferredLanguages = config.preferredLanguages, + ) + } + private fun toDto(config: AniListConfig): AniListConfigDto { return AniListConfigDto( nameMatchingMode = config.nameMatchingMode?.fromNameMatchingMode(), diff --git a/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigUpdateMapper.kt b/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigUpdateMapper.kt index 28a4a1ff..3d3dbfae 100644 --- a/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigUpdateMapper.kt +++ b/komf-app/src/main/kotlin/snd/komf/app/api/mappers/AppConfigUpdateMapper.kt @@ -6,6 +6,7 @@ import snd.komf.api.config.AniListConfigUpdateRequest import snd.komf.api.config.AppriseConfigUpdateRequest import snd.komf.api.config.BookMetadataConfigUpdateRequest import snd.komf.api.config.DiscordConfigUpdateRequest +import snd.komf.api.config.EHentaiConfigUpdateRequest import snd.komf.api.config.EventListenerConfigUpdateRequest import snd.komf.api.config.KavitaConfigUpdateRequest import snd.komf.api.config.KomfConfigUpdateRequest @@ -31,6 +32,7 @@ import snd.komf.notifications.apprise.AppriseConfig import snd.komf.notifications.discord.DiscordConfig import snd.komf.providers.AniListConfig import snd.komf.providers.BookMetadataConfig +import snd.komf.providers.EHentaiConfig import snd.komf.providers.MangaBakaConfig import snd.komf.providers.MangaDexConfig import snd.komf.providers.MetadataProvidersConfig @@ -153,6 +155,8 @@ class AppConfigUpdateMapper { ?.let { providerConfig(config.comicVine, it) } ?: config.comicVine, hentag = patch.hentag.getOrNull() ?.let { providerConfig(config.hentag, it) } ?: config.hentag, + eHentai = patch.eHentai.getOrNull() + ?.let { eHentaiProviderConfig(config.eHentai, it) } ?: config.eHentai, mangaBaka = patch.mangaBaka.getOrNull() ?.let { mangaBakaProviderConfig(config.mangaBaka, it) } ?: config.mangaBaka, webtoons = patch.webtoons.getOrNull() @@ -186,6 +190,28 @@ class AppConfigUpdateMapper { ) } + private fun eHentaiProviderConfig(config: EHentaiConfig, patch: EHentaiConfigUpdateRequest): EHentaiConfig { + return config.copy( + priority = patch.priority.getOrNull() ?: config.priority, + enabled = patch.enabled.getOrNull() ?: config.enabled, + mediaType = patch.mediaType.getOrNull()?.toMediaType() ?: config.mediaType, + authorRoles = patch.authorRoles.getOrNull()?.map { it.toAuthorRole() } ?: config.authorRoles, + artistRoles = patch.artistRoles.getOrNull()?.map { it.toAuthorRole() } ?: config.artistRoles, + seriesMetadata = patch.seriesMetadata.getOrNull() + ?.let { seriesMetadataConfig(config.seriesMetadata, it) } + ?: config.seriesMetadata, + bookMetadata = patch.bookMetadata.getOrNull() + ?.let { bookMetadataConfig(config.bookMetadata, it) } + ?: config.bookMetadata, + nameMatchingMode = when (val mode = patch.nameMatchingMode) { + PatchValue.None -> null + is PatchValue.Some -> mode.value.toNameMatchingMode() + PatchValue.Unset -> config.nameMatchingMode + }, + preferredLanguages = patch.preferredLanguages.getOrNull() ?: config.preferredLanguages + ) + } + private fun aniListProviderConfig(config: AniListConfig, patch: AniListConfigUpdateRequest): AniListConfig { return config.copy( priority = patch.priority.getOrNull() ?: config.priority, diff --git a/komf-app/src/main/kotlin/snd/komf/app/api/mappers/CommonTypesMapper.kt b/komf-app/src/main/kotlin/snd/komf/app/api/mappers/CommonTypesMapper.kt index 62fae3e3..66b5faee 100644 --- a/komf-app/src/main/kotlin/snd/komf/app/api/mappers/CommonTypesMapper.kt +++ b/komf-app/src/main/kotlin/snd/komf/app/api/mappers/CommonTypesMapper.kt @@ -92,6 +92,7 @@ fun CoreProviders.fromProvider() = when (this) { CoreProviders.BANGUMI -> KomfCoreProviders.BANGUMI CoreProviders.BOOK_WALKER -> KomfCoreProviders.BOOK_WALKER CoreProviders.COMIC_VINE -> KomfCoreProviders.COMIC_VINE + CoreProviders.EHENTAI -> KomfCoreProviders.EHENTAI CoreProviders.HENTAG -> KomfCoreProviders.HENTAG CoreProviders.KODANSHA -> KomfCoreProviders.KODANSHA CoreProviders.MAL -> KomfCoreProviders.MAL @@ -109,6 +110,7 @@ fun KomfProviders.toProvider() = when (this) { KomfCoreProviders.BANGUMI -> CoreProviders.BANGUMI KomfCoreProviders.BOOK_WALKER -> CoreProviders.BOOK_WALKER KomfCoreProviders.COMIC_VINE -> CoreProviders.COMIC_VINE + KomfCoreProviders.EHENTAI -> CoreProviders.EHENTAI KomfCoreProviders.HENTAG -> CoreProviders.HENTAG KomfCoreProviders.KODANSHA -> CoreProviders.KODANSHA KomfCoreProviders.MAL -> CoreProviders.MAL diff --git a/komf-app/src/main/kotlin/snd/komf/app/config/ConfigLoader.kt b/komf-app/src/main/kotlin/snd/komf/app/config/ConfigLoader.kt index d8954a97..9a5c8d46 100644 --- a/komf-app/src/main/kotlin/snd/komf/app/config/ConfigLoader.kt +++ b/komf-app/src/main/kotlin/snd/komf/app/config/ConfigLoader.kt @@ -126,6 +126,7 @@ class ConfigLoader(private val yaml: Yaml) { config.metadataProviders.defaultProviders.bangumi.enabled.not() && config.metadataProviders.defaultProviders.comicVine.enabled.not() && config.metadataProviders.defaultProviders.hentag.enabled.not() && + config.metadataProviders.defaultProviders.eHentai.enabled.not() && config.metadataProviders.defaultProviders.mangaBaka.enabled.not() && config.metadataProviders.defaultProviders.webtoons.enabled.not() && config.metadataProviders.libraryProviders.isEmpty() diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/CoreProviders.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/CoreProviders.kt index 5e2f5bb4..cf109af9 100644 --- a/komf-core/src/commonMain/kotlin/snd/komf/providers/CoreProviders.kt +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/CoreProviders.kt @@ -5,6 +5,7 @@ enum class CoreProviders { BANGUMI, BOOK_WALKER, COMIC_VINE, + EHENTAI, HENTAG, KODANSHA, MAL, diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt index 7c229e9a..bde0cee3 100644 --- a/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt @@ -41,6 +41,7 @@ data class ProvidersConfig( val bangumi: ProviderConfig = ProviderConfig(), val comicVine: ProviderConfig = ProviderConfig(), val hentag: ProviderConfig = ProviderConfig(), + val eHentai: EHentaiConfig = EHentaiConfig(), val mangaBaka: MangaBakaConfig = MangaBakaConfig(), val webtoons: ProviderConfig = ProviderConfig(), ) @@ -58,6 +59,21 @@ data class ProviderConfig( val artistRoles: Collection = listOf(PENCILLER, INKER, COLORIST, LETTERER, COVER), ) +@Serializable +data class EHentaiConfig( + val priority: Int = 10, + val enabled: Boolean = false, + val seriesMetadata: SeriesMetadataConfig = SeriesMetadataConfig(), + val bookMetadata: BookMetadataConfig = BookMetadataConfig(), + val nameMatchingMode: NameMatchingMode? = null, + val mediaType: MediaType = MANGA, + + val preferredLanguages: List = listOf("en", "ja"), + + val authorRoles: Collection = listOf(WRITER), + val artistRoles: Collection = listOf(PENCILLER, INKER, COLORIST, LETTERER, COVER), +) + @Serializable data class MangaBakaConfig( val priority: Int = 10, diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt index a92e54d4..f444dbc5 100644 --- a/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt @@ -34,6 +34,9 @@ import snd.komf.providers.comicvine.ComicVineClient import snd.komf.providers.comicvine.ComicVineMetadataMapper import snd.komf.providers.comicvine.ComicVineMetadataProvider import snd.komf.providers.comicvine.ComicVineRateLimiter +import snd.komf.providers.ehentai.EHentaiClient +import snd.komf.providers.ehentai.EHentaiMetadataMapper +import snd.komf.providers.ehentai.EHentaiMetadataProvider import snd.komf.providers.hentag.HentagClient import snd.komf.providers.hentag.HentagMetadataMapper import snd.komf.providers.hentag.HentagMetadataProvider @@ -260,6 +263,25 @@ class ProvidersModule( } ) + /* Load limiting: 4-5 sequential requests usually okay before having to wait for ~5 seconds */ + private val eHentaiClient = EHentaiClient( + baseHttpClientJson.config { + install(HttpRequestRateLimiter) { + interval = 6.seconds + eventsPerInterval = 4 + allowBurst = true + } + install(HttpRequestRetry) { + defaultRetry() + } + }, + baseHttpClientJson.config { + install(HttpRequestRetry) { + defaultRetry() + } + } + ) + private val mangaBakaClient = MangaBakaApiClient( baseHttpClientJson.config { install(HttpRequestRateLimiter) { @@ -411,6 +433,12 @@ class ProvidersModule( defaultNameMatcher ), hentagPriority = config.hentag.priority, + eHentai = createEHentaiMetadataProvider( + config.eHentai, + eHentaiClient, + defaultNameMatcher + ), + eHentaiPriority = config.eHentai.priority, mangaBaka = createMangaBakaMetadataProvider( config = config.mangaBaka, datasource = when (config.mangaBaka.mode) { @@ -761,6 +789,28 @@ class ProvidersModule( ) } + private fun createEHentaiMetadataProvider( + config: EHentaiConfig, + client: EHentaiClient, + defaultNameMatcher: NameSimilarityMatcher, + ): EHentaiMetadataProvider? { + if (config.enabled.not()) return null + + val eHentaiMetadataMapper = EHentaiMetadataMapper( + metadataConfig = config.seriesMetadata, + authorRoles = config.authorRoles, + preferredLanguages = config.preferredLanguages, + ) + + val ehentaiSimilarityMatcher: NameSimilarityMatcher = + config.nameMatchingMode?.let { nameSimilarityMatcher(it) } ?: defaultNameMatcher + return EHentaiMetadataProvider( + client, + eHentaiMetadataMapper, + ehentaiSimilarityMatcher, + config.seriesMetadata.thumbnail, + ) + } private fun createMangaBakaMetadataProvider( config: MangaBakaConfig, @@ -860,6 +910,9 @@ class ProvidersModule( private val hentag: HentagMetadataProvider?, private val hentagPriority: Int, + private val eHentai: EHentaiMetadataProvider?, + private val eHentaiPriority: Int, + private val mangaBaka: MangaBakaMetadataProvider?, private val mangaBakaPriority: Int, @@ -880,6 +933,7 @@ class ProvidersModule( bangumi?.let { it to bangumiPriority }, comicVine?.let { it to comicVinePriority }, hentag?.let { it to hentagPriority }, + eHentai?.let { it to eHentaiPriority }, mangaBaka?.let { it to mangaBakaPriority }, webtoons?.let { it to webtoonsPriority } ) @@ -901,6 +955,7 @@ class ProvidersModule( CoreProviders.BANGUMI -> bangumi CoreProviders.COMIC_VINE -> comicVine CoreProviders.HENTAG -> hentag + CoreProviders.EHENTAI -> eHentai CoreProviders.MANGA_BAKA -> mangaBaka CoreProviders.WEBTOONS -> webtoons } diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiClient.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiClient.kt new file mode 100644 index 00000000..efb2e542 --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiClient.kt @@ -0,0 +1,119 @@ +package snd.komf.providers.ehentai + +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.request.get +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.contentType +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.add +import kotlinx.serialization.json.addJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import kotlinx.serialization.json.putJsonArray +import snd.komf.model.Image +import snd.komf.providers.ehentai.model.EHentaiBook +import snd.komf.providers.ehentai.model.EHentaiResponse + +class EHentaiClient( + private val apiClient: HttpClient, + private val imgClient: HttpClient +) { + companion object { + private const val API_URL = "https://api.e-hentai.org/api.php" + private const val BASE_URL = "https://e-hentai.org" + private const val MAX_PAGES = 3 + private val GALLERY_REGEX = """/g/(\d+)/([a-f0-9]+)""".toRegex() + private val NEXT_URL_REGEX = """var\s+nexturl="([^"]+)"""".toRegex() + + private val JSON_PARSER = Json { + ignoreUnknownKeys = true + isLenient = true + } + } + + suspend fun searchByGidList( + gidList: List> + ): EHentaiResponse { + if (gidList.isEmpty()) return EHentaiResponse() + + /* Load limiting: 25 entries per request */ + val chunks = gidList.chunked(25) + + return coroutineScope { + val responses = chunks.map { chunk -> + async { + val responseText = apiClient.post(API_URL) { + contentType(ContentType.Application.Json) + setBody(buildJsonObject { + put("method", "gdata") + putJsonArray("gidlist") { + chunk.forEach { (gid, token) -> + addJsonArray { + add(gid) + add(token) + } + } + } + put("namespace", 1) + }) + }.bodyAsText() + JSON_PARSER.decodeFromString(responseText) + } + }.awaitAll() + + EHentaiResponse( + gmetadata = responses.flatMap { it.gmetadata }, + error = responses.firstNotNullOfOrNull { it.error } + ) + } + } + + suspend fun searchByTitle(title: String): EHentaiResponse { + val gidList = mutableListOf>() + var currentUrl: String? = null + + var currentPage = 0 + while (currentPage < MAX_PAGES) { + val htmlResponse = if (currentUrl == null) { + apiClient.get(BASE_URL) { + url { parameters.append("f_search", title) } + }.bodyAsText() + } else { + apiClient.get(currentUrl).bodyAsText() + } + + val gidInPage = GALLERY_REGEX.findAll(htmlResponse) + .map { it.groupValues[1].toInt() to it.groupValues[2] } + .toList() + + if (gidInPage.isEmpty()) break + gidList.addAll(gidInPage) + + currentUrl = NEXT_URL_REGEX.find(htmlResponse) + ?.groupValues?.get(1) + ?.replace("&", "&") ?: break + + currentPage++ + } + + val distinctGidList = gidList.distinct() + return when { + distinctGidList.isNotEmpty() -> searchByGidList(distinctGidList) + else -> EHentaiResponse(error = "Empty gidList for search: $title") + } + } + + suspend fun getThumbnail(book: EHentaiBook): Image? { + return book.thumb?.ifBlank { null }?.let { url -> + val bytes = imgClient.get(url).body() + Image(bytes) + } + } +} \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiMetadataMapper.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiMetadataMapper.kt new file mode 100644 index 00000000..eda87a38 --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiMetadataMapper.kt @@ -0,0 +1,145 @@ +package snd.komf.providers.ehentai + +import kotlinx.datetime.TimeZone +import kotlinx.datetime.number +import kotlinx.datetime.toLocalDateTime +import snd.komf.model.Author +import snd.komf.model.AuthorRole +import snd.komf.model.Image +import snd.komf.model.ProviderSeriesId +import snd.komf.model.ProviderSeriesMetadata +import snd.komf.model.ReleaseDate +import snd.komf.model.SeriesMetadata +import snd.komf.model.SeriesSearchResult +import snd.komf.model.SeriesStatus +import snd.komf.model.SeriesTitle +import snd.komf.model.TitleType +import snd.komf.model.WebLink +import snd.komf.providers.CoreProviders +import snd.komf.providers.MetadataConfigApplier +import snd.komf.providers.SeriesMetadataConfig +import snd.komf.providers.ehentai.model.EHentaiBook + +class EHentaiMetadataMapper( + private val metadataConfig: SeriesMetadataConfig, + private val authorRoles: Collection, + private val preferredLanguages: List, +) { + + fun toSeriesMetadata( + book: EHentaiBook, + thumbnail: Image? + ): ProviderSeriesMetadata { + val rawTags = book.tags.orEmpty() + + val language = EHentaiTagMapper.BCP47_MAP[rawTags.find { it.startsWith("language:") }] + + val authors = rawTags.asSequence() + .filter { it.startsWith("artist:") || it.startsWith("group:") } + .map { it.substringAfter(":") } + .flatMap { name -> authorRoles.map { Author(name, it) } } + .toList() + + val finalTags = rawTags.asSequence() + .filterNot { it.startsWith("language:") || it.startsWith("artist:") || it.startsWith("group:") } + .map { it.substringAfter(":") } + .distinct() + .toList() + + val title = when { + language == "ja" && !book.titleJpn.isNullOrBlank() -> { + SeriesTitle( + EHentaiParser.parseTitle(book.titleJpn).bestMatch, + TitleType.NATIVE, + "ja" + ) + } + + else -> { + val type = when (language) { + null -> null + "ja" -> TitleType.ROMAJI + else -> TitleType.LOCALIZED + } + SeriesTitle( + EHentaiParser.parseTitle(book.title).bestMatch, + type, + language + ) + } + } + + val link = WebLink("e-hentai", "https://e-hentai.org/g/${book.gid}/${book.token}") + + val metadata = SeriesMetadata( + title = title, + language = language, + releaseDate = book.posted?.toLocalDateTime(TimeZone.UTC)?.let { + ReleaseDate(it.year, it.month.number, it.day) + }, + tags = finalTags, + authors = authors, + links = listOf(link), + thumbnail = thumbnail, + score = book.rating, + status = SeriesStatus.ENDED, + ageRating = EHentaiTagMapper.mapAgeRating(book.category, rawTags) + ) + + return MetadataConfigApplier.apply( + ProviderSeriesMetadata( + id = ProviderSeriesId("${book.gid};${book.token}"), + metadata = metadata + ), + metadataConfig + ) + } + + fun toSeriesSearchResult(result: EHentaiBook): SeriesSearchResult { + return SeriesSearchResult( + resultId = "${result.gid};${result.token}", + url = "https://e-hentai.org/g/${result.gid}/${result.token}", + imageUrl = result.thumb, + title = result.title, + provider = CoreProviders.EHENTAI + ) + } + + fun applyLanguagePreference(books: List): List { + val validLanguages = preferredLanguages + .filter { EHentaiTagMapper.BCP47_MAP.containsValue(it) } + .distinct() + + if (validLanguages.isEmpty()) { + return books + } + + val preferredBooksGrouped = mutableMapOf>() + validLanguages.forEach { preferredBooksGrouped[it] = mutableListOf() } + + val noLanguageBooks = mutableListOf() + + books.forEach { book -> + val hasLanguageTag = book.tags?.any { it.startsWith("language:") } == true + val langTag = book.tags?.firstOrNull { it.startsWith("language:") } + val bookLang = EHentaiTagMapper.BCP47_MAP[langTag] + + when { + bookLang in validLanguages -> preferredBooksGrouped[bookLang]!!.add(book) + !hasLanguageTag -> noLanguageBooks.add(book) + } + } + + val sortedPreferredBooks = validLanguages.flatMap { lang -> + preferredBooksGrouped[lang]!!.sortedByDescending { it.rating ?: 0.0 } + } + + noLanguageBooks.sortByDescending { it.rating ?: 0.0 } + + if (sortedPreferredBooks.isNotEmpty() || noLanguageBooks.isNotEmpty()) { + return sortedPreferredBooks + noLanguageBooks + } + + return emptyList() + } +} \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiMetadataProvider.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiMetadataProvider.kt new file mode 100644 index 00000000..678e4662 --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiMetadataProvider.kt @@ -0,0 +1,108 @@ +package snd.komf.providers.ehentai + +import io.github.reactivecircus.cache4k.Cache +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import snd.komf.model.Image +import snd.komf.model.MatchQuery +import snd.komf.model.ProviderBookId +import snd.komf.model.ProviderBookMetadata +import snd.komf.model.ProviderSeriesId +import snd.komf.model.ProviderSeriesMetadata +import snd.komf.model.SeriesSearchResult +import snd.komf.providers.CoreProviders +import snd.komf.providers.MetadataProvider +import snd.komf.providers.ehentai.model.EHentaiBook +import snd.komf.util.NameSimilarityMatcher +import kotlin.time.Duration.Companion.minutes + +class EHentaiMetadataProvider( + private val eHentaiClient: EHentaiClient, + private val metadataMapper: EHentaiMetadataMapper, + private val nameMatcher: NameSimilarityMatcher, + private val fetchSeriesCovers: Boolean +) : MetadataProvider { + + private val cache = Cache.Builder() + .expireAfterWrite(5.minutes) + .build() + + override fun providerName() = CoreProviders.EHENTAI + + private suspend fun getBookOrThrow(seriesId: ProviderSeriesId): EHentaiBook { + return cache.get(seriesId) { + val response = eHentaiClient.searchByGidList(listOf(EHentaiParser.parseGid(seriesId))) + val book = response.gmetadata.firstOrNull() ?: throw RuntimeException("Gallery not found") + if (book.error != null) { + throw RuntimeException("E-Hentai API Error for $seriesId: ${book.error}") + } + book + } + } + + override suspend fun getSeriesMetadata(seriesId: ProviderSeriesId): ProviderSeriesMetadata { + val book = getBookOrThrow(seriesId) + val thumbnail = if (fetchSeriesCovers) eHentaiClient.getThumbnail(book) else null + return metadataMapper.toSeriesMetadata(book, thumbnail) + } + + override suspend fun getSeriesCover(seriesId: ProviderSeriesId): Image? { + val book = getBookOrThrow(seriesId) + return eHentaiClient.getThumbnail(book) + } + + override suspend fun getBookMetadata(seriesId: ProviderSeriesId, bookId: ProviderBookId): ProviderBookMetadata { + throw UnsupportedOperationException() + } + + override suspend fun searchSeries(seriesName: String, limit: Int): Collection { + val queries = EHentaiParser.getSearchQueries(seriesName) + + val rawResults = kotlinx.coroutines.coroutineScope { + queries.map { query -> + async { eHentaiClient.searchByTitle(query.take(400)).gmetadata } + }.awaitAll().flatten() + }.filter { it.error == null }.distinctBy { it.gid } + + val processedResults = metadataMapper.applyLanguagePreference(rawResults) + + return processedResults.map { result -> + metadataMapper.toSeriesSearchResult(result) + .also { cache.put(ProviderSeriesId(it.resultId), result) } + } + } + + override suspend fun matchSeriesMetadata(matchQuery: MatchQuery): ProviderSeriesMetadata? { + // Usually downloaded books from E-Hentai have complete title name. + // We assume that user have not renamed their books and directly put them into the library. + // So this method is based on book's title and use regex to search the gallery. + val searchName = matchQuery.bookQualifier?.name ?: matchQuery.seriesName + + val searchResults = kotlinx.coroutines.coroutineScope { + EHentaiParser.getSearchQueries(searchName).map { query -> + async { eHentaiClient.searchByTitle(query.take(400)).gmetadata } + }.awaitAll().flatten() + } + .filter { it.error == null } + .distinctBy { it.gid } + + val processedResults = metadataMapper.applyLanguagePreference(searchResults) + + return processedResults + .firstOrNull { book -> + val matchTitle = matchesName(searchName, book.title) + val matchTitleJpn = book.titleJpn?.let { matchesName(searchName, it) } ?: false + matchTitle || matchTitleJpn + } + ?.let { book -> + val cover = if (fetchSeriesCovers) eHentaiClient.getThumbnail(book) else null + metadataMapper.toSeriesMetadata(book, cover).also { cache.put(it.id, book) } + } + } + + private fun matchesName(name: String, nameToMatch: String): Boolean { + val localVariants = EHentaiParser.getSearchQueries(name) + val remoteVariants = EHentaiParser.getSearchQueries(nameToMatch) + return localVariants.any { local -> nameMatcher.matches(local, remoteVariants) } + } +} \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiParser.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiParser.kt new file mode 100644 index 00000000..bc7095eb --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiParser.kt @@ -0,0 +1,77 @@ +package snd.komf.providers.ehentai + +import snd.komf.model.ProviderSeriesId +import snd.komf.providers.ehentai.model.EHentaiParsedTitle + +object EHentaiParser { + + private val platformPrefixRegex = """ + ^\[(pixiv|fanbox|fantia|patreon|gumroad|ci-en)([\s/,&|]+(pixiv|fanbox|fantia|patreon|gumroad|ci-en))*] + """ + .trimIndent() + .toRegex(RegexOption.IGNORE_CASE) + + fun parseGid(seriesId: ProviderSeriesId): Pair { + val parts = seriesId.value.split(";", limit = 2) + + require(parts.size == 2) { "Invalid E-Hentai ID format: ${seriesId.value}" } + val gid = parts[0].toIntOrNull() + ?: throw IllegalArgumentException("Invalid GID (Not a number) in ID: ${seriesId.value}") + val token = parts[1] + + return Pair(gid, token) + } + + fun parseTitle(rawTitle: String): EHentaiParsedTitle { + if (rawTitle.isBlank()) return EHentaiParsedTitle("", "") + + // Remove all trailing [xxx] tags + val trailingRegex = """(?:\s*\[[^]]+])+$""".toRegex() + val withoutTrailing = rawTitle.replace(trailingRegex, "").trim() + + /// Remove leading convention and artist information + val leadingRegex = """^(?:\([^)]+\)\s*)?(?:\[[^]]+]\s*)?""".toRegex() + var baseTitle = withoutTrailing.replace(leadingRegex, "").trim() + + // Pure date digits or artist's image pack only + if (baseTitle.matches("""^[\d\-.\s]+$""".toRegex()) || + platformPrefixRegex.containsMatchIn(rawTitle) + ) { + baseTitle = withoutTrailing + } + + // Take the final translated name or original name to the right of "|" + val bestMatch = if (baseTitle.contains("|")) { + baseTitle.substringAfterLast("|").trim() + } else { + baseTitle + } + + if (baseTitle.isBlank()) { + baseTitle = rawTitle + } + + return EHentaiParsedTitle(match = baseTitle, bestMatch = bestMatch.ifBlank { baseTitle }) + } + + /** + * ### Generate up to 3 search variants + * ``` md + * 1. [a (b)] c (d) [e] [f] + * 2. c (d) + * 3. c + * ``` + */ + fun getSearchQueries(rawTitle: String): List { + val parsed = parseTitle(rawTitle) + + val coreTitleRegex = """(?:\s*[（(][^)）]+[)）])+$""".toRegex() + val coreTitle = parsed.bestMatch.replace(coreTitleRegex, "").trim() + val finalCoreTitle = coreTitle.ifBlank { parsed.bestMatch } + + return listOf(rawTitle, parsed.bestMatch, finalCoreTitle) + .map { it.trim() } + .filter { it.isNotBlank() } + .distinct() + } +} \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiTagMapper.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiTagMapper.kt new file mode 100644 index 00000000..4578a4a7 --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/EHentaiTagMapper.kt @@ -0,0 +1,113 @@ +package snd.komf.providers.ehentai + +object EHentaiTagMapper { + + val BCP47_MAP: Map = mapOf( + "language:afrikaans" to "af", + "language:albanian" to "sq", + "language:arabic" to "ar", + "language:aramaic" to "arc", // ISO 639-2 + "language:armenian" to "hy", + "language:bengali" to "bn", + "language:bosnian" to "bs", + "language:bulgarian" to "bg", + "language:burmese" to "my", + "language:catalan" to "ca", + "language:cebuano" to "ceb", // ISO 639-2 + "language:chinese" to "zh", + "language:cree" to "cr", + "language:creole" to "ht", + "language:croatian" to "hr", + "language:czech" to "cs", + "language:danish" to "da", + "language:dutch" to "nl", + "language:english" to "en", + "language:esperanto" to "eo", + "language:estonian" to "et", + "language:finnish" to "fi", + "language:french" to "fr", + "language:georgian" to "ka", + "language:german" to "de", + "language:greek" to "el", + "language:gujarati" to "gu", + "language:hebrew" to "he", + "language:hindi" to "hi", + "language:hmong" to "hmn", // ISO 639-2 + "language:hungarian" to "hu", + "language:icelandic" to "is", + "language:indonesian" to "id", + "language:irish" to "ga", + "language:italian" to "it", + "language:japanese" to "ja", + "language:javanese" to "jv", + "language:kannada" to "kn", + "language:kazakh" to "kk", + "language:khmer" to "km", + "language:korean" to "ko", + "language:kurdish" to "ku", + "language:ladino" to "lad", // ISO 639-2 + "language:lao" to "lo", + "language:latin" to "la", + "language:latvian" to "lv", + "language:marathi" to "mr", + "language:mongolian" to "mn", + "language:ndebele" to "nd", + "language:nepali" to "ne", + "language:norwegian" to "no", + "language:oromo" to "om", + "language:papiamento" to "pap", // ISO 639-2 + "language:pashto" to "ps", + "language:persian" to "fa", + "language:polish" to "pl", + "language:portuguese" to "pt", + "language:punjabi" to "pa", + "language:romanian" to "ro", + "language:russian" to "ru", + "language:sango" to "sg", + "language:sanskrit" to "sa", + "language:serbian" to "sr", + "language:shona" to "sn", + "language:slovak" to "sk", + "language:slovenian" to "sl", + "language:somali" to "so", + "language:spanish" to "es", + "language:swahili" to "sw", + "language:swedish" to "sv", + "language:tagalog" to "tl", + "language:tamil" to "ta", + "language:telugu" to "te", + "language:thai" to "th", + "language:tibetan" to "bo", + "language:tigrinya" to "ti", + "language:turkish" to "tr", + "language:ukrainian" to "uk", + "language:urdu" to "ur", + "language:vietnamese" to "vi", + "language:welsh" to "cy", + "language:yiddish" to "yi", + "language:zulu" to "zu" + ) + + fun mapAgeRating(category: String?, tags: List): Int? { + val hasExtremeTags = tags.any { tag -> + val cleanTag = if (tag.contains(":")) tag.substringAfter(":") else tag + cleanTag in listOf("guro", "ryona", "snuff", "scat") + } + if (hasExtremeTags) return 18 + + return when (category?.lowercase()) { + "non-h" -> 15 + "doujinshi", + "manga", + "artist cg", + "game cg", + "image set", + "western", + "cosplay", + "misc", + "private" -> 18 + + else -> null + } + } +} \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiBook.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiBook.kt new file mode 100644 index 00000000..2be5f68a --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiBook.kt @@ -0,0 +1,116 @@ +package snd.komf.providers.ehentai.model + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlin.math.round +import kotlin.time.Instant + +@Serializable +data class EHentaiBook( + val gid: Int, + val token: String, + @Serializable(with = HtmlUnescapeStringSerializer::class) + val title: String, + @SerialName("title_jpn") + @Serializable(with = HtmlUnescapeStringSerializer::class) + val titleJpn: String? = null, + val category: String? = null, + val thumb: String? = null, + val uploader: String? = null, + @Serializable(with = InstantEpochSecondsSerializer::class) + val posted: Instant? = null, + @SerialName("filecount") + val fileCount: String? = null, + @SerialName("filesize") + val fileSize: Long? = null, + val expunged: Boolean? = null, + @Serializable(with = StringToRoundedDoubleSerializer::class) + val rating: Double? = null, + @SerialName("torrentcount") + val torrentCount: String? = null, + val torrents: List? = null, + val tags: List? = null, + @SerialName("parent_gid") + val parentGid: String? = null, + @SerialName("parent_key") + val parentKey: String? = null, + @SerialName("current_gid") + val currentGid: String? = null, + @SerialName("current_key") + val currentKey: String? = null, + @SerialName("first_gid") + val firstGid: String? = null, + @SerialName("first_key") + val firstKey: String? = null, + val error: String? = null +) + +@Serializable +data class EHentaiTorrent( + val hash: String, + @Serializable(with = InstantEpochSecondsSerializer::class) + val added: Instant, + @Serializable(with = HtmlUnescapeStringSerializer::class) + val name: String, + @SerialName("tsize") + val tSize: String, + @SerialName("fsize") + val fSize: String +) + +/** + * Automatically convert a rating string (e.g. "4.68") to a rounded Double (e.g. 5.0) + */ +object StringToRoundedDoubleSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("StringToRoundedDouble", PrimitiveKind.STRING) + + override fun deserialize(decoder: Decoder): Double { + val stringValue = decoder.decodeString() + val doubleValue = stringValue.toDoubleOrNull() ?: 0.0 + return round(doubleValue) + } + + override fun serialize(encoder: Encoder, value: Double) { + encoder.encodeString(value.toString()) + } +} + +/** + * Automatically unescape HTML entities during JSON parsing + */ +object HtmlUnescapeStringSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("HtmlUnescapeString", PrimitiveKind.STRING) + + override fun deserialize(decoder: Decoder): String { + return decoder.decodeString() + .replace("&", "&") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("<", "<") + .replace(">", ">") + } + + override fun serialize(encoder: Encoder, value: String) { + encoder.encodeString(value) + } +} + +object InstantEpochSecondsSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("kotlinx.datetime.Instant", PrimitiveKind.STRING) + + override fun deserialize(decoder: Decoder): Instant = + Instant.fromEpochSeconds(decoder.decodeString().toLong()) + + override fun serialize(encoder: Encoder, value: Instant) = + encoder.encodeString(value.epochSeconds.toString()) +} \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiParsedTitle.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiParsedTitle.kt new file mode 100644 index 00000000..8f8f5868 --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiParsedTitle.kt @@ -0,0 +1,18 @@ +package snd.komf.providers.ehentai.model + +data class EHentaiParsedTitle( + /** + * ### Match title + * ``` md + * (Convention)[Artist/Circle] Title [Language] [Attribute tags] -> Title + * ``` + */ + val match: String, + /** + * ### The rightmost title from a matched title + * ```md + * Title1 | Title2 | Title3 -> Title3 + * ``` + */ + val bestMatch: String +) \ No newline at end of file diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiResponse.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiResponse.kt new file mode 100644 index 00000000..4f589424 --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ehentai/model/EHentaiResponse.kt @@ -0,0 +1,10 @@ +package snd.komf.providers.ehentai.model + +import kotlinx.serialization.Serializable + +@Serializable +data class EHentaiResponse( + val gmetadata: List = emptyList(), + val gid: Int? = null, + val error: String? = null +) \ No newline at end of file