feat(scrapper): add PostgreSQL storage and scheduled link updates

.github/workflows/dotnet.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 8.0.x
+        dotnet-version: 9.0.x
     - name: Restore dependencies
       run: dotnet restore
     - name: Build

LinkTracker.sln

@@ -11,6 +11,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LinkTracker.Shared", "src\L
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LinkTracker.Scrapper", "src\LinkTracker.Scrapper\LinkTracker.Scrapper.csproj", "{093435F1-2E61-4005-81AE-AC4CFE7C17AE}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LinkTracker.Scrapper.Tests", "tests\LinkTracker.Scrapper.Tests\LinkTracker.Scrapper.Tests.csproj", "{1789B984-4451-4B95-8101-AC885C600986}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -57,6 +61,18 @@ Global
 		{093435F1-2E61-4005-81AE-AC4CFE7C17AE}.Release|x64.Build.0 = Release|Any CPU
 		{093435F1-2E61-4005-81AE-AC4CFE7C17AE}.Release|x86.ActiveCfg = Release|Any CPU
 		{093435F1-2E61-4005-81AE-AC4CFE7C17AE}.Release|x86.Build.0 = Release|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Debug|x64.Build.0 = Debug|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Debug|x86.Build.0 = Debug|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Release|x64.ActiveCfg = Release|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Release|x64.Build.0 = Release|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Release|x86.ActiveCfg = Release|Any CPU
+		{1789B984-4451-4B95-8101-AC885C600986}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -65,5 +81,6 @@ Global
 		{5BDAA58E-C3C7-4A50-B867-04CD893D1D84} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{58B21565-1CB0-407F-948D-05F51AF335DF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{093435F1-2E61-4005-81AE-AC4CFE7C17AE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{1789B984-4451-4B95-8101-AC885C600986} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
 	EndGlobalSection
 EndGlobal

README.md

@@ -1,40 +1,186 @@
-# LinkTracker Bot
+# LinkTracker
 
-LinkTracker is a microservice-based system that monitors GitHub repositories and StackOverflow questions, sending real-time update notifications via Telegram.
+[![.NET](https://github.com/666mxvbee/LinkTracker/actions/workflows/dotnet.yml/badge.svg)](https://github.com/666mxvbee/LinkTracker/actions/workflows/dotnet.yml)
+[![Docker Image CI](https://github.com/666mxvbee/LinkTracker/actions/workflows/docker-image.yml/badge.svg)](https://github.com/666mxvbee/LinkTracker/actions/workflows/docker-image.yml)
+[![License](https://img.shields.io/github/license/666mxvbee/LinkTracker)](LICENSE)
+[![Release](https://img.shields.io/github/v/release/666mxvbee/LinkTracker?include_prereleases&sort=semver)](https://github.com/666mxvbee/LinkTracker/releases)
+[![.NET 9](https://img.shields.io/badge/.NET-9.0-512BD4?logo=dotnet)](https://dotnet.microsoft.com/)
+[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-16-4169E1?logo=postgresql&logoColor=white)](https://www.postgresql.org/)
+[![Docker](https://img.shields.io/badge/Docker-Compose-2496ED?logo=docker&logoColor=white)](docker-compose.yml)
+[![Tests](https://img.shields.io/badge/tests-xUnit%20%2B%20Testcontainers-5A2D82)](tests/LinkTracker.Scrapper.Tests)
+[![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20Docker-lightgrey)](docker-compose.yml)
+
+LinkTracker is a .NET 9 microservice application for tracking GitHub repositories and StackOverflow questions. The Bot service handles Telegram interaction, and the Scrapper service stores subscriptions, checks links on a schedule, and sends update notifications back to the Bot.
 
 ## Project Structure
 
-    src/LinkTracker.Bot - ASP.NET Core Web API for Telegram interaction.
+```text
+src/LinkTracker.Bot       Telegram bot HTTP service
+src/LinkTracker.Scrapper  Subscription storage and scheduled update checker
+src/LinkTracker.Shared    Shared DTOs
+migrations/               SQL migrations applied by Scrapper on startup
+```
 
-    src/LinkTracker.Scrapper - Quartz-based worker for resource monitoring.
+## Prerequisites
 
-    src/LinkTracker.Shared - Common models and DTOs shared between services.
+- .NET 9 SDK
+- Docker Desktop
+- Telegram bot token from BotFather
 
-## Quick Start (Docker)
+## Configuration
 
-The easiest way to run the entire infrastructure is using Docker Compose.
+Create `.env` in the repository root:
 
-### 1. Prerequisites
-* [Docker Desktop](https://www.docker.com/products/docker-desktop/) installed and running.
-* A Telegram Bot Token from [@BotFather](https://t.me/botfather).
+```env
+TELEGRAM_BOT_TOKEN=your-telegram-bot-token
 
-### 2. Configuration
-Create a file named `.env` in the root directory of the project:
+POSTGRES_DB=linktracker
+POSTGRES_USER=linktracker
+POSTGRES_PASSWORD=linktracker
+```
 
-```env
-TELEGRAM_BOT_TOKEN=your_token_here
+`.env` is ignored by git. Use `.env.example` as the template.
+
+Scrapper database settings are in `src/LinkTracker.Scrapper/appsettings.json`:
+
+```json
+"Database": {
+  "AccessType": "SQL",
+  "ConnectionString": "Host=localhost;Port=5433;Database=linktracker;Username=linktracker;Password=linktracker",
+  "RunMigrations": true
+}
 ```
-Note: The .env file is ignored by git for security purposes.
 
-### 3. Launch
+`AccessType` can be:
 
-Run the following command in the root folder:
-Bash
+```text
+SQL  raw SQL repositories via Npgsql
+ORM  EF Core repositories
+```
+
+Scheduler settings:
+
+```json
+"Scrapper": {
+  "CheckIntervalSeconds": 30,
+  "BatchSize": 100,
+  "Parallelism": 4,
+  "GitHubBaseUrl": "https://api.github.com/",
+  "StackOverflowBaseUrl": "https://api.stackexchange.com/2.3/"
+}
+```
 
-docker-compose up --build
+`BatchSize` is clamped by the application to `50..500`. `Parallelism` controls how many links are processed concurrently.
 
-Once started:
+## Run With Docker Compose
+
+Run all services:
+
+```powershell
+docker compose up --build
+```
+
+Endpoints:
+
+```text
+Bot API:       http://localhost:5100
+Scrapper API: http://localhost:5000
+PostgreSQL:   localhost:5433
+```
+
+Inside Docker, Scrapper connects to PostgreSQL by service name:
+
+```text
+Host=postgres;Port=5432
+```
+
+## Run From IDE
+
+Start PostgreSQL first:
+
+```powershell
+docker compose up postgres -d
+```
+
+Then run the services from IDE or terminal:
+
+```powershell
+dotnet run --project src\LinkTracker.Scrapper
+dotnet run --project src\LinkTracker.Bot
+```
 
-    Bot API: http://localhost:5100
+Scrapper applies SQL migrations from `migrations/` automatically when `Database:RunMigrations` is `true`.
+
+## Useful Manual Checks
+
+List database tables:
+
+```powershell
+docker exec -e PGPASSWORD=linktracker linktracker-postgres psql -U linktracker -d linktracker -c "\dt"
+```
+
+Expected domain tables:
+
+```text
+chats
+links
+chat_links
+tags
+chat_link_tags
+```
+
+DbUp also creates:
+
+```text
+schemaversions
+```
+
+Open Scrapper Swagger:
+
+```text
+http://localhost:5000/swagger
+```
+
+Basic API flow:
+
+```text
+POST   /tg-chat/{id}
+POST   /links      with Tg-Chat-Id header
+GET    /links      with Tg-Chat-Id header
+DELETE /links      with Tg-Chat-Id header
+GET    /tags
+POST   /tags
+PUT    /tags/{id}
+DELETE /tags/{id}
+```
+
+## Update Checking
+
+Scrapper uses Quartz to periodically process tracked links in batches.
+
+For GitHub links, it detects new:
+
+```text
+Issue
+Pull request
+```
+
+For StackOverflow links, it detects new:
+
+```text
+Answer
+Question comment
+Answer comment
+```
+
+Notifications include:
+
+```text
+type of update
+title
+user name
+creation time
+text preview limited to 200 characters
+```
 
-    Scrapper API: http://localhost:5000
+The notification sender is abstracted behind `IMessageSender`. The current implementation is HTTP from Scrapper to Bot; another implementation such as Kafka can be added later without changing the scheduler business logic.

src/LinkTracker.Scrapper/Clients/GitHubClient.cs

@@ -1,35 +1,98 @@
 using System.Text.Json;
+using LinkTracker.Scrapper.Services.Updates;
 
 namespace LinkTracker.Scrapper.Clients;
 
-public class GitHubClient(HttpClient httpClient, ILogger<GitHubClient> logger)
+public class GitHubClient(HttpClient httpClient) : ILinkUpdateChecker
 {
-    public async Task<DateTimeOffset?> GetLastUpdate(string owner, string repo)
+    public bool CanHandle(string url)
     {
-        try
+        return Uri.TryCreate(url, UriKind.Absolute, out var uri)
+            && uri.Host.Contains("github.com", StringComparison.OrdinalIgnoreCase);
+    }
+
+    public async Task<IReadOnlyList<DetectedLinkUpdate>> CheckUpdatesAsync(
+        string url,
+        DateTimeOffset since,
+        CancellationToken cancellationToken)
+    {
+        if (!TryParseRepository(url, out var owner, out var repo))
         {
-            httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("LinkTrackerBot/1.0");
+            return [];
+        }
+
+        var issues = await FetchAsync(owner, repo, "issues", "Issue", since, cancellationToken);
+        var pulls = await FetchAsync(owner, repo, "pulls", "Pull request", since, cancellationToken);
+
+        return issues
+            .Concat(pulls)
+            .OrderBy(update => update.CreatedAt)
+            .ToArray();
+    }
+
+    private async Task<IReadOnlyList<DetectedLinkUpdate>> FetchAsync(
+        string owner,
+        string repo,
+        string resource,
+        string kind,
+        DateTimeOffset since,
+        CancellationToken cancellationToken)
+    {
+        using var response = await httpClient.GetAsync(
+            $"repos/{owner}/{repo}/{resource}?state=all&sort=created&direction=desc&per_page=100",
+            cancellationToken);
+
+        response.EnsureSuccessStatusCode();
+
+        var json = await response.Content.ReadFromJsonAsync<JsonElement>(cancellationToken);
 
-            var response = await httpClient.GetAsync($"https://api.github.com/repos/{owner}/{repo}");
+        var updates = new List<DetectedLinkUpdate>();
 
-            if (!response.IsSuccessStatusCode)
+        foreach (var item in json.EnumerateArray())
+        {
+            if (resource == "issues" && item.TryGetProperty("pull_request", out _))
             {
-                logger.LogWarning("GitHub API made error {Code} to {Owner}/{Repo}", response.StatusCode, owner, repo);
-                return null;
+                continue;
             }
 
-            var json = await response.Content.ReadFromJsonAsync<JsonElement>();
+            var createdAt = item.GetProperty("created_at").GetDateTimeOffset();
 
-            if (json.TryGetProperty("pushed_at", out var dateProp))
+            if (createdAt <= since)
             {
-                return dateProp.GetDateTimeOffset();
+                continue;
             }
+
+            updates.Add(new DetectedLinkUpdate(
+                Url: item.GetProperty("html_url").GetString() ?? string.Empty,
+                Kind: kind,
+                Title: item.GetProperty("title").GetString() ?? string.Empty,
+                UserName: item.GetProperty("user").GetProperty("login").GetString() ?? "unknown",
+                CreatedAt: createdAt,
+                Preview: PreviewBuilder.Build(item.GetProperty("body").GetString())));
         }
-        catch (Exception ex)
+
+        return updates;
+    }
+
+    private static bool TryParseRepository(string url, out string owner, out string repo)
+    {
+        owner = string.Empty;
+        repo = string.Empty;
+
+        if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
+        {
+            return false;
+        }
+
+        var parts = uri.AbsolutePath.Trim('/').Split('/', StringSplitOptions.RemoveEmptyEntries);
+
+        if (parts.Length < 2)
         {
-            logger.LogError(ex, "Error GitHub API");
+            return false;
         }
 
-        return null;
+        owner = parts[0];
+        repo = parts[1];
+        return true;
     }
 }