From 14533fb3a5b2db4b15785717eb4c90fb5bac37b8 Mon Sep 17 00:00:00 2001 From: 666mxvbee Date: Tue, 26 May 2026 23:01:11 +0300 Subject: [PATCH 1/3] feat(scrapper): implement orm repositories --- .../LinkTracker.Scrapper.csproj | 4 - .../Repositories/Orm/OrmLinkRepository.cs | 242 ++++++++++++++++++ .../Repositories/Orm/OrmTagRepository.cs | 96 +++++++ 3 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 src/LinkTracker.Scrapper/Repositories/Orm/OrmLinkRepository.cs create mode 100644 src/LinkTracker.Scrapper/Repositories/Orm/OrmTagRepository.cs diff --git a/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj b/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj index 9fcf772..c206ac9 100644 --- a/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj +++ b/src/LinkTracker.Scrapper/LinkTracker.Scrapper.csproj @@ -35,8 +35,4 @@ PreserveNewest - - - - diff --git a/src/LinkTracker.Scrapper/Repositories/Orm/OrmLinkRepository.cs b/src/LinkTracker.Scrapper/Repositories/Orm/OrmLinkRepository.cs new file mode 100644 index 0000000..19b958e --- /dev/null +++ b/src/LinkTracker.Scrapper/Repositories/Orm/OrmLinkRepository.cs @@ -0,0 +1,242 @@ +using LinkTracker.Scrapper.Database; +using LinkTracker.Scrapper.Database.Entities; +using LinkTracker.Shared.Models; +using Microsoft.EntityFrameworkCore; + +namespace LinkTracker.Scrapper.Repositories.Orm; + +public class OrmLinkRepository(LinkTrackerDbContext dbContext) : ILinkRepository +{ + public void AddChat(long chatId) + { + if (dbContext.Chats.Any(chat => chat.Id == chatId)) + { + return; + } + + dbContext.Chats.Add(new ChatEntity + { + Id = chatId + }); + + dbContext.SaveChanges(); + } + + public void RemoveChat(long chatId) + { + var chat = dbContext.Chats.FirstOrDefault(chat => chat.Id == chatId); + + if (chat is null) + { + return; + } + + dbContext.Chats.Remove(chat); + dbContext.SaveChanges(); + } + + public bool ChatExists(long chatId) + { + return dbContext.Chats + .AsNoTracking() + .Any(chat => chat.Id == chatId); + } + + public LinkResponse? AddLink(long chatId, string url, string[]? tags) + { + var normalizedTags = NormalizeTags(tags); + + using var transaction = dbContext.Database.BeginTransaction(); + + if (!dbContext.Chats.Any(chat => chat.Id == chatId)) + { + return null; + } + + var link = dbContext.Links.FirstOrDefault(link => link.Url == url); + + if (link is null) + { + link = new LinkEntity + { + Url = url, + LastCheckedAt = DateTimeOffset.UtcNow + }; + + dbContext.Links.Add(link); + dbContext.SaveChanges(); + } + + var subscriptionExists = dbContext.ChatLinks.Any(chatLink => + chatLink.ChatId == chatId && chatLink.LinkId == link.Id); + + if (subscriptionExists) + { + return null; + } + + dbContext.ChatLinks.Add(new ChatLinkEntity + { + ChatId = chatId, + LinkId = link.Id + }); + + dbContext.SaveChanges(); + + foreach (var tagName in normalizedTags) + { + var tag = dbContext.Tags.FirstOrDefault(tag => tag.Name == tagName); + + if (tag is null) + { + tag = new TagEntity + { + Name = tagName + }; + + dbContext.Tags.Add(tag); + dbContext.SaveChanges(); + } + + dbContext.ChatLinkTags.Add(new ChatLinkTagEntity + { + ChatId = chatId, + LinkId = link.Id, + TagId = tag.Id + }); + } + + dbContext.SaveChanges(); + transaction.Commit(); + + return new LinkResponse(link.Id, link.Url, normalizedTags); + } + + public bool RemoveLink(long chatId, string url) + { + var chatLink = dbContext.ChatLinks.FirstOrDefault(chatLink => + chatLink.ChatId == chatId && chatLink.Link.Url == url); + + if (chatLink is null) + { + return false; + } + + dbContext.ChatLinks.Remove(chatLink); + return dbContext.SaveChanges() > 0; + } + + public IEnumerable GetLinks(long chatId, string? tag = null, int offset = 0, int limit = 100) + { + var query = dbContext.ChatLinks + .AsNoTracking() + .Where(chatLink => chatLink.ChatId == chatId); + + if (!string.IsNullOrWhiteSpace(tag)) + { + var normalizedTag = tag.Trim(); + + query = query.Where(chatLink => + chatLink.ChatLinkTags.Any(chatLinkTag => + EF.Functions.ILike(chatLinkTag.Tag.Name, normalizedTag))); + } + + var chatLinks = query + .Include(chatLink => chatLink.Link) + .Include(chatLink => chatLink.ChatLinkTags) + .ThenInclude(chatLinkTag => chatLinkTag.Tag) + .OrderBy(chatLink => chatLink.CreatedAt) + .ThenBy(chatLink => chatLink.LinkId) + .Skip(NormalizeOffset(offset)) + .Take(NormalizeLimit(limit)) + .AsSplitQuery() + .ToList(); + + return chatLinks + .Select(chatLink => new LinkResponse( + chatLink.Link.Id, + chatLink.Link.Url, + chatLink.ChatLinkTags + .Select(chatLinkTag => chatLinkTag.Tag.Name) + .OrderBy(tagName => tagName) + .ToArray())) + .ToList(); + } + + public IEnumerable<(string Url, long[] ChatIds, DateTimeOffset LastUpdate)> GetLinksForUpdate( + int offset = 0, + int limit = 100) + { + var links = dbContext.Links + .AsNoTracking() + .Where(link => link.ChatLinks.Any()) + .OrderBy(link => link.Id) + .Skip(NormalizeOffset(offset)) + .Take(NormalizeLimit(limit)) + .Select(link => new + { + link.Id, + link.Url, + link.LastCheckedAt + }) + .ToList(); + + var linkIds = links.Select(link => link.Id).ToArray(); + + var chatIdsByLinkId = dbContext.ChatLinks + .AsNoTracking() + .Where(chatLink => linkIds.Contains(chatLink.LinkId)) + .Select(chatLink => new + { + chatLink.LinkId, + chatLink.ChatId + }) + .ToList() + .GroupBy(chatLink => chatLink.LinkId) + .ToDictionary( + group => group.Key, + group => group + .Select(chatLink => chatLink.ChatId) + .OrderBy(chatId => chatId) + .ToArray()); + + return links + .Select(link => ( + link.Url, + chatIdsByLinkId.GetValueOrDefault(link.Id, Array.Empty()), + link.LastCheckedAt)) + .ToList(); + } + + public void UpdateLastCheckTime(string url, DateTimeOffset lastUpdate) + { + var link = dbContext.Links.FirstOrDefault(link => link.Url == url); + + if (link is null) + { + return; + } + + link.LastCheckedAt = lastUpdate.ToUniversalTime(); + dbContext.SaveChanges(); + } + + private static string[] NormalizeTags(string[]? tags) + { + return tags? + .Where(tag => !string.IsNullOrWhiteSpace(tag)) + .Select(tag => tag.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? Array.Empty(); + } + + private static int NormalizeOffset(int offset) + { + return Math.Max(0, offset); + } + + private static int NormalizeLimit(int limit) + { + return Math.Clamp(limit, 1, 1000); + } +} \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Repositories/Orm/OrmTagRepository.cs b/src/LinkTracker.Scrapper/Repositories/Orm/OrmTagRepository.cs new file mode 100644 index 0000000..4a7f387 --- /dev/null +++ b/src/LinkTracker.Scrapper/Repositories/Orm/OrmTagRepository.cs @@ -0,0 +1,96 @@ +using LinkTracker.Scrapper.Database; +using LinkTracker.Scrapper.Database.Entities; +using LinkTracker.Shared.Models; +using Microsoft.EntityFrameworkCore; + +namespace LinkTracker.Scrapper.Repositories.Orm; + +public class OrmTagRepository(LinkTrackerDbContext dbContext) : ITagRepository +{ + public TagResponse Create(string name) + { + var normalizedName = name.Trim(); + + var existingTag = dbContext.Tags + .AsNoTracking() + .FirstOrDefault(tag => tag.Name == normalizedName); + + if (existingTag is not null) + { + return ToResponse(existingTag); + } + + var tag = new TagEntity + { + Name = normalizedName + }; + + dbContext.Tags.Add(tag); + dbContext.SaveChanges(); + + return ToResponse(tag); + } + + public TagResponse? Get(long id) + { + return dbContext.Tags + .AsNoTracking() + .Where(tag => tag.Id == id) + .Select(tag => new TagResponse(tag.Id, tag.Name)) + .FirstOrDefault(); + } + + public IEnumerable GetAll(int offset = 0, int limit = 100) + { + return dbContext.Tags + .AsNoTracking() + .OrderBy(tag => tag.Id) + .Skip(NormalizeOffset(offset)) + .Take(NormalizeLimit(limit)) + .Select(tag => new TagResponse(tag.Id, tag.Name)) + .ToList(); + } + + public TagResponse? Update(long id, string name) + { + var tag = dbContext.Tags.FirstOrDefault(tag => tag.Id == id); + + if (tag is null) + { + return null; + } + + tag.Name = name.Trim(); + dbContext.SaveChanges(); + + return ToResponse(tag); + } + + public bool Delete(long id) + { + var tag = dbContext.Tags.FirstOrDefault(tag => tag.Id == id); + + if (tag is null) + { + return false; + } + + dbContext.Tags.Remove(tag); + return dbContext.SaveChanges() > 0; + } + + private static TagResponse ToResponse(TagEntity tag) + { + return new TagResponse(tag.Id, tag.Name); + } + + private static int NormalizeOffset(int offset) + { + return Math.Max(0, offset); + } + + private static int NormalizeLimit(int limit) + { + return Math.Clamp(limit, 1, 1000); + } +} \ No newline at end of file From 207a094bda3f18a0f5a08f3491457fca12646a48 Mon Sep 17 00:00:00 2001 From: 666mxvbee Date: Wed, 27 May 2026 16:36:40 +0300 Subject: [PATCH 2/3] feat(scrapper): wire database repositories --- src/LinkTracker.Scrapper/Program.cs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/LinkTracker.Scrapper/Program.cs b/src/LinkTracker.Scrapper/Program.cs index 7a92368..17d9a0b 100644 --- a/src/LinkTracker.Scrapper/Program.cs +++ b/src/LinkTracker.Scrapper/Program.cs @@ -4,7 +4,10 @@ using LinkTracker.Scrapper.Jobs; using LinkTracker.Scrapper.Configuration; using LinkTracker.Scrapper.Database; -using Microsoft.EntityFrameworkCore.Migrations; +using LinkTracker.Scrapper.Repositories.Sql; +using LinkTracker.Scrapper.Repositories.Orm; +using Microsoft.EntityFrameworkCore; +using Npgsql; var builder = WebApplication.CreateBuilder(args); @@ -14,7 +17,25 @@ builder.Services.Configure( builder.Configuration.GetSection(DatabaseOptions.SectionName)); -builder.Services.AddSingleton(); +var databaseOptions = builder.Configuration + .GetSection(DatabaseOptions.SectionName) + .Get() ?? new DatabaseOptions(); + +builder.Services.AddSingleton(_ => NpgsqlDataSource.Create(databaseOptions.ConnectionString)); + +builder.Services.AddDbContext(options => + options.UseNpgsql(databaseOptions.ConnectionString)); + +if (databaseOptions.AccessType.Equals("ORM", StringComparison.OrdinalIgnoreCase)) +{ + builder.Services.AddScoped(); + builder.Services.AddScoped(); +} +else +{ + builder.Services.AddScoped(); + builder.Services.AddScoped(); +} builder.Services.AddHttpClient(); builder.Services.AddHttpClient(); @@ -53,4 +74,4 @@ app.MapControllers(); -app.Run(); \ No newline at end of file +app.Run(); From 8600138179ccb2ab8c94b3cfc32ed58a57125a0e Mon Sep 17 00:00:00 2001 From: 666mxvbee Date: Wed, 27 May 2026 16:39:21 +0300 Subject: [PATCH 3/3] feat(scrapper): add tag endpoints --- .../Controllers/LinksController.cs | 23 +++----- .../Controllers/TagsController.cs | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 src/LinkTracker.Scrapper/Controllers/TagsController.cs diff --git a/src/LinkTracker.Scrapper/Controllers/LinksController.cs b/src/LinkTracker.Scrapper/Controllers/LinksController.cs index c8393c7..087bb1f 100644 --- a/src/LinkTracker.Scrapper/Controllers/LinksController.cs +++ b/src/LinkTracker.Scrapper/Controllers/LinksController.cs @@ -9,27 +9,18 @@ namespace LinkTracker.Scrapper.Controllers; public class LinksController(ILinkRepository repo) : ControllerBase { [HttpGet] - public IActionResult Get([FromHeader(Name = "Tg-Chat-Id")] long chatId, [FromQuery] string? tag = null) + public IActionResult Get( + [FromHeader(Name = "Tg-Chat-Id")] long chatId, + [FromQuery] string? tag = null, + [FromQuery] int offset = 0, + [FromQuery] int limit = 100) { if (!repo.ChatExists(chatId)) { return NotFound("Chat is not registered"); } - var links = repo.GetLinks(chatId).ToList(); - - if (!string.IsNullOrEmpty(tag)) - { - links = links - .Where(l => l.Tags != null && l.Tags.Contains(tag, StringComparer.OrdinalIgnoreCase)) - .ToList(); - } - - var responseLinks = links.Select(l => new LinkResponse( - l.Id, - l.Url, - l.Tags ?? Array.Empty() - )).ToArray(); + var responseLinks = repo.GetLinks(chatId, tag, offset, limit).ToArray(); return Ok(new ListLinksResponse(responseLinks, responseLinks.Length)); } @@ -66,4 +57,4 @@ public IActionResult Remove([FromHeader(Name = "Tg-Chat-Id")] long chatId, [From return Ok(req); } -} \ No newline at end of file +} diff --git a/src/LinkTracker.Scrapper/Controllers/TagsController.cs b/src/LinkTracker.Scrapper/Controllers/TagsController.cs new file mode 100644 index 0000000..a0aca03 --- /dev/null +++ b/src/LinkTracker.Scrapper/Controllers/TagsController.cs @@ -0,0 +1,58 @@ +using LinkTracker.Scrapper.Repositories; +using LinkTracker.Shared.Models; +using Microsoft.AspNetCore.Mvc; + +namespace LinkTracker.Scrapper.Controllers; + +[ApiController] +[Route("tags")] +public class TagsController(ITagRepository repo) : ControllerBase +{ + [HttpGet] + public IActionResult GetAll([FromQuery] int offset = 0, [FromQuery] int limit = 100) + { + var tags = repo.GetAll(offset, limit).ToArray(); + + return Ok(tags); + } + + [HttpGet("{id:long}")] + public IActionResult Get(long id) + { + var tag = repo.Get(id); + + return tag is null ? NotFound() : Ok(tag); + } + + [HttpPost] + public IActionResult Create([FromBody] CreateTagRequest request) + { + if (string.IsNullOrWhiteSpace(request.Name)) + { + return BadRequest("Tag name is required"); + } + + var tag = repo.Create(request.Name); + + return CreatedAtAction(nameof(Get), new { id = tag.Id }, tag); + } + + [HttpPut("{id:long}")] + public IActionResult Update(long id, [FromBody] UpdateTagRequest request) + { + if (string.IsNullOrWhiteSpace(request.Name)) + { + return BadRequest("Tag name is required"); + } + + var tag = repo.Update(id, request.Name); + + return tag is null ? NotFound() : Ok(tag); + } + + [HttpDelete("{id:long}")] + public IActionResult Delete(long id) + { + return repo.Delete(id) ? NoContent() : NotFound(); + } +}