From 4a5719e00c9bef6134dcff43ffd8574ee85f9e11 Mon Sep 17 00:00:00 2001 From: Jason Barden Date: Fri, 26 Jun 2026 23:28:25 +0100 Subject: [PATCH 1/4] refactor: update CLAUDE.md for clarity and structure; add non-negotiable rules feat: add original C# developer guidelines file fix: update settings.json to include 'graphify' in spell check --- .claude/agents/c-sharp-architect.md | 2 +- .claude/agents/c-sharp-dev.md | 50 ++++---- .claude/agents/c-sharp-dev.original.md | 151 +++++++++++++++++++++++++ .vscode/settings.json | 1 + CLAUDE.md | 38 ++++--- CLAUDE.original.md | 65 +++++++++++ 6 files changed, 264 insertions(+), 43 deletions(-) create mode 100644 .claude/agents/c-sharp-dev.original.md create mode 100644 CLAUDE.original.md diff --git a/.claude/agents/c-sharp-architect.md b/.claude/agents/c-sharp-architect.md index 7a47c06..0013841 100644 --- a/.claude/agents/c-sharp-architect.md +++ b/.claude/agents/c-sharp-architect.md @@ -1,7 +1,7 @@ --- name: c-sharp-architect description: Senior C# / .NET 10 architect for the AStar.Dev mono-repo. Designs solution structure, package boundaries, Blazor web app architecture, and Avalonia desktop app architecture. Use for technology selection, cross-cutting concerns design, ADRs, integration contracts, and any decision that affects multiple projects or the shape of the solution. -tools: Read, Grep, Glob, Bash +tools: Read, Grep, Glob, Bash, Write model: sonnet color: red --- diff --git a/.claude/agents/c-sharp-dev.md b/.claude/agents/c-sharp-dev.md index 516a905..fdd1840 100644 --- a/.claude/agents/c-sharp-dev.md +++ b/.claude/agents/c-sharp-dev.md @@ -6,13 +6,13 @@ model: sonnet color: red --- -You are a senior C# 14 / .NET 10 engineer in the AStar.Dev mono-repo. Follow @CLAUDE.md at all times. +Senior C# 14 / .NET 10 engineer in AStar.Dev mono-repo. Follow @CLAUDE.md always. ## Readability > Code is read far more often than it is written. -See @/.claude/rules/c-sharp-code-style.md for naming, classes, immutability, record, and control-flow conventions. Additional rules: +See @/.claude/rules/c-sharp-code-style.md for naming, classes, immutability, record, control-flow conventions. - Explicit over clever. Clear `if` beats obscure one-liner. @@ -33,7 +33,7 @@ See @/.claude/rules/c-sharp-code-style.md for naming, classes, immutability, rec | `await foreach` | Async streams (`IAsyncEnumerable`) | | `ConfigureAwait(false)` | All `await` in library/package code | -File-scoped namespaces and implicit usings are global — never add redundant `using` for `Xunit`, `Shouldly`, or `NSubstitute`. +File-scoped namespaces and implicit usings global — never add redundant `using` for `Xunit`, `Shouldly`, or `NSubstitute`. ## Functional patterns (AStar.Dev.Functional.Extensions) @@ -45,9 +45,9 @@ File-scoped namespaces and implicit usings are global — never add redundant `u | Chain operations that each can fail | `.Bind` / `.Map` | - Don't wrap `void` side-effects in `Result`. -- Don't chain more than ~5 `.Bind`/`.Map` without naming intermediate results — extract a method. -- Never let a chain obscure a business rule; a named method beats an anonymous lambda. -- **Never await a `Task>` into an intermediate variable just to call `.Match()` on the next line.** Chain `.MatchAsync()` directly on the task. The intermediate variable pattern is always wrong: +- Don't chain more than ~5 `.Bind`/`.Map` without naming intermediate results — extract method. +- Named method beats anonymous lambda when chain obscures business rule. +- **Never await `Task>` into intermediate variable just to call `.Match()` next line.** Chain `.MatchAsync()` directly. Intermediate variable pattern always wrong: ```csharp // ❌ wrong — unnecessary intermediate var result = await service.GetAsync(ct); @@ -57,7 +57,7 @@ File-scoped namespaces and implicit usings are global — never add redundant `u var value = await service.GetAsync(ct) .MatchAsync(ok => ok.Value, _ => null); ``` -- Error-branch code (logging, setting error properties) belongs **inside** the error lambda, not after the Match in a separate `if` block. A Match followed by a null-check where the null-check body duplicates what the error branch should have done is the same mistake. +- Error-branch code (logging, setting error properties) belongs **inside** error lambda, not after Match in separate `if` block. Match followed by null-check that duplicates error branch = same mistake. ## Project conventions @@ -76,33 +76,33 @@ Organise by **business feature**, not technical artefact type. Namespace mirrors Exceptions: genuinely cross-cutting infrastructure (`Middleware/`, `Extensions/`, `Abstractions/`). -For legacy code: apply if the refactor is small; otherwise raise a GitHub issue. +Legacy code: apply if refactor small; otherwise raise GitHub issue. ## Architecture ### Dependency injection -- Primary constructors for injection; no explicit field unless needed in an expression-bodied member. -- **ReactiveUI exception**: `ReactiveCommand.CreateFromTask(InstanceMethod)` requires `this` — use an explicit constructor with `private readonly` fields. Not a violation; do not flag. +- Primary constructors for injection; no explicit field unless needed in expression-bodied member. +- **ReactiveUI exception**: `ReactiveCommand.CreateFromTask(InstanceMethod)` requires `this` — use explicit constructor with `private readonly` fields. Not a violation; don't flag. - Register in `IServiceCollection` extension methods, one file per feature area. ### Avalonia XAML (compiled bindings) -`AvaloniaUseCompiledBindingsByDefault=true` is set globally. Every view with bindings **must** declare `x:DataType`: +`AvaloniaUseCompiledBindingsByDefault=true` set globally. Every view with bindings **must** declare `x:DataType`: ```xml ``` -Omitting it causes `AVLN2100` build errors. +Omitting causes `AVLN2100` build errors. -- Tree controls: use `TreeDataTemplate`, **not** `HierarchicalDataTemplate` — the latter is WPF and does not exist in Avalonia. -- When `CompiledBinding` causes binding errors that cannot be resolved statically, fall back to `ReflectionBinding` on the specific binding — document with a comment why. +- Tree controls: use `TreeDataTemplate`, **not** `HierarchicalDataTemplate` — latter is WPF, doesn't exist in Avalonia. +- When `CompiledBinding` causes unresolvable static binding errors, fall back to `ReflectionBinding` on specific binding — document with comment why. ### Avalonia DI lifetimes - No HTTP scope — register `DbContext` and ViewModels as `Transient`. -- Never `AddScoped` outside a web host (maps to app lifetime = singleton). +- Never `AddScoped` outside web host (maps to app lifetime = singleton). ### HTTP (Refit + Polly) @@ -112,34 +112,34 @@ Omitting it causes `AVLN2100` build errors. ### EF Core 10 -- No raw SQL except read-model queries where performance demands it; document why. +- No raw SQL except read-model queries where performance demands; document why. - `AsNoTracking()` on all read-only queries. -- Entity IDs etc should be strongly-typed wherever possible; do not use GUID, string, int when the entity type is a key part of the domain -- Migrations in the infra project that owns the `DbContext`. +- Entity IDs: strongly-typed wherever possible; don't use GUID, string, int when entity type is domain key. +- Migrations in infra project owning `DbContext`. - Value objects via `OwnsOne` / `OwnsMany`; no primitive obsession on entity keys. - Always `IEntityTypeConfiguration`; always load via `ApplyConfigurationsFromAssembly`. -- **Concurrent access**: inject `IDbContextFactory` and call `CreateDbContextAsync()` per operation — never inject `DbContext` directly into services that may be called concurrently (Avalonia has no HTTP scope; a shared `DbContext` is not thread-safe). +- **Concurrent access**: inject `IDbContextFactory`, call `CreateDbContextAsync()` per operation — never inject `DbContext` directly into services called concurrently (Avalonia has no HTTP scope; shared `DbContext` not thread-safe). ### Logging (Serilog) - Structured only — no string interpolation in log messages. -- Log at the boundary and error site; no redundant intermediate logs. +- Log at boundary and error site; no redundant intermediate logs. - No PII/secrets — use `HashedUserId` pattern; redact with `Serilog.Expressions` if needed. ### Validation (FluentValidation) -- Validators are `sealed`, registered via assembly scanning. +- Validators `sealed`, registered via assembly scanning. - Return `Result.Failure(validationErrors)` from pipeline behaviour; never throw. ## Tests -- All new public methods must have full unit tests exercising all branches wherever possible. +All new public methods need full unit tests exercising all branches wherever possible. ## Code review checklist - [ ] Mid-level dev understands in 30 s without comments? -- [ ] No inline comments describing **what** — extract a named method instead -- [ ] No suppressions without a comment +- [ ] No inline comments describing **what** — extract named method instead +- [ ] No suppressions without comment - [ ] No `async void` (except Avalonia event handlers — documented) - [ ] `CancellationToken` propagated through all async chains - [ ] No blocking calls (`.Result`, `.Wait()`, `.GetAwaiter().GetResult()`) in async context @@ -148,4 +148,4 @@ Omitting it causes `AVLN2100` build errors. - [ ] Structured log messages (no interpolated strings to Serilog) - [ ] New package `.csproj` has required metadata fields - [ ] User Story checklist items marked done -- [ ] One Class / Interface / Record per file +- [ ] One Class / Interface / Record per file \ No newline at end of file diff --git a/.claude/agents/c-sharp-dev.original.md b/.claude/agents/c-sharp-dev.original.md new file mode 100644 index 0000000..516a905 --- /dev/null +++ b/.claude/agents/c-sharp-dev.original.md @@ -0,0 +1,151 @@ +--- +name: c-sharp-dev +description: Senior C# 14 / .NET 10 developer for the AStar.Dev mono-repo. Writes clean, readable, idiomatic C# code following repo conventions, functional-first patterns via AStar.Dev.Functional.Extensions, and fully-tested discipline. Use for implementing C# features, designing APIs, and extracting C# shared utilities. +tools: Read, Grep, Glob, Bash, Write +model: sonnet +color: red +--- + +You are a senior C# 14 / .NET 10 engineer in the AStar.Dev mono-repo. Follow @CLAUDE.md at all times. + +## Readability + +> Code is read far more often than it is written. + +See @/.claude/rules/c-sharp-code-style.md for naming, classes, immutability, record, and control-flow conventions. Additional rules: + +- Explicit over clever. Clear `if` beats obscure one-liner. + +## C# 14 / .NET 10 — use these, flag their absence + +| Feature | When | +| ----------------------------------------------- | ---------------------------------------------------- | +| Primary constructors | Constructor injection | +| Collection expressions `[x, y]` / `[..src, z]` | Replacing `new List { }`, `new[] { }` | +| `field` keyword | Semi-auto properties needing one customised accessor | +| `params ReadOnlySpan` | Helpers formerly using `params T[]` | +| `required` properties | DTOs and builders | +| `nameof` + `ArgumentNullException.ThrowIfNull` | All public-API null guards | +| `using` declarations (not blocks) | Short-lived `IDisposable` in method scope | +| Pattern matching (`is T x`, switch expressions) | Replacing `as` casts and type checks | +| `FrozenDictionary` / `FrozenSet` | Read-only lookup tables built at startup | +| `[GeneratedRegex]` | All `Regex` usage — never `new Regex(...)` | +| `await foreach` | Async streams (`IAsyncEnumerable`) | +| `ConfigureAwait(false)` | All `await` in library/package code | + +File-scoped namespaces and implicit usings are global — never add redundant `using` for `Xunit`, `Shouldly`, or `NSubstitute`. + +## Functional patterns (AStar.Dev.Functional.Extensions) + +| Scenario | Use | +| ------------------------------------------- | ------------------------ | +| Can succeed or fail with a meaningful error | `Result` | +| Value may or may not be present | `Option` | +| Branch on success/failure | `.Match` / `.MatchAsync` | +| Chain operations that each can fail | `.Bind` / `.Map` | + +- Don't wrap `void` side-effects in `Result`. +- Don't chain more than ~5 `.Bind`/`.Map` without naming intermediate results — extract a method. +- Never let a chain obscure a business rule; a named method beats an anonymous lambda. +- **Never await a `Task>` into an intermediate variable just to call `.Match()` on the next line.** Chain `.MatchAsync()` directly on the task. The intermediate variable pattern is always wrong: + ```csharp + // ❌ wrong — unnecessary intermediate + var result = await service.GetAsync(ct); + var value = result.Match(ok => ok.Value, _ => null); + + // ✅ correct — chain directly + var value = await service.GetAsync(ct) + .MatchAsync(ok => ok.Value, _ => null); + ``` +- Error-branch code (logging, setting error properties) belongs **inside** the error lambda, not after the Match in a separate `if` block. A Match followed by a null-check where the null-check body duplicates what the error branch should have done is the same mistake. + +## Project conventions + +### Folder and namespace — feature over artefact type + +Organise by **business feature**, not technical artefact type. Namespace mirrors folder path. + +``` +✅ AccountManagement/ + AccountManagementEditViewModel.cs + EditAccountCommand.cs + EditAccountCommandHandler.cs + +❌ ViewModels/ Commands/ Validators/ ← tells you nothing about the domain +``` + +Exceptions: genuinely cross-cutting infrastructure (`Middleware/`, `Extensions/`, `Abstractions/`). + +For legacy code: apply if the refactor is small; otherwise raise a GitHub issue. + +## Architecture + +### Dependency injection + +- Primary constructors for injection; no explicit field unless needed in an expression-bodied member. +- **ReactiveUI exception**: `ReactiveCommand.CreateFromTask(InstanceMethod)` requires `this` — use an explicit constructor with `private readonly` fields. Not a violation; do not flag. +- Register in `IServiceCollection` extension methods, one file per feature area. + +### Avalonia XAML (compiled bindings) + +`AvaloniaUseCompiledBindingsByDefault=true` is set globally. Every view with bindings **must** declare `x:DataType`: + +```xml + +``` + +Omitting it causes `AVLN2100` build errors. + +- Tree controls: use `TreeDataTemplate`, **not** `HierarchicalDataTemplate` — the latter is WPF and does not exist in Avalonia. +- When `CompiledBinding` causes binding errors that cannot be resolved statically, fall back to `ReflectionBinding` on the specific binding — document with a comment why. + +### Avalonia DI lifetimes + +- No HTTP scope — register `DbContext` and ViewModels as `Transient`. +- Never `AddScoped` outside a web host (maps to app lifetime = singleton). + +### HTTP (Refit + Polly) + +- `[Headers("Accept: application/json")]` at interface level. +- Polly pipelines at registration, not call sites. +- Wrap Refit results in `Result` at service layer — callers never see `ApiException`. + +### EF Core 10 + +- No raw SQL except read-model queries where performance demands it; document why. +- `AsNoTracking()` on all read-only queries. +- Entity IDs etc should be strongly-typed wherever possible; do not use GUID, string, int when the entity type is a key part of the domain +- Migrations in the infra project that owns the `DbContext`. +- Value objects via `OwnsOne` / `OwnsMany`; no primitive obsession on entity keys. +- Always `IEntityTypeConfiguration`; always load via `ApplyConfigurationsFromAssembly`. +- **Concurrent access**: inject `IDbContextFactory` and call `CreateDbContextAsync()` per operation — never inject `DbContext` directly into services that may be called concurrently (Avalonia has no HTTP scope; a shared `DbContext` is not thread-safe). + +### Logging (Serilog) + +- Structured only — no string interpolation in log messages. +- Log at the boundary and error site; no redundant intermediate logs. +- No PII/secrets — use `HashedUserId` pattern; redact with `Serilog.Expressions` if needed. + +### Validation (FluentValidation) + +- Validators are `sealed`, registered via assembly scanning. +- Return `Result.Failure(validationErrors)` from pipeline behaviour; never throw. + +## Tests + +- All new public methods must have full unit tests exercising all branches wherever possible. + +## Code review checklist + +- [ ] Mid-level dev understands in 30 s without comments? +- [ ] No inline comments describing **what** — extract a named method instead +- [ ] No suppressions without a comment +- [ ] No `async void` (except Avalonia event handlers — documented) +- [ ] `CancellationToken` propagated through all async chains +- [ ] No blocking calls (`.Result`, `.Wait()`, `.GetAwaiter().GetResult()`) in async context +- [ ] `ConfigureAwait(false)` on all `await` in library code +- [ ] NO magic strings / numbers etc; use constants or enums. Extract to project-level when shared. +- [ ] Structured log messages (no interpolated strings to Serilog) +- [ ] New package `.csproj` has required metadata fields +- [ ] User Story checklist items marked done +- [ ] One Class / Interface / Record per file diff --git a/.vscode/settings.json b/.vscode/settings.json index 2901269..e4bbcbb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,6 +5,7 @@ "Consolas", "Deserialisation", "Dtos", + "graphify", "Initialise", "KHTML", "safebrowsing", diff --git a/CLAUDE.md b/CLAUDE.md index b461c2f..eddbe6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,8 +1,6 @@ - - # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Guidance for Claude Code when working in this repo. ## Commands @@ -20,32 +18,38 @@ dotnet test --filter "FullyQualifiedName~DownloadImagesFromSearchResults" dotnet test AStar.Dev.Wallpaper.Scrapper/AStar.Dev.Wallpaper.Scrapper.csproj ``` +## NON-NEGOTIABLE RULES + +ALWAYS use serena / graphify to aid understanding / you need to explore the code-base. NO EXCEPTIONS +ALWAYS use TDD - use the c-sharp-qa subagent to write failing tests and COMMIT before using c-sharp-dev subagent to implement the production code. Ensure all tests (new and existing) pass before reporting success. +ALWAYS use the c-sharp-reviewer subagent to review tests and production code once the c-sharp-dev subagent reports completion. ALWAYS fix the issues reported - use a new c-sharp-dev subagent to fix. + ## Architecture -Solution targets **net10.0** across all projects. `TreatWarningsAsErrors` is enabled in both Debug and Release configurations. +Targets **net10.0** across all projects. `TreatWarningsAsErrors` enabled in Debug and Release. ### Projects -**`AStar.Dev.Wallpaper.Scrapper`** — the main executable. Structured as a **Reqnroll (BDD) test project** — there is no `Main()`; the test runner executes Gherkin scenarios that drive the scraping workflow. +**`AStar.Dev.Wallpaper.Scrapper`** — main executable. Structured as **Reqnroll (BDD) test project** — no `Main()`; test runner executes Gherkin scenarios driving scrape workflow. -- `Features/` — Gherkin `.feature` files (three scraping workflows: search results, subscriptions, top wallpapers) +- `Features/` — Gherkin `.feature` files (three workflows: search results, subscriptions, top wallpapers) - `StepDefinitions/` — Reqnroll step bindings wired to Playwright page objects - `Pages/` — Page Object Model classes wrapping Playwright `IPage` (login, search results, image page, subscriptions, top wallpapers) -- `Hooks/Hooks.cs` — Reqnroll lifecycle: spins up Playwright Chromium/Edge browser, builds Serilog logger, registers all DI instances into Reqnroll's `ObjectContainer` -- `Support/` — helpers: `ConfigurationFactory` (loads config), `ConfigurationSaver`, `DirectoryHelper`, `ImageRetrieverHelper`, `ImageSaveHelper`, `TagsFactory` +- `Hooks/Hooks.cs` — Reqnroll lifecycle: spins up Playwright Chromium/Edge, builds Serilog logger, registers DI into Reqnroll's `ObjectContainer` +- `Support/` — helpers: `ConfigurationFactory`, `ConfigurationSaver`, `DirectoryHelper`, `ImageRetrieverHelper`, `ImageSaveHelper`, `TagsFactory` - `Models/` — strongly-typed bindings for `appsettings.json` sections (`ScrapeConfiguration`, `SearchConfiguration`, `ScrapeDirectories`, `UserConfiguration`, etc.) - `DTOs/` — deserialization models for `tagsTextToIgnore.json` and `tagsToIgnoreCompletely.json` **`AStar.Dev.Guard.Clauses`** — NuGet package. Single `GuardAgainst.Null()` helper. -**`AStar.Dev.Infrastructure.FilesDb`** — NuGet package. EF Core (SQL Server) infrastructure for file metadata. `EnumerableExtensions` provides filtering/sorting/duplicate-detection over `FileDetail` collections. Note: marked with `[Refactor]` — may duplicate logic in the files API project. +**`AStar.Dev.Infrastructure.FilesDb`** — NuGet package. EF Core (SQL Server) infrastructure for file metadata. `EnumerableExtensions` provides filtering/sorting/duplicate-detection over `FileDetail` collections. Marked `[Refactor]` — may duplicate logic in files API project. -**`AStar.Dev.Utilities`** — NuGet package. Extension methods and helpers shared across AStar packages (`StringExtensions`, `LinqExtensions`, `EncryptionExtensions`, `EnumExtensions`, `PathOperationExtensions`, `RegexExtensions`, `ObjectExtensions`, `ApplicationPathsProvider`). +**`AStar.Dev.Utilities`** — NuGet package. Shared extension methods/helpers (`StringExtensions`, `LinqExtensions`, `EncryptionExtensions`, `EnumExtensions`, `PathOperationExtensions`, `RegexExtensions`, `ObjectExtensions`, `ApplicationPathsProvider`). ### Configuration -`ConfigurationFactory` merges two sources (in order, later overrides earlier): -1. `appsettings.json` in the project root — non-secret defaults +`ConfigurationFactory` merges two sources (later overrides earlier): +1. `appsettings.json` in project root — non-secret defaults 2. **User Secrets** (`UserSecretsId: c35e09dc-dc30-416a-95a6-ec1a5ba1b43f`) — SQL connection string, login credentials Sensitive fields (`connectionStrings.sqlServer`, `userConfiguration.password`) must live in User Secrets, not `appsettings.json`. @@ -54,12 +58,12 @@ Sensitive fields (`connectionStrings.sqlServer`, `userConfiguration.password`) m ### Logging -Serilog, configured via `appsettings.json`. Writes to Console and Seq (`http://localhost:5341`). The `Hooks` constructor also hard-codes a Seq sink at the same address. +Serilog via `appsettings.json`. Writes to Console and Seq (`http://localhost:5341`). `Hooks` constructor also hard-codes Seq sink at same address. ### Tag filtering -Two JSON files control which wallpapers are skipped: -- `tagsToIgnoreCompletely.json` — skip any image whose tags include these values -- `tagsTextToIgnore.json` — skip based on partial text match in tag names +Two JSON files control skipped wallpapers: +- `tagsToIgnoreCompletely.json` — skip images whose tags include these values +- `tagsTextToIgnore.json` — skip on partial text match in tag names -Both are loaded by `TagsFactory` and registered into Reqnroll's DI container in `Hooks`. +Both loaded by `TagsFactory`, registered into Reqnroll DI in `Hooks`. \ No newline at end of file diff --git a/CLAUDE.original.md b/CLAUDE.original.md new file mode 100644 index 0000000..b461c2f --- /dev/null +++ b/CLAUDE.original.md @@ -0,0 +1,65 @@ + + +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Commands + +```bash +# Build entire solution +dotnet build AStar.Dev.Web.Scrapper.slnx + +# Run all Reqnroll scenarios +dotnet test AStar.Dev.Web.Scrapper.slnx + +# Run a specific scenario by name +dotnet test --filter "FullyQualifiedName~DownloadImagesFromSearchResults" + +# Run a specific project's tests +dotnet test AStar.Dev.Wallpaper.Scrapper/AStar.Dev.Wallpaper.Scrapper.csproj +``` + +## Architecture + +Solution targets **net10.0** across all projects. `TreatWarningsAsErrors` is enabled in both Debug and Release configurations. + +### Projects + +**`AStar.Dev.Wallpaper.Scrapper`** — the main executable. Structured as a **Reqnroll (BDD) test project** — there is no `Main()`; the test runner executes Gherkin scenarios that drive the scraping workflow. + +- `Features/` — Gherkin `.feature` files (three scraping workflows: search results, subscriptions, top wallpapers) +- `StepDefinitions/` — Reqnroll step bindings wired to Playwright page objects +- `Pages/` — Page Object Model classes wrapping Playwright `IPage` (login, search results, image page, subscriptions, top wallpapers) +- `Hooks/Hooks.cs` — Reqnroll lifecycle: spins up Playwright Chromium/Edge browser, builds Serilog logger, registers all DI instances into Reqnroll's `ObjectContainer` +- `Support/` — helpers: `ConfigurationFactory` (loads config), `ConfigurationSaver`, `DirectoryHelper`, `ImageRetrieverHelper`, `ImageSaveHelper`, `TagsFactory` +- `Models/` — strongly-typed bindings for `appsettings.json` sections (`ScrapeConfiguration`, `SearchConfiguration`, `ScrapeDirectories`, `UserConfiguration`, etc.) +- `DTOs/` — deserialization models for `tagsTextToIgnore.json` and `tagsToIgnoreCompletely.json` + +**`AStar.Dev.Guard.Clauses`** — NuGet package. Single `GuardAgainst.Null()` helper. + +**`AStar.Dev.Infrastructure.FilesDb`** — NuGet package. EF Core (SQL Server) infrastructure for file metadata. `EnumerableExtensions` provides filtering/sorting/duplicate-detection over `FileDetail` collections. Note: marked with `[Refactor]` — may duplicate logic in the files API project. + +**`AStar.Dev.Utilities`** — NuGet package. Extension methods and helpers shared across AStar packages (`StringExtensions`, `LinqExtensions`, `EncryptionExtensions`, `EnumExtensions`, `PathOperationExtensions`, `RegexExtensions`, `ObjectExtensions`, `ApplicationPathsProvider`). + +### Configuration + +`ConfigurationFactory` merges two sources (in order, later overrides earlier): +1. `appsettings.json` in the project root — non-secret defaults +2. **User Secrets** (`UserSecretsId: c35e09dc-dc30-416a-95a6-ec1a5ba1b43f`) — SQL connection string, login credentials + +Sensitive fields (`connectionStrings.sqlServer`, `userConfiguration.password`) must live in User Secrets, not `appsettings.json`. + +`ConfigurationFactory` also normalises `SearchConfiguration.SearchString` and `Subscriptions` URLs (strips base URL prefix and trailing page number) before returning. + +### Logging + +Serilog, configured via `appsettings.json`. Writes to Console and Seq (`http://localhost:5341`). The `Hooks` constructor also hard-codes a Seq sink at the same address. + +### Tag filtering + +Two JSON files control which wallpapers are skipped: +- `tagsToIgnoreCompletely.json` — skip any image whose tags include these values +- `tagsTextToIgnore.json` — skip based on partial text match in tag names + +Both are loaded by `TagsFactory` and registered into Reqnroll's DI container in `Hooks`. From 687959b5f9787e866ee6c2dc0395ba982cd85bb6 Mon Sep 17 00:00:00 2001 From: Jason Barden Date: Sat, 27 Jun 2026 08:25:13 +0100 Subject: [PATCH 2/4] test: RED - add failing tests for ScrapeConfiguration export/import (#47) Closes #47 (tests only - no production code) - 12 new GivenAnImportExportService tests for scrape config file I/O - 11 new GivenAScrapeConfigurationService tests including 3 REDACTED skip tests Co-Authored-By: Claude Sonnet 4.6 --- .../GivenAScrapeConfigurationService.cs | 243 ++++++++++++++++++ .../Services/GivenAnImportExportService.cs | 181 ++++++++++++- 2 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAScrapeConfigurationService.cs diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAScrapeConfigurationService.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAScrapeConfigurationService.cs new file mode 100644 index 0000000..37369be --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAScrapeConfigurationService.cs @@ -0,0 +1,243 @@ +using AStar.Dev.Infrastructure.FilesDb.Data; +using AStar.Dev.Infrastructure.FilesDb.Models; +using AStar.Dev.Wallpaper.Scrapper; +using AStar.Dev.Wallpaper.Scrapper.Services; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Services; + +public sealed class GivenAScrapeConfigurationService : IAsyncLifetime +{ + private const string ExistingPassword = "real-database-password"; + private const string ExistingSessionCookie = "real-session-cookie"; + private const string ExistingApiKey = "real-api-key"; + private const string ExistingSqlite = "Data Source=production.db"; + private const string ExistingCategoryId = "existing-cat"; + + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + private IDbContextFactory factory = null!; + private ScrapeConfigurationService sut = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder() + .UseSqlite(connection) + .Options; + + await using var seedContext = new FilesContext(options); + await seedContext.Database.EnsureCreatedAsync(); + seedContext.ScrapeConfiguration.Add(CreateInitialScrapeConfigurationEntity()); + await seedContext.SaveChangesAsync(); + + factory = Substitute.For>(); + factory.CreateDbContextAsync(Arg.Any()) + .Returns(_ => Task.FromResult(new FilesContext(options))); + + sut = new ScrapeConfigurationService(factory); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + [Fact] + public async Task when_exporting_then_context_is_created() + { + await sut.ExportScrapeConfigurationAsync(CancellationToken.None); + + await factory.Received(1).CreateDbContextAsync(Arg.Any()); + } + + [Fact] + public async Task when_importing_then_context_is_created() + { + await sut.ImportScrapeConfigurationAsync(CreateImportEntity(), CancellationToken.None); + + await factory.Received(1).CreateDbContextAsync(Arg.Any()); + } + + [Fact] + public async Task when_importing_then_existing_entity_is_updated() + { + var importEntity = CreateImportEntity(sqlite: "Data Source=updated.db"); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.ConnectionStrings) + .FirstAsync(); + result.ConnectionStrings.Sqlite.ShouldBe("Data Source=updated.db"); + } + + [Fact] + public async Task when_importing_password_is_redacted_then_existing_password_is_preserved() + { + var importEntity = CreateImportEntity(password: ApplicationMetadata.Redacted); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.UserConfiguration) + .FirstAsync(); + result.UserConfiguration.Password.ShouldBe(ExistingPassword); + } + + [Fact] + public async Task when_importing_session_cookie_is_redacted_then_existing_session_cookie_is_preserved() + { + var importEntity = CreateImportEntity(sessionCookie: ApplicationMetadata.Redacted); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.UserConfiguration) + .FirstAsync(); + result.UserConfiguration.SessionCookie.ShouldBe(ExistingSessionCookie); + } + + [Fact] + public async Task when_importing_api_key_is_redacted_then_existing_api_key_is_preserved() + { + var importEntity = CreateImportEntity(apiKey: ApplicationMetadata.Redacted); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.SearchConfiguration) + .FirstAsync(); + result.SearchConfiguration.ApiKey.ShouldBe(ExistingApiKey); + } + + [Fact] + public async Task when_importing_then_search_categories_are_upserted_by_id() + { + var importEntity = CreateImportEntity(categories: + [ + new SearchCategories { Id = ExistingCategoryId, Name = "Updated Name", TotalPages = 10, IncludeInSearch = true }, + new SearchCategories { Id = "new-cat", Name = "Brand New Category", TotalPages = 5, IncludeInSearch = true } + ]); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.SearchConfiguration) + .ThenInclude(x => x.SearchCategories) + .FirstAsync(); + result.SearchConfiguration.SearchCategories.Count.ShouldBe(2); + } + + [Fact] + public async Task when_importing_new_search_category_then_it_is_added() + { + var importEntity = CreateImportEntity(categories: + [ + new SearchCategories { Id = "new-cat", Name = "New Category", TotalPages = 5, IncludeInSearch = true } + ]); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.SearchConfiguration) + .ThenInclude(x => x.SearchCategories) + .FirstAsync(); + result.SearchConfiguration.SearchCategories.ShouldContain(c => c.Id == "new-cat"); + } + + [Fact] + public async Task when_importing_existing_search_category_then_it_is_updated_not_duplicated() + { + var importEntity = CreateImportEntity(categories: + [ + new SearchCategories { Id = ExistingCategoryId, Name = "Updated Category Name", TotalPages = 20, IncludeInSearch = true } + ]); + + await sut.ImportScrapeConfigurationAsync(importEntity, CancellationToken.None); + + await using var verifyCtx = new FilesContext(options); + var result = await verifyCtx.ScrapeConfiguration + .Include(x => x.SearchConfiguration) + .ThenInclude(x => x.SearchCategories) + .FirstAsync(); + result.SearchConfiguration.SearchCategories.Count.ShouldBe(1); + result.SearchConfiguration.SearchCategories[0].Name.ShouldBe("Updated Category Name"); + } + + [Fact] + public async Task when_exporting_returns_entity_with_connection_strings() + { + var result = await sut.ExportScrapeConfigurationAsync(CancellationToken.None); + + result.ConnectionStrings.ShouldNotBeNull(); + result.ConnectionStrings.Sqlite.ShouldBe(ExistingSqlite); + } + + [Fact] + public async Task when_exporting_returns_entity_with_user_configuration() + { + var result = await sut.ExportScrapeConfigurationAsync(CancellationToken.None); + + result.UserConfiguration.ShouldNotBeNull(); + result.UserConfiguration.Password.ShouldBe(ExistingPassword); + } + + private static ScrapeConfigurationEntity CreateInitialScrapeConfigurationEntity() => new() + { + ConnectionStrings = new ConnectionStrings { Sqlite = ExistingSqlite }, + UserConfiguration = new UserConfiguration + { + LoginEmailAddress = "user@example.com", + Username = "testuser", + Password = ExistingPassword, + SessionCookie = ExistingSessionCookie + }, + SearchConfiguration = new SearchConfiguration + { + BaseUrl = "https://example.com", + ApiKey = ExistingApiKey, + SearchCategories = + [ + new SearchCategories + { + Id = ExistingCategoryId, + Name = "Existing Category", + TotalPages = 5, + IncludeInSearch = true + } + ] + }, + ScrapeDirectories = new ScrapeDirectories { RootDirectory = "/tmp/scrape" } + }; + + private static ScrapeConfigurationEntity CreateImportEntity( + string sqlite = "Data Source=updated.db", + string password = "new-password", + string sessionCookie = "new-session-cookie", + string apiKey = "new-api-key", + List? categories = null) => new() + { + ConnectionStrings = new ConnectionStrings { Sqlite = sqlite }, + UserConfiguration = new UserConfiguration + { + LoginEmailAddress = "updated@example.com", + Username = "updateduser", + Password = password, + SessionCookie = sessionCookie + }, + SearchConfiguration = new SearchConfiguration + { + BaseUrl = "https://updated.com", + ApiKey = apiKey, + SearchCategories = categories ?? [] + }, + ScrapeDirectories = new ScrapeDirectories { RootDirectory = "/tmp/updated" } + }; +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs index 11a7793..8469887 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs @@ -1,18 +1,22 @@ using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.FilesDb.Models; using AStar.Dev.Wallpaper.Scrapper; using AStar.Dev.Wallpaper.Scrapper.Services; using Serilog; using System.IO.Abstractions; +using System.Text.Json; using FileClassificationDomain = AStar.Dev.Infrastructure.FilesDb.Models.FileClassification; namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Services; public sealed class GivenAnImportExportService { - private static readonly string ScrapperDirectory = Path.GetDirectoryName(ApplicationMetadata.FileClassificationsExportFilePath)!; + private static readonly string ScrapperDirectory = Path.GetDirectoryName(ApplicationMetadata.FileClassificationsExportFilePath)!; + private static readonly string ScrapeConfigScrapperDirectory = Path.GetDirectoryName(ApplicationMetadata.ScrapeConfigurationExportFilePath)!; private const string CelebrityClassificationName = "Test Celebrity"; private const string NormalClassificationName = "Test Normal"; + private const string ValidPassword = "super-secret-password"; private const string ValidClassificationsJson = """ [ @@ -53,6 +57,34 @@ public sealed class GivenAnImportExportService ] """; + private const string ValidScrapeConfigJson = """ + { + "connectionStrings": { "sqlite": "Data Source=test.db" }, + "userConfiguration": { "loginEmailAddress": "test@example.com", "username": "testuser", "password": "REDACTED", "sessionCookie": "REDACTED" }, + "searchConfiguration": { + "baseUrl": "https://example.com", + "apiKey": "REDACTED", + "searchCategories": [{ "id": "cat1", "name": "Category 1", "lastKnownImageCount": 0, "lastPageVisited": 0, "totalPages": 10, "includeInSearch": true }], + "searchString": "test", + "topWallpapers": "", + "searchStringPrefix": "", + "searchStringSuffix": "", + "subscriptions": "", + "imagePauseInSeconds": 1, + "startingPageNumber": 1, + "totalPages": 10, + "subscriptionsStartingPageNumber": 0, + "subscriptionsTotalPages": 0, + "topWallpapersTotalPages": 0, + "topWallpapersStartingPageNumber": 0, + "loginUrl": "", + "useHeadless": true, + "slowMotionDelay": null + }, + "scrapeDirectories": { "rootDirectory": "/tmp/scrape", "baseSaveDirectory": "saves", "baseDirectory": "base", "baseDirectoryFamous": "famous", "subDirectoryName": "sub" } + } + """; + private readonly MockFileSystem mockFileSystem; private readonly ILogger mockLogger; private readonly IImportExportService sut; @@ -183,15 +215,162 @@ public void when_file_system_throws_during_export_then_logger_receives_error_cal mockLogger.Received(1).Error(Arg.Any(), Arg.Any(), Arg.Any()); } + [Fact] + public void when_importing_scrape_config_and_file_does_not_exist_then_failure_result_is_returned() => + sut.ImportScrapeConfigurationFromFile() + .ShouldBeOfType>(); + + [Fact] + public void when_importing_scrape_config_and_file_does_not_exist_then_logger_receives_error_call() + { + sut.ImportScrapeConfigurationFromFile(); + + mockLogger.Received(1).Error(Arg.Any(), Arg.Any()); + } + + [Fact] + public void when_importing_scrape_config_and_file_contains_null_json_then_failure_result_is_returned() + { + mockFileSystem.Directory.CreateDirectory(ScrapeConfigScrapperDirectory); + mockFileSystem.File.WriteAllText(ApplicationMetadata.ScrapeConfigurationExportFilePath, "null"); + + sut.ImportScrapeConfigurationFromFile() + .ShouldBeOfType>(); + } + + [Fact] + public void when_importing_scrape_config_and_file_contains_null_json_then_logger_receives_error_call() + { + mockFileSystem.Directory.CreateDirectory(ScrapeConfigScrapperDirectory); + mockFileSystem.File.WriteAllText(ApplicationMetadata.ScrapeConfigurationExportFilePath, "null"); + + sut.ImportScrapeConfigurationFromFile(); + + mockLogger.Received(1).Error(Arg.Any(), Arg.Any()); + } + + [Fact] + public void when_importing_valid_scrape_config_then_result_is_ok() + { + SetupValidScrapeConfigImportFile(); + + sut.ImportScrapeConfigurationFromFile() + .ShouldBeOfType>(); + } + + [Fact] + public void when_importing_valid_scrape_config_then_correct_connection_string_is_mapped() + { + SetupValidScrapeConfigImportFile(); + + sut.ImportScrapeConfigurationFromFile() + .ShouldBeOfType>() + .Value.ConnectionStrings.Sqlite.ShouldBe("Data Source=test.db"); + } + + [Fact] + public void when_importing_valid_scrape_config_then_password_field_is_preserved_from_db() + { + SetupValidScrapeConfigImportFile(); + + sut.ImportScrapeConfigurationFromFile() + .ShouldBeOfType>() + .Value.UserConfiguration.Password.ShouldBe(ApplicationMetadata.Redacted); + } + + [Fact] + public void when_exporting_scrape_config_then_file_is_written_to_expected_path() + { + mockFileSystem.Directory.CreateDirectory(ScrapeConfigScrapperDirectory); + + sut.ExportScrapeConfigurationToFile(CreateScrapeConfigurationEntityWithSensitiveData()); + + mockFileSystem.File.Exists(ApplicationMetadata.ScrapeConfigurationExportFilePath).ShouldBeTrue(); + } + + [Fact] + public void when_exporting_scrape_config_then_logger_receives_information_call() + { + mockFileSystem.Directory.CreateDirectory(ScrapeConfigScrapperDirectory); + + sut.ExportScrapeConfigurationToFile(CreateScrapeConfigurationEntityWithSensitiveData()); + + mockLogger.Received(1).Information(Arg.Any(), Arg.Any()); + } + + [Fact] + public void when_exporting_scrape_config_then_password_is_redacted_in_exported_file() + { + mockFileSystem.Directory.CreateDirectory(ScrapeConfigScrapperDirectory); + + sut.ExportScrapeConfigurationToFile(CreateScrapeConfigurationEntityWithSensitiveData()); + + var json = mockFileSystem.File.ReadAllText(ApplicationMetadata.ScrapeConfigurationExportFilePath); + using var doc = JsonDocument.Parse(json); + doc.RootElement.GetProperty("userConfiguration").GetProperty("password").GetString() + .ShouldBe(ApplicationMetadata.Redacted); + } + + [Fact] + public void when_file_system_throws_during_scrape_config_export_then_exception_is_rethrown() + { + var throwingFileSystem = Substitute.For(); + throwingFileSystem.File.When(f => f.WriteAllText(Arg.Any(), Arg.Any())) + .Throw(new IOException("Disk full")); + var throwingSut = new ImportExportService(throwingFileSystem, mockLogger); + + var act = () => throwingSut.ExportScrapeConfigurationToFile(CreateScrapeConfigurationEntityWithSensitiveData()); + + act.ShouldThrow(); + } + + [Fact] + public void when_file_system_throws_during_scrape_config_export_then_logger_receives_error_call() + { + var throwingFileSystem = Substitute.For(); + throwingFileSystem.File.When(f => f.WriteAllText(Arg.Any(), Arg.Any())) + .Throw(new IOException("Disk full")); + var throwingSut = new ImportExportService(throwingFileSystem, mockLogger); + + Should.Throw(() => throwingSut.ExportScrapeConfigurationToFile(CreateScrapeConfigurationEntityWithSensitiveData())); + + mockLogger.Received(1).Error(Arg.Any(), Arg.Any(), Arg.Any()); + } + private void SetupValidImportFile() { mockFileSystem.Directory.CreateDirectory(ScrapperDirectory); mockFileSystem.File.WriteAllText(ApplicationMetadata.FileClassificationsExportFilePath, ValidClassificationsJson); } + private void SetupValidScrapeConfigImportFile() + { + mockFileSystem.Directory.CreateDirectory(ScrapeConfigScrapperDirectory); + mockFileSystem.File.WriteAllText(ApplicationMetadata.ScrapeConfigurationExportFilePath, ValidScrapeConfigJson); + } + private static List CreateDomainClassifications() => [ new() { Id = 1, Name = CelebrityClassificationName, Celebrity = true, IncludeInSearch = true }, new() { Id = 2, Name = NormalClassificationName, Celebrity = false, IncludeInSearch = true } ]; + + private static ScrapeConfigurationEntity CreateScrapeConfigurationEntityWithSensitiveData() => new() + { + ConnectionStrings = new ConnectionStrings { Sqlite = "Data Source=production.db" }, + UserConfiguration = new UserConfiguration + { + LoginEmailAddress = "user@example.com", + Username = "testuser", + Password = ValidPassword, + SessionCookie = "actual-session-cookie" + }, + SearchConfiguration = new SearchConfiguration + { + BaseUrl = "https://example.com", + ApiKey = "actual-api-key", + SearchCategories = [] + }, + ScrapeDirectories = new ScrapeDirectories { RootDirectory = "/tmp/scrape" } + }; } From 5b3666193ca60ad87c1b3dfe862f3d4c0e9ac469 Mon Sep 17 00:00:00 2001 From: Jason Barden Date: Sat, 27 Jun 2026 09:26:22 +0100 Subject: [PATCH 3/4] feat(config): export/import ScrapeConfiguration entity graph (#47) Closes #47 - Add ApplicationMetadata.Redacted constant and ScrapeConfigurationExportFilePath - Add 6 DTO records (ScrapeConfigurationDto and children) with init properties - Add ScrapeConfigurationDtoExtensions.ToDto (redacts Password/SessionCookie/ApiKey) and ToDomain - Add ScrapeConfigurationService with ExportScrapeConfigurationAsync and ImportScrapeConfigurationAsync (REDACTED-skip + SearchCategory upsert-by-Id) - Extend IImportExportService / ImportExportService with ExportScrapeConfigurationToFile and ImportScrapeConfigurationFromFile - Add Export Config / Import Config buttons to MainWindow (disabled during operation) - Guard FilesContext.OnConfiguring with if (!optionsBuilder.IsConfigured) for testability - Register ScrapeConfigurationService in DI Co-Authored-By: Claude Sonnet 4.6 --- .../Data/FilesContext.cs | 8 +- src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs | 1 + .../ApplicationMetadata.cs | 5 +- .../DTOs/ConnectionStringsDto.cs | 6 + .../DTOs/ScrapeConfigurationDto.cs | 9 ++ .../DTOs/ScrapeConfigurationDtoExtensions.cs | 109 ++++++++++++++++++ .../DTOs/ScrapeDirectoriesDto.cs | 10 ++ .../DTOs/SearchCategoryDto.cs | 11 ++ .../DTOs/SearchConfigurationDto.cs | 23 ++++ .../DTOs/UserConfigurationDto.cs | 9 ++ .../MainWindow.axaml | 2 + .../MainWindow.axaml.cs | 47 +++++++- .../Services/IImportExportService.cs | 3 + .../Services/ImportExportService.cs | 48 +++++++- .../Services/ScrapeConfigurationService.cs | 107 +++++++++++++++++ 15 files changed, 386 insertions(+), 12 deletions(-) create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/ConnectionStringsDto.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDto.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDtoExtensions.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeDirectoriesDto.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchCategoryDto.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchConfigurationDto.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/DTOs/UserConfigurationDto.cs create mode 100644 src/AStar.Dev.Wallpaper.Scrapper/Services/ScrapeConfigurationService.cs diff --git a/src/AStar.Dev.Infrastructure.FilesDb/Data/FilesContext.cs b/src/AStar.Dev.Infrastructure.FilesDb/Data/FilesContext.cs index 5f2460b..a9f1eb1 100644 --- a/src/AStar.Dev.Infrastructure.FilesDb/Data/FilesContext.cs +++ b/src/AStar.Dev.Infrastructure.FilesDb/Data/FilesContext.cs @@ -1,4 +1,4 @@ -using AStar.Dev.Infrastructure.FilesDb.Models; +using AStar.Dev.Infrastructure.FilesDb.Models; using Microsoft.EntityFrameworkCore; namespace AStar.Dev.Infrastructure.FilesDb.Data; @@ -76,7 +76,11 @@ public FilesContext() public DbSet DownloadedFileClassifications { get; set; } = null!; /// - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => _ = optionsBuilder.UseSqlite("Data Source=/home/jbarden/Documents/Scrapper/files.db"); + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if(!optionsBuilder.IsConfigured) + _ = optionsBuilder.UseSqlite("Data Source=/home/jbarden/Documents/Scrapper/files.db"); + } /// /// The overridden OnModelCreating method diff --git a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs index e0cd66e..5fd8ed6 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs @@ -86,6 +86,7 @@ public override void OnFrameworkInitializationCompleted() .AddTransient() .AddTransient() .AddTransient() + .AddTransient() .AddTransient>(sp => () => sp.GetRequiredService()) .AddTransient(); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/ApplicationMetadata.cs b/src/AStar.Dev.Wallpaper.Scrapper/ApplicationMetadata.cs index 0feca64..dd1c9f5 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/ApplicationMetadata.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/ApplicationMetadata.cs @@ -8,8 +8,11 @@ public static class ApplicationMetadata { public const string Name = "AStar.Dev.Wallpaper.Scrapper"; public const string Version = "1.0.0"; + public const string Redacted = "REDACTED"; public static string ApplicationFolder => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!.CombinePath("..", "..", ".."); - public static string FileClassificationsExportFilePath => Path.Combine(SpecialDirectories.MyDocuments, "Scrapper", "FileClassifications.json"); + public static string FileClassificationsExportFilePath => SpecialDirectories.MyDocuments.CombinePath("Scrapper", "FileClassifications.json"); + + public static string ScrapeConfigurationExportFilePath => SpecialDirectories.MyDocuments.CombinePath("Scrapper", "ScrapeConfiguration.json"); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ConnectionStringsDto.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ConnectionStringsDto.cs new file mode 100644 index 0000000..83c41e3 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ConnectionStringsDto.cs @@ -0,0 +1,6 @@ +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public sealed record ConnectionStringsDto +{ + public string Sqlite { get; init; } = string.Empty; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDto.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDto.cs new file mode 100644 index 0000000..6c3e8a3 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDto.cs @@ -0,0 +1,9 @@ +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public sealed record ScrapeConfigurationDto +{ + public ConnectionStringsDto ConnectionStrings { get; init; } = new(); + public UserConfigurationDto UserConfiguration { get; init; } = new(); + public SearchConfigurationDto SearchConfiguration { get; init; } = new(); + public ScrapeDirectoriesDto ScrapeDirectories { get; init; } = new(); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDtoExtensions.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDtoExtensions.cs new file mode 100644 index 0000000..a6fe900 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeConfigurationDtoExtensions.cs @@ -0,0 +1,109 @@ +using ScrapeConfigurationEntityDomain = AStar.Dev.Infrastructure.FilesDb.Models.ScrapeConfigurationEntity; +using SearchCategoriesDomain = AStar.Dev.Infrastructure.FilesDb.Models.SearchCategories; +using ConnectionStringsDomain = AStar.Dev.Infrastructure.FilesDb.Models.ConnectionStrings; +using UserConfigurationDomain = AStar.Dev.Infrastructure.FilesDb.Models.UserConfiguration; +using SearchConfigurationDomain = AStar.Dev.Infrastructure.FilesDb.Models.SearchConfiguration; +using ScrapeDirectoriesDomain = AStar.Dev.Infrastructure.FilesDb.Models.ScrapeDirectories; + +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public static class ScrapeConfigurationDtoExtensions +{ + public static ScrapeConfigurationDto ToDto(this ScrapeConfigurationEntityDomain entity) => new() + { + ConnectionStrings = new ConnectionStringsDto { Sqlite = entity.ConnectionStrings.Sqlite }, + UserConfiguration = new UserConfigurationDto + { + LoginEmailAddress = entity.UserConfiguration.LoginEmailAddress, + Username = entity.UserConfiguration.Username, + Password = ApplicationMetadata.Redacted, + SessionCookie = ApplicationMetadata.Redacted + }, + SearchConfiguration = new SearchConfigurationDto + { + BaseUrl = entity.SearchConfiguration.BaseUrl, + ApiKey = ApplicationMetadata.Redacted, + SearchCategories = [.. entity.SearchConfiguration.SearchCategories.Select(c => new SearchCategoryDto + { + Id = c.Id, + Name = c.Name, + LastKnownImageCount = c.LastKnownImageCount, + LastPageVisited = c.LastPageVisited, + TotalPages = c.TotalPages, + IncludeInSearch = c.IncludeInSearch + })], + SearchString = entity.SearchConfiguration.SearchString, + TopWallpapers = entity.SearchConfiguration.TopWallpapers, + SearchStringPrefix = entity.SearchConfiguration.SearchStringPrefix, + SearchStringSuffix = entity.SearchConfiguration.SearchStringSuffix, + Subscriptions = entity.SearchConfiguration.Subscriptions, + ImagePauseInSeconds = entity.SearchConfiguration.ImagePauseInSeconds, + StartingPageNumber = entity.SearchConfiguration.StartingPageNumber, + TotalPages = entity.SearchConfiguration.TotalPages, + SubscriptionsStartingPageNumber = entity.SearchConfiguration.SubscriptionsStartingPageNumber, + SubscriptionsTotalPages = entity.SearchConfiguration.SubscriptionsTotalPages, + TopWallpapersTotalPages = entity.SearchConfiguration.TopWallpapersTotalPages, + TopWallpapersStartingPageNumber = entity.SearchConfiguration.TopWallpapersStartingPageNumber, + LoginUrl = entity.SearchConfiguration.LoginUrl, + UseHeadless = entity.SearchConfiguration.UseHeadless, + SlowMotionDelay = entity.SearchConfiguration.SlowMotionDelay + }, + ScrapeDirectories = new ScrapeDirectoriesDto + { + RootDirectory = entity.ScrapeDirectories.RootDirectory, + BaseSaveDirectory = entity.ScrapeDirectories.BaseSaveDirectory, + BaseDirectory = entity.ScrapeDirectories.BaseDirectory, + BaseDirectoryFamous = entity.ScrapeDirectories.BaseDirectoryFamous, + SubDirectoryName = entity.ScrapeDirectories.SubDirectoryName + } + }; + + public static ScrapeConfigurationEntityDomain ToDomain(this ScrapeConfigurationDto dto) => new() + { + ConnectionStrings = new ConnectionStringsDomain { Sqlite = dto.ConnectionStrings.Sqlite }, + UserConfiguration = new UserConfigurationDomain + { + LoginEmailAddress = dto.UserConfiguration.LoginEmailAddress, + Username = dto.UserConfiguration.Username, + Password = dto.UserConfiguration.Password, + SessionCookie = dto.UserConfiguration.SessionCookie + }, + SearchConfiguration = new SearchConfigurationDomain + { + BaseUrl = dto.SearchConfiguration.BaseUrl, + ApiKey = dto.SearchConfiguration.ApiKey, + SearchCategories = [.. dto.SearchConfiguration.SearchCategories.Select(c => new SearchCategoriesDomain + { + Id = c.Id, + Name = c.Name, + LastKnownImageCount = c.LastKnownImageCount, + LastPageVisited = c.LastPageVisited, + TotalPages = c.TotalPages, + IncludeInSearch = c.IncludeInSearch + })], + SearchString = dto.SearchConfiguration.SearchString, + TopWallpapers = dto.SearchConfiguration.TopWallpapers, + SearchStringPrefix = dto.SearchConfiguration.SearchStringPrefix, + SearchStringSuffix = dto.SearchConfiguration.SearchStringSuffix, + Subscriptions = dto.SearchConfiguration.Subscriptions, + ImagePauseInSeconds = dto.SearchConfiguration.ImagePauseInSeconds, + StartingPageNumber = dto.SearchConfiguration.StartingPageNumber, + TotalPages = dto.SearchConfiguration.TotalPages, + SubscriptionsStartingPageNumber = dto.SearchConfiguration.SubscriptionsStartingPageNumber, + SubscriptionsTotalPages = dto.SearchConfiguration.SubscriptionsTotalPages, + TopWallpapersTotalPages = dto.SearchConfiguration.TopWallpapersTotalPages, + TopWallpapersStartingPageNumber = dto.SearchConfiguration.TopWallpapersStartingPageNumber, + LoginUrl = dto.SearchConfiguration.LoginUrl, + UseHeadless = dto.SearchConfiguration.UseHeadless, + SlowMotionDelay = dto.SearchConfiguration.SlowMotionDelay + }, + ScrapeDirectories = new ScrapeDirectoriesDomain + { + RootDirectory = dto.ScrapeDirectories.RootDirectory, + BaseSaveDirectory = dto.ScrapeDirectories.BaseSaveDirectory, + BaseDirectory = dto.ScrapeDirectories.BaseDirectory, + BaseDirectoryFamous = dto.ScrapeDirectories.BaseDirectoryFamous, + SubDirectoryName = dto.ScrapeDirectories.SubDirectoryName + } + }; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeDirectoriesDto.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeDirectoriesDto.cs new file mode 100644 index 0000000..bcadf9a --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/ScrapeDirectoriesDto.cs @@ -0,0 +1,10 @@ +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public sealed record ScrapeDirectoriesDto +{ + public string RootDirectory { get; init; } = string.Empty; + public string BaseSaveDirectory { get; init; } = string.Empty; + public string BaseDirectory { get; init; } = string.Empty; + public string BaseDirectoryFamous { get; init; } = string.Empty; + public string SubDirectoryName { get; init; } = string.Empty; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchCategoryDto.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchCategoryDto.cs new file mode 100644 index 0000000..adbd5c0 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchCategoryDto.cs @@ -0,0 +1,11 @@ +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public sealed record SearchCategoryDto +{ + public string Id { get; init; } = string.Empty; + public string Name { get; init; } = string.Empty; + public int LastKnownImageCount { get; init; } + public int LastPageVisited { get; init; } + public int TotalPages { get; init; } + public bool IncludeInSearch { get; init; } = true; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchConfigurationDto.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchConfigurationDto.cs new file mode 100644 index 0000000..221d797 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/SearchConfigurationDto.cs @@ -0,0 +1,23 @@ +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public sealed record SearchConfigurationDto +{ + public string BaseUrl { get; init; } = string.Empty; + public string ApiKey { get; init; } = string.Empty; + public List SearchCategories { get; init; } = []; + public string SearchString { get; init; } = string.Empty; + public string TopWallpapers { get; init; } = string.Empty; + public string SearchStringPrefix { get; init; } = string.Empty; + public string SearchStringSuffix { get; init; } = string.Empty; + public string Subscriptions { get; init; } = string.Empty; + public int ImagePauseInSeconds { get; init; } + public int StartingPageNumber { get; init; } + public int TotalPages { get; init; } + public int SubscriptionsStartingPageNumber { get; init; } + public int SubscriptionsTotalPages { get; init; } + public int TopWallpapersTotalPages { get; init; } + public int TopWallpapersStartingPageNumber { get; init; } + public string LoginUrl { get; init; } = string.Empty; + public bool UseHeadless { get; init; } + public float? SlowMotionDelay { get; init; } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/UserConfigurationDto.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/UserConfigurationDto.cs new file mode 100644 index 0000000..635c72f --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/UserConfigurationDto.cs @@ -0,0 +1,9 @@ +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +public sealed record UserConfigurationDto +{ + public string LoginEmailAddress { get; init; } = string.Empty; + public string Username { get; init; } = string.Empty; + public string Password { get; init; } = string.Empty; + public string SessionCookie { get; init; } = string.Empty; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml b/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml index e5a0451..61d54a1 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml +++ b/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml @@ -10,6 +10,8 @@