From 6a3658206958bc92792610876be3d8e99a756d5b Mon Sep 17 00:00:00 2001 From: Jiri Manas Date: Sun, 22 Feb 2026 09:57:15 +0100 Subject: [PATCH] Add GitHub App credentials configuration via admin UI Adds a complete admin UI for configuring GitHub App credentials (App ID, Private Key, App Name) without needing SSH access to the server. Credentials are stored in the database and validated against the GitHub API before saving. Features: - Configuration form with PEM file upload support - In-memory credential cache (GitHubAppCredentialCache singleton) loaded from DB at startup, checked before env var fallback - Credential validation via GitHub API (JWT + GET /app) - Connect/disconnect GitHub organizations - Reset configuration to start fresh - Soft-delete aware upserts to prevent UNIQUE constraint violations - Full i18n support (en, zh, ko, ja) Backend: - GitHubAppCredentialCache: thread-safe singleton for DB-backed creds - GitHubAppService: JWT generation, installation token management - AdminGitHubImportService: config CRUD, org management, batch import - API endpoints: GET/POST/DELETE /config, POST/DELETE /installations, GET /install-url, GET /installations/{id}/repos, POST /batch-import Frontend: - Admin page at /admin/github-import - Config form shown only when not configured - Connected orgs list with disconnect button - Repository browser with search, filter, pagination - Batch import with department/language selection --- .../Admin/AdminGitHubImportEndpoints.cs | 148 +++ .../Models/Admin/GitHubImportModels.cs | 100 ++ src/OpenDeepWiki/Program.cs | 13 +- .../Admin/AdminGitHubImportService.cs | 504 +++++++++ .../Admin/IAdminGitHubImportService.cs | 15 + .../Services/Admin/SystemSettingDefaults.cs | 136 ++- .../GitHub/GitHubAppCredentialCache.cs | 69 ++ .../Services/GitHub/GitHubAppService.cs | 255 +++++ web/app/admin/github-import/page.tsx | 967 ++++++++++++++++++ web/i18n/messages/en/admin.json | 79 ++ web/i18n/messages/ja/admin.json | 79 ++ web/i18n/messages/ko/admin.json | 79 ++ web/i18n/messages/zh/admin.json | 79 ++ web/lib/admin-api.ts | 147 +++ 14 files changed, 2628 insertions(+), 42 deletions(-) create mode 100644 src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs create mode 100644 src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs create mode 100644 src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs create mode 100644 src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs create mode 100644 src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs create mode 100644 src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs create mode 100644 web/app/admin/github-import/page.tsx diff --git a/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs b/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs new file mode 100644 index 00000000..23c6363d --- /dev/null +++ b/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs @@ -0,0 +1,148 @@ +using OpenDeepWiki.Models.Admin; +using OpenDeepWiki.Services.Admin; +using OpenDeepWiki.Services.Auth; +using OpenDeepWiki.Services.GitHub; + +namespace OpenDeepWiki.Endpoints.Admin; + +public static class AdminGitHubImportEndpoints +{ + public static RouteGroupBuilder MapAdminGitHubImportEndpoints(this RouteGroupBuilder group) + { + var github = group.MapGroup("/github") + .WithTags("Admin - GitHub Import"); + + github.MapGet("/status", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + var result = await service.GetStatusAsync(cancellationToken); + return Results.Ok(new { success = true, data = result }); + }); + + github.MapGet("/config", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + var result = await service.GetGitHubConfigAsync(cancellationToken); + return Results.Ok(new { success = true, data = result }); + }); + + github.MapPost("/config", async ( + SaveGitHubConfigRequest request, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.SaveGitHubConfigAsync(request, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapDelete("/config", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + await service.ResetGitHubConfigAsync(cancellationToken); + return Results.Ok(new { success = true }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapGet("/install-url", async ( + Microsoft.Extensions.Configuration.IConfiguration configuration, + GitHubAppCredentialCache cache) => + { + var appName = cache.AppName + ?? configuration["GitHub:App:Name"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Name") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_NAME") + ?? "deepwiki-keboola"; + var url = $"https://github.com/apps/{appName}/installations/new"; + return Results.Ok(new { success = true, data = new { url, appName } }); + }); + + github.MapPost("/installations", async ( + StoreInstallationRequest request, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.StoreInstallationAsync(request.InstallationId, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapDelete("/installations/{installationId}", async ( + string installationId, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + await service.DisconnectInstallationAsync(installationId, cancellationToken); + return Results.Ok(new { success = true }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapGet("/installations/{installationId:long}/repos", async ( + long installationId, + int page, + int perPage, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.ListInstallationReposAsync(installationId, page, perPage, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (Exception ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapPost("/batch-import", async ( + BatchImportRequest request, + IAdminGitHubImportService service, + IUserContext userContext, + CancellationToken cancellationToken) => + { + try + { + var userId = userContext.UserId; + if (string.IsNullOrEmpty(userId)) + return Results.Unauthorized(); + + var result = await service.BatchImportAsync(request, userId, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + return group; + } +} diff --git a/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs b/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs new file mode 100644 index 00000000..0d2fd0d3 --- /dev/null +++ b/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs @@ -0,0 +1,100 @@ +namespace OpenDeepWiki.Models.Admin; + +public class GitHubStatusResponse +{ + public bool Configured { get; set; } + public string? AppName { get; set; } + public List Installations { get; set; } = new(); +} + +public class GitHubInstallationDto +{ + public string? Id { get; set; } + public long InstallationId { get; set; } + public string AccountLogin { get; set; } = string.Empty; + public string AccountType { get; set; } = string.Empty; + public long AccountId { get; set; } + public string? AvatarUrl { get; set; } + public string? DepartmentId { get; set; } + public string? DepartmentName { get; set; } + public DateTime CreatedAt { get; set; } +} + +public class StoreInstallationRequest +{ + public long InstallationId { get; set; } +} + +public class GitHubRepoDto +{ + public long Id { get; set; } + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string DefaultBranch { get; set; } = "main"; + public string CloneUrl { get; set; } = string.Empty; + public string HtmlUrl { get; set; } = string.Empty; + public bool AlreadyImported { get; set; } +} + +public class GitHubRepoListDto +{ + public int TotalCount { get; set; } + public List Repositories { get; set; } = new(); + public int Page { get; set; } + public int PerPage { get; set; } +} + +public class BatchImportRequest +{ + public long InstallationId { get; set; } + public string DepartmentId { get; set; } = string.Empty; + public string LanguageCode { get; set; } = "en"; + public List Repos { get; set; } = new(); +} + +public class BatchImportRepo +{ + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public string CloneUrl { get; set; } = string.Empty; + public string DefaultBranch { get; set; } = "main"; + public bool Private { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } +} + +public class BatchImportResult +{ + public int TotalRequested { get; set; } + public int Imported { get; set; } + public int Skipped { get; set; } + public List SkippedRepos { get; set; } = new(); + public List ImportedRepos { get; set; } = new(); +} + +public class SaveGitHubConfigRequest +{ + public string AppId { get; set; } = string.Empty; + public string AppName { get; set; } = string.Empty; + public string PrivateKey { get; set; } = string.Empty; +} + +public class GitHubConfigResponse +{ + public bool HasAppId { get; set; } + public bool HasPrivateKey { get; set; } + public string? AppId { get; set; } + public string? AppName { get; set; } + /// + /// Credential source: "database", "environment", or "none". + /// + public string Source { get; set; } = "none"; +} diff --git a/src/OpenDeepWiki/Program.cs b/src/OpenDeepWiki/Program.cs index f22e89c1..19a8746c 100644 --- a/src/OpenDeepWiki/Program.cs +++ b/src/OpenDeepWiki/Program.cs @@ -11,6 +11,7 @@ using OpenDeepWiki.Infrastructure; using OpenDeepWiki.Services.Admin; using OpenDeepWiki.Services.Auth; +using OpenDeepWiki.Services.GitHub; using OpenDeepWiki.Services.Chat; using OpenDeepWiki.Services.MindMap; using OpenDeepWiki.Services.Notifications; @@ -302,6 +303,11 @@ // 注册处理日志服务（使用 Singleton，因为它内部使用 IServiceScopeFactory 创建独立 scope） builder.Services.AddSingleton(); + // 注册 GitHub App 服务 + builder.Services.AddSingleton(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + // 注册管理端服务 builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -397,6 +403,10 @@ var settingsService = scope.ServiceProvider.GetRequiredService(); var wikiOptions = scope.ServiceProvider.GetRequiredService>(); await SystemSettingDefaults.ApplyToWikiGeneratorOptions(wikiOptions.Value, settingsService); + + // Load GitHub App credentials from DB into the in-memory cache + var githubCache = app.Services.GetRequiredService(); + await githubCache.LoadFromDbAsync(settingsService); } // 启用 CORS @@ -457,8 +467,7 @@ static void LoadEnvFile(IConfigurationBuilder configuration) var envPaths = new[] { Path.Combine(Directory.GetCurrentDirectory(), ".env"), - Path.Combine(AppContext.BaseDirectory, ".env"), - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".env"), + Path.Combine(AppContext.BaseDirectory, ".env") }; foreach (var envPath in envPaths) diff --git a/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs b/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs new file mode 100644 index 00000000..e0f9a7fa --- /dev/null +++ b/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs @@ -0,0 +1,504 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Security.Claims; +using System.Security.Cryptography; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using OpenDeepWiki.EFCore; +using OpenDeepWiki.Entities; +using OpenDeepWiki.Models.Admin; +using OpenDeepWiki.Services.GitHub; + +namespace OpenDeepWiki.Services.Admin; + +public class AdminGitHubImportService : IAdminGitHubImportService +{ + private readonly IContext _context; + private readonly IGitHubAppService _gitHubAppService; + private readonly IAdminSettingsService _settingsService; + private readonly IHttpClientFactory _httpClientFactory; + private readonly GitHubAppCredentialCache _cache; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + + public AdminGitHubImportService( + IContext context, + IGitHubAppService gitHubAppService, + IAdminSettingsService settingsService, + IHttpClientFactory httpClientFactory, + GitHubAppCredentialCache cache, + IConfiguration configuration, + ILogger logger) + { + _context = context; + _gitHubAppService = gitHubAppService; + _settingsService = settingsService; + _httpClientFactory = httpClientFactory; + _cache = cache; + _configuration = configuration; + _logger = logger; + } + + public async Task GetStatusAsync(CancellationToken cancellationToken = default) + { + var response = new GitHubStatusResponse + { + Configured = _gitHubAppService.IsConfigured + }; + + if (!response.Configured) + return response; + + // Load stored installations from DB + var installations = await _context.GitHubAppInstallations + .Where(i => !i.IsDeleted) + .ToListAsync(cancellationToken); + + // Load department names for linked installations + var departmentIds = installations + .Where(i => i.DepartmentId != null) + .Select(i => i.DepartmentId!) + .Distinct() + .ToList(); + + var departments = departmentIds.Count > 0 + ? await _context.Departments + .Where(d => departmentIds.Contains(d.Id)) + .ToDictionaryAsync(d => d.Id, d => d.Name, cancellationToken) + : new Dictionary(); + + response.Installations = installations.Select(i => new GitHubInstallationDto + { + Id = i.Id, + InstallationId = i.InstallationId, + AccountLogin = i.AccountLogin, + AccountType = i.AccountType, + AccountId = i.AccountId, + AvatarUrl = i.AvatarUrl, + DepartmentId = i.DepartmentId, + DepartmentName = i.DepartmentId != null && departments.ContainsKey(i.DepartmentId) + ? departments[i.DepartmentId] + : null, + CreatedAt = i.CreatedAt + }).ToList(); + + return response; + } + + public async Task StoreInstallationAsync(long installationId, CancellationToken cancellationToken = default) + { + // Check if already stored (including soft-deleted, to avoid UNIQUE constraint violation) + var existing = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.InstallationId == installationId, cancellationToken); + + if (existing != null) + { + // Re-activate if soft-deleted + if (existing.IsDeleted) + { + existing.IsDeleted = false; + existing.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + _logger.LogInformation("Re-activated GitHub App installation {InstallationId} for {AccountLogin}", + installationId, existing.AccountLogin); + } + + return new GitHubInstallationDto + { + Id = existing.Id, + InstallationId = existing.InstallationId, + AccountLogin = existing.AccountLogin, + AccountType = existing.AccountType, + AccountId = existing.AccountId, + AvatarUrl = existing.AvatarUrl, + DepartmentId = existing.DepartmentId, + CreatedAt = existing.CreatedAt + }; + } + + // Fetch installation details from GitHub + var allInstallations = await _gitHubAppService.ListInstallationsAsync(cancellationToken); + var info = allInstallations.FirstOrDefault(i => i.Id == installationId); + + if (info == null) + throw new InvalidOperationException($"Installation {installationId} not found on GitHub. Ensure the app is installed."); + + var entity = new GitHubAppInstallation + { + Id = Guid.NewGuid().ToString(), + InstallationId = installationId, + AccountLogin = info.AccountLogin, + AccountType = info.AccountType, + AccountId = info.AccountId, + AvatarUrl = info.AvatarUrl, + }; + + _context.GitHubAppInstallations.Add(entity); + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("Stored GitHub App installation {InstallationId} for {AccountLogin}", + installationId, info.AccountLogin); + + return new GitHubInstallationDto + { + Id = entity.Id, + InstallationId = entity.InstallationId, + AccountLogin = entity.AccountLogin, + AccountType = entity.AccountType, + AccountId = entity.AccountId, + AvatarUrl = entity.AvatarUrl, + CreatedAt = entity.CreatedAt + }; + } + + public async Task DisconnectInstallationAsync(string installationId, CancellationToken cancellationToken = default) + { + var entity = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.Id == installationId && !i.IsDeleted, cancellationToken); + + if (entity == null) + throw new InvalidOperationException($"Installation {installationId} not found."); + + entity.IsDeleted = true; + entity.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("Disconnected GitHub App installation {InstallationId} for {AccountLogin}", + entity.InstallationId, entity.AccountLogin); + } + + public async Task ListInstallationReposAsync( + long installationId, int page, int perPage, CancellationToken cancellationToken = default) + { + var result = await _gitHubAppService.ListInstallationReposAsync(installationId, page, perPage, cancellationToken); + + // Check which repos are already imported + var repoUrls = result.Repositories.Select(r => r.CloneUrl).ToList(); + var existingUrls = await _context.Repositories + .Where(r => repoUrls.Contains(r.GitUrl) && !r.IsDeleted) + .Select(r => r.GitUrl) + .ToListAsync(cancellationToken); + + var existingUrlSet = new HashSet(existingUrls, StringComparer.OrdinalIgnoreCase); + + return new GitHubRepoListDto + { + TotalCount = result.TotalCount, + Page = page, + PerPage = perPage, + Repositories = result.Repositories.Select(r => new GitHubRepoDto + { + Id = r.Id, + FullName = r.FullName, + Name = r.Name, + Owner = r.Owner, + Private = r.Private, + Description = r.Description, + Language = r.Language, + StargazersCount = r.StargazersCount, + ForksCount = r.ForksCount, + DefaultBranch = r.DefaultBranch, + CloneUrl = r.CloneUrl, + HtmlUrl = r.HtmlUrl, + AlreadyImported = existingUrlSet.Contains(r.CloneUrl) + }).ToList() + }; + } + + public async Task BatchImportAsync( + BatchImportRequest request, string ownerUserId, CancellationToken cancellationToken = default) + { + var result = new BatchImportResult + { + TotalRequested = request.Repos.Count + }; + + // Verify department exists + var department = await _context.Departments + .FirstOrDefaultAsync(d => d.Id == request.DepartmentId && !d.IsDeleted, cancellationToken); + + if (department == null) + throw new InvalidOperationException($"Department {request.DepartmentId} not found."); + + // Get existing repos by clone URL to avoid duplicates + var requestUrls = request.Repos.Select(r => r.CloneUrl).ToList(); + var existingRepos = await _context.Repositories + .Where(r => requestUrls.Contains(r.GitUrl) && !r.IsDeleted) + .ToDictionaryAsync(r => r.GitUrl, r => r, StringComparer.OrdinalIgnoreCase, cancellationToken); + + foreach (var repo in request.Repos) + { + if (existingRepos.ContainsKey(repo.CloneUrl)) + { + result.Skipped++; + result.SkippedRepos.Add(repo.FullName); + _logger.LogDebug("Skipping {Repo}: already imported", repo.FullName); + continue; + } + + // Create Repository entity + var repoEntity = new Repository + { + Id = Guid.NewGuid().ToString(), + OwnerUserId = ownerUserId, + GitUrl = repo.CloneUrl, + RepoName = repo.Name, + OrgName = repo.Owner, + IsPublic = !repo.Private, + Status = RepositoryStatus.Pending, + PrimaryLanguage = repo.Language, + StarCount = repo.StargazersCount, + ForkCount = repo.ForksCount + }; + + _context.Repositories.Add(repoEntity); + + // Create default branch + var branch = new RepositoryBranch + { + Id = Guid.NewGuid().ToString(), + RepositoryId = repoEntity.Id, + BranchName = repo.DefaultBranch + }; + + _context.RepositoryBranches.Add(branch); + + // Create branch language + var branchLanguage = new BranchLanguage + { + Id = Guid.NewGuid().ToString(), + RepositoryBranchId = branch.Id, + LanguageCode = request.LanguageCode, + IsDefault = true + }; + + _context.BranchLanguages.Add(branchLanguage); + + // Assign to department + var assignment = new RepositoryAssignment + { + Id = Guid.NewGuid().ToString(), + RepositoryId = repoEntity.Id, + DepartmentId = request.DepartmentId, + AssigneeUserId = ownerUserId + }; + + _context.RepositoryAssignments.Add(assignment); + + result.Imported++; + result.ImportedRepos.Add(repo.FullName); + } + + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation( + "Batch import complete: {Imported} imported, {Skipped} skipped out of {Total} requested", + result.Imported, result.Skipped, result.TotalRequested); + + return result; + } + + public async Task GetGitHubConfigAsync(CancellationToken cancellationToken = default) + { + var dbAppId = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_ID"); + var dbPrivateKey = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + var dbAppName = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_NAME"); + + var hasDbAppId = !string.IsNullOrWhiteSpace(dbAppId?.Value); + var hasDbPrivateKey = !string.IsNullOrWhiteSpace(dbPrivateKey?.Value); + + if (hasDbAppId || hasDbPrivateKey) + { + return new GitHubConfigResponse + { + HasAppId = hasDbAppId, + HasPrivateKey = hasDbPrivateKey, + AppId = dbAppId?.Value, + AppName = dbAppName?.Value, + Source = "database" + }; + } + + // Check environment variables / configuration + var envAppId = _configuration["GitHub:App:Id"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Id") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_ID"); + var envPrivateKey = _configuration["GitHub:App:PrivateKey"] + ?? Environment.GetEnvironmentVariable("GitHub__App__PrivateKey") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_PRIVATE_KEY"); + var envAppName = _configuration["GitHub:App:Name"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Name") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_NAME"); + + if (!string.IsNullOrWhiteSpace(envAppId) || !string.IsNullOrWhiteSpace(envPrivateKey)) + { + return new GitHubConfigResponse + { + HasAppId = !string.IsNullOrWhiteSpace(envAppId), + HasPrivateKey = !string.IsNullOrWhiteSpace(envPrivateKey), + AppId = envAppId, + AppName = envAppName, + Source = "environment" + }; + } + + return new GitHubConfigResponse { Source = "none" }; + } + + public async Task SaveGitHubConfigAsync(SaveGitHubConfigRequest request, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(request.AppId)) + throw new InvalidOperationException("App ID is required."); + + string privateKeyBase64; + + if (!string.IsNullOrWhiteSpace(request.PrivateKey)) + { + // New private key provided -- validate and encode + if (!request.PrivateKey.Contains("-----BEGIN")) + throw new InvalidOperationException("Private Key must be a valid PEM file (should contain '-----BEGIN')."); + + privateKeyBase64 = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(request.PrivateKey)); + } + else + { + // No new private key -- use existing from DB or cache + var existingKey = _cache.PrivateKeyBase64; + if (string.IsNullOrWhiteSpace(existingKey)) + { + var dbKey = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + existingKey = dbKey?.Value; + } + if (string.IsNullOrWhiteSpace(existingKey)) + throw new InvalidOperationException("Private Key is required (no existing key found)."); + + privateKeyBase64 = existingKey; + } + + // Validate credentials by attempting to generate JWT and call GitHub API + await ValidateGitHubCredentialsAsync(request.AppId, privateKeyBase64, cancellationToken); + + // Upsert settings into SystemSettings with category "github" + await UpsertSettingAsync("GITHUB_APP_ID", request.AppId, "github", "GitHub App ID"); + await UpsertSettingAsync("GITHUB_APP_PRIVATE_KEY", privateKeyBase64, "github", "GitHub App private key (base64-encoded PEM)"); + await UpsertSettingAsync("GITHUB_APP_NAME", request.AppName, "github", "GitHub App name (URL slug)"); + await _context.SaveChangesAsync(cancellationToken); + + // Update the in-memory cache so IsConfigured returns true immediately + _cache.Update(request.AppId, privateKeyBase64, request.AppName); + + _logger.LogInformation("GitHub App credentials saved to database for App ID {AppId}", request.AppId); + + return new GitHubConfigResponse + { + HasAppId = true, + HasPrivateKey = true, + AppId = request.AppId, + AppName = request.AppName, + Source = "database" + }; + } + + public async Task ResetGitHubConfigAsync(CancellationToken cancellationToken = default) + { + // Soft-delete all GitHub config settings from SystemSettings + var githubSettings = await _context.SystemSettings + .Where(s => s.Category == "github" && !s.IsDeleted) + .ToListAsync(cancellationToken); + + foreach (var setting in githubSettings) + { + setting.IsDeleted = true; + setting.UpdatedAt = DateTime.UtcNow; + } + + await _context.SaveChangesAsync(cancellationToken); + + // Clear the in-memory cache + _cache.Clear(); + + _logger.LogInformation("GitHub App configuration has been reset"); + } + + private async Task ValidateGitHubCredentialsAsync(string appId, string privateKeyBase64, CancellationToken cancellationToken) + { + try + { + var pemBytes = Convert.FromBase64String(privateKeyBase64); + var pemContent = System.Text.Encoding.UTF8.GetString(pemBytes); + + var rsa = RSA.Create(); + rsa.ImportFromPem(pemContent.AsSpan()); + + var securityKey = new RsaSecurityKey(rsa); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.RsaSha256); + + var now = DateTime.UtcNow; + var token = new JwtSecurityToken( + issuer: appId, + claims: new[] + { + new Claim("iat", new DateTimeOffset(now.AddSeconds(-60)).ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) + }, + expires: now.AddMinutes(10), + signingCredentials: credentials + ); + + var jwt = new JwtSecurityTokenHandler().WriteToken(token); + + // Call GitHub API to verify the credentials + using var client = _httpClientFactory.CreateClient("GitHubApp"); + client.BaseAddress = new Uri("https://api.github.com"); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OpenDeepWiki", "1.0")); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + client.Timeout = TimeSpan.FromSeconds(15); + + var response = await client.GetAsync("/app", cancellationToken); + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"GitHub API returned {(int)response.StatusCode}: {body}"); + } + } + catch (InvalidOperationException) + { + throw; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to validate GitHub App credentials: {ex.Message}", ex); + } + } + + private async Task UpsertSettingAsync(string key, string? value, string category, string description) + { + // Check for any existing row (including soft-deleted) to avoid UNIQUE constraint violation + var existing = await _context.SystemSettings + .FirstOrDefaultAsync(s => s.Key == key); + + if (existing != null) + { + existing.Value = value; + existing.Description = description; + existing.Category = category; + existing.IsDeleted = false; + existing.UpdatedAt = DateTime.UtcNow; + } + else + { + _context.SystemSettings.Add(new SystemSetting + { + Id = Guid.NewGuid().ToString(), + Key = key, + Value = value, + Description = description, + Category = category, + CreatedAt = DateTime.UtcNow + }); + } + } +} diff --git a/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs b/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs new file mode 100644 index 00000000..fb8dc5a0 --- /dev/null +++ b/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs @@ -0,0 +1,15 @@ +using OpenDeepWiki.Models.Admin; + +namespace OpenDeepWiki.Services.Admin; + +public interface IAdminGitHubImportService +{ + Task GetStatusAsync(CancellationToken cancellationToken = default); + Task StoreInstallationAsync(long installationId, CancellationToken cancellationToken = default); + Task ListInstallationReposAsync(long installationId, int page, int perPage, CancellationToken cancellationToken = default); + Task BatchImportAsync(BatchImportRequest request, string ownerUserId, CancellationToken cancellationToken = default); + Task GetGitHubConfigAsync(CancellationToken cancellationToken = default); + Task SaveGitHubConfigAsync(SaveGitHubConfigRequest request, CancellationToken cancellationToken = default); + Task DisconnectInstallationAsync(string installationId, CancellationToken cancellationToken = default); + Task ResetGitHubConfigAsync(CancellationToken cancellationToken = default); +} diff --git a/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs b/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs index e227291c..e0d7b278 100644 --- a/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs +++ b/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs @@ -9,64 +9,86 @@ namespace OpenDeepWiki.Services.Admin; /// -/// 系统设置帮助类，用于从环境变量初始化默认设置 +/// Helper class for initializing default system settings from environment variables. /// public static class SystemSettingDefaults { /// - /// Wiki生成器相关的默认设置 + /// Default settings for the wiki generator. /// - public static readonly (string Key, string Category, string Description)[] WikiGeneratorDefaults = + public static readonly (string Key, string Category, string Description)[] WikiGeneratorDefaults = [ - ("WIKI_CATALOG_MODEL", "ai", "目录生成使用的AI模型"), - ("WIKI_CATALOG_ENDPOINT", "ai", "目录生成API端点"), - ("WIKI_CATALOG_API_KEY", "ai", "目录生成API密钥"), - ("WIKI_CATALOG_REQUEST_TYPE", "ai", "目录生成请求类型"), - ("WIKI_CONTENT_MODEL", "ai", "内容生成使用的AI模型"), - ("WIKI_CONTENT_ENDPOINT", "ai", "内容生成API端点"), - ("WIKI_CONTENT_API_KEY", "ai", "内容生成API密钥"), - ("WIKI_CONTENT_REQUEST_TYPE", "ai", "内容生成请求类型"), - ("WIKI_TRANSLATION_MODEL", "ai", "翻译使用的AI模型"), - ("WIKI_TRANSLATION_ENDPOINT", "ai", "翻译API端点"), - ("WIKI_TRANSLATION_API_KEY", "ai", "翻译API密钥"), - ("WIKI_TRANSLATION_REQUEST_TYPE", "ai", "翻译请求类型"), - ("WIKI_LANGUAGES", "ai", "支持的语言列表（逗号分隔）"), - ("WIKI_PARALLEL_COUNT", "ai", "并行生成文档数量"), - ("WIKI_MAX_OUTPUT_TOKENS", "ai", "最大输出Token数量"), - ("WIKI_DOCUMENT_GENERATION_TIMEOUT_MINUTES", "ai", "文档生成超时时间（分钟）"), - ("WIKI_TRANSLATION_TIMEOUT_MINUTES", "ai", "翻译超时时间（分钟）"), - ("WIKI_TITLE_TRANSLATION_TIMEOUT_MINUTES", "ai", "标题翻译超时时间（分钟）"), - ("WIKI_README_MAX_LENGTH", "ai", "README内容最大长度"), - ("WIKI_DIRECTORY_TREE_MAX_DEPTH", "ai", "目录树最大深度"), - ("WIKI_MAX_RETRY_ATTEMPTS", "ai", "最大重试次数"), - ("WIKI_RETRY_DELAY_MS", "ai", "重试延迟时间（毫秒）"), - ("WIKI_PROMPTS_DIRECTORY", "ai", "提示模板目录") + ("WIKI_CATALOG_MODEL", "ai", "AI model used for catalog generation"), + ("WIKI_CATALOG_ENDPOINT", "ai", "API endpoint for catalog generation"), + ("WIKI_CATALOG_API_KEY", "ai", "API key for catalog generation"), + ("WIKI_CATALOG_REQUEST_TYPE", "ai", "Request type for catalog generation"), + ("WIKI_CONTENT_MODEL", "ai", "AI model used for content generation"), + ("WIKI_CONTENT_ENDPOINT", "ai", "API endpoint for content generation"), + ("WIKI_CONTENT_API_KEY", "ai", "API key for content generation"), + ("WIKI_CONTENT_REQUEST_TYPE", "ai", "Request type for content generation"), + ("WIKI_TRANSLATION_MODEL", "ai", "AI model used for translation"), + ("WIKI_TRANSLATION_ENDPOINT", "ai", "API endpoint for translation"), + ("WIKI_TRANSLATION_API_KEY", "ai", "API key for translation"), + ("WIKI_TRANSLATION_REQUEST_TYPE", "ai", "Request type for translation"), + ("WIKI_LANGUAGES", "ai", "Supported languages (comma-separated)"), + ("WIKI_PARALLEL_COUNT", "ai", "Number of parallel document generation tasks"), + ("WIKI_MAX_OUTPUT_TOKENS", "ai", "Maximum output token count"), + ("WIKI_DOCUMENT_GENERATION_TIMEOUT_MINUTES", "ai", "Document generation timeout (minutes)"), + ("WIKI_TRANSLATION_TIMEOUT_MINUTES", "ai", "Translation timeout (minutes)"), + ("WIKI_TITLE_TRANSLATION_TIMEOUT_MINUTES", "ai", "Title translation timeout (minutes)"), + ("WIKI_README_MAX_LENGTH", "ai", "Maximum README content length"), + ("WIKI_DIRECTORY_TREE_MAX_DEPTH", "ai", "Maximum directory tree depth"), + ("WIKI_MAX_RETRY_ATTEMPTS", "ai", "Maximum retry attempts"), + ("WIKI_RETRY_DELAY_MS", "ai", "Retry delay (milliseconds)"), + ("WIKI_PROMPTS_DIRECTORY", "ai", "Prompt templates directory") ]; /// - /// 初始化系统设置默认值（仅在数据库中不存在对应设置时生效） + /// Default settings for the GitHub App integration. + /// Only seeded if env vars are present (unlike wiki settings which always seed). + /// + public static readonly (string Key, string Category, string Description)[] GitHubAppDefaults = + [ + ("GITHUB_APP_ID", "github", "GitHub App ID"), + ("GITHUB_APP_NAME", "github", "GitHub App name (URL slug)"), + ("GITHUB_APP_PRIVATE_KEY", "github", "GitHub App private key (base64-encoded PEM)") + ]; + + /// + /// Initialize default system settings (only for keys not already in the database). /// public static async Task InitializeDefaultsAsync(IConfiguration configuration, IContext context) { - var existingKeys = await context.SystemSettings + var existingSettings = await context.SystemSettings .Where(s => !s.IsDeleted) - .Select(s => s.Key) .ToListAsync(); + var existingByKey = existingSettings.ToDictionary(s => s.Key); + var settingsToAdd = new List(); + var hasChanges = false; - // 准备 WikiGeneratorOptions 的默认值，方便在没有环境变量时也能写入 + // Prepare WikiGeneratorOptions defaults so values can be written even without env vars var wikiOptionDefaults = new WikiGeneratorOptions(); var wikiSection = configuration.GetSection(WikiGeneratorOptions.SectionName); wikiSection.Bind(wikiOptionDefaults); - // 将 PostConfigure 内的环境变量覆盖也应用到默认选项，确保与运行时一致 + // Apply environment variable overrides to match runtime behavior ApplyEnvironmentOverrides(wikiOptionDefaults, configuration); - // 处理Wiki生成器相关设置 + // Process wiki generator settings foreach (var (key, category, description) in WikiGeneratorDefaults) { - if (!existingKeys.Contains(key)) + if (existingByKey.TryGetValue(key, out var existing)) + { + // Update description if it changed (e.g. translated from Chinese to English) + if (existing.Description != description) + { + existing.Description = description; + hasChanges = true; + } + } + else { var envValue = GetEnvironmentOrConfigurationValue(configuration, key); var fallbackValue = GetOptionDefaultValue(wikiOptionDefaults, key); @@ -84,31 +106,65 @@ public static async Task InitializeDefaultsAsync(IConfiguration configuration, I } } + // Process GitHub App settings (only seed if env var has a value) + foreach (var (key, category, description) in GitHubAppDefaults) + { + if (existingByKey.TryGetValue(key, out var existing)) + { + if (existing.Description != description) + { + existing.Description = description; + hasChanges = true; + } + } + else + { + var envValue = GetEnvironmentOrConfigurationValue(configuration, key); + if (!string.IsNullOrWhiteSpace(envValue)) + { + settingsToAdd.Add(new SystemSetting + { + Id = Guid.NewGuid().ToString(), + Key = key, + Value = envValue, + Description = description, + Category = category, + CreatedAt = DateTime.UtcNow + }); + } + } + } + if (settingsToAdd.Count > 0) { context.SystemSettings.AddRange(settingsToAdd); + hasChanges = true; + } + + if (hasChanges) + { await context.SaveChangesAsync(); } } /// - /// 从环境变量或配置中获取值 + /// Get value from environment variable or configuration. /// private static string? GetEnvironmentOrConfigurationValue(IConfiguration configuration, string key) { - // 优先从环境变量获取 + // Prefer environment variable var envValue = Environment.GetEnvironmentVariable(key); if (!string.IsNullOrWhiteSpace(envValue)) { return envValue; } - // 从配置中获取（支持appsettings.json等） + // Fall back to configuration (appsettings.json etc.) return configuration[key]; } /// - /// 将环境变量中的值应用到 WikiGeneratorOptions，以保持与运行时一致 + /// Apply environment variable values to WikiGeneratorOptions for runtime consistency. /// private static void ApplyEnvironmentOverrides(WikiGeneratorOptions options, IConfiguration configuration) { @@ -123,7 +179,7 @@ private static void ApplyEnvironmentOverrides(WikiGeneratorOptions options, ICon } /// - /// 将系统设置应用到WikiGeneratorOptions + /// Apply system settings to WikiGeneratorOptions. /// public static async Task ApplyToWikiGeneratorOptions(WikiGeneratorOptions options, IAdminSettingsService settingsService) { @@ -138,7 +194,7 @@ public static async Task ApplyToWikiGeneratorOptions(WikiGeneratorOptions option } /// - /// 从 WikiGeneratorOptions 中获取默认值（字符串） + /// Get the default value from WikiGeneratorOptions as a string. /// private static string? GetOptionDefaultValue(WikiGeneratorOptions options, string key) { @@ -172,7 +228,7 @@ public static async Task ApplyToWikiGeneratorOptions(WikiGeneratorOptions option } /// - /// 将单个设置应用到WikiGeneratorOptions + /// Apply a single setting to WikiGeneratorOptions. /// public static void ApplySettingToOption(WikiGeneratorOptions options, string key, string value) { diff --git a/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs b/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs new file mode 100644 index 00000000..273a3f6d --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs @@ -0,0 +1,69 @@ +using OpenDeepWiki.Services.Admin; + +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Thread-safe singleton cache for GitHub App credentials loaded from the database. +/// Checked first by GitHubAppService before falling back to env vars/config. +/// +public class GitHubAppCredentialCache +{ + private readonly object _lock = new(); + private string? _appId; + private string? _privateKeyBase64; + private string? _appName; + + public string? AppId + { + get { lock (_lock) return _appId; } + } + + public string? PrivateKeyBase64 + { + get { lock (_lock) return _privateKeyBase64; } + } + + public string? AppName + { + get { lock (_lock) return _appName; } + } + + /// + /// Update all cached credentials atomically. + /// + public void Update(string? appId, string? privateKeyBase64, string? appName) + { + lock (_lock) + { + _appId = string.IsNullOrWhiteSpace(appId) ? null : appId; + _privateKeyBase64 = string.IsNullOrWhiteSpace(privateKeyBase64) ? null : privateKeyBase64; + _appName = string.IsNullOrWhiteSpace(appName) ? null : appName; + } + } + + /// + /// Clear all cached credentials atomically. + /// + public void Clear() + { + lock (_lock) + { + _appId = null; + _privateKeyBase64 = null; + _appName = null; + } + } + + /// + /// Load credentials from the SystemSettings database table. + /// Called at startup and after saving new credentials via the admin UI. + /// + public async Task LoadFromDbAsync(IAdminSettingsService settingsService) + { + var appId = await settingsService.GetSettingByKeyAsync("GITHUB_APP_ID"); + var privateKey = await settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + var appName = await settingsService.GetSettingByKeyAsync("GITHUB_APP_NAME"); + + Update(appId?.Value, privateKey?.Value, appName?.Value); + } +} diff --git a/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs b/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs new file mode 100644 index 00000000..88e51245 --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs @@ -0,0 +1,255 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using OpenDeepWiki.EFCore; +using OpenDeepWiki.Entities; + +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Implements GitHub App authentication and API operations. +/// Uses JWT signed with the app's private key to generate installation tokens. +/// +public class GitHubAppService : IGitHubAppService +{ + private readonly IConfiguration _configuration; + private readonly GitHubAppCredentialCache _cache; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IContext _context; + private readonly ILogger _logger; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true + }; + + public GitHubAppService( + IConfiguration configuration, + GitHubAppCredentialCache cache, + IHttpClientFactory httpClientFactory, + IContext context, + ILogger logger) + { + _configuration = configuration; + _cache = cache; + _httpClientFactory = httpClientFactory; + _context = context; + _logger = logger; + } + + private string? AppId => + _cache.AppId + ?? _configuration["GitHub:App:Id"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Id") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_ID"); + + private string? PrivateKeyBase64 => + _cache.PrivateKeyBase64 + ?? _configuration["GitHub:App:PrivateKey"] + ?? Environment.GetEnvironmentVariable("GitHub__App__PrivateKey") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_PRIVATE_KEY"); + + public bool IsConfigured => !string.IsNullOrWhiteSpace(AppId) && !string.IsNullOrWhiteSpace(PrivateKeyBase64); + + /// + /// Generate a JWT for authenticating as the GitHub App. + /// JWT is valid for 10 minutes (GitHub maximum). + /// + private string GenerateJwt() + { + if (!IsConfigured) + throw new InvalidOperationException("GitHub App is not configured. Set GitHub:App:Id and GitHub:App:PrivateKey."); + + var pemBytes = Convert.FromBase64String(PrivateKeyBase64!); + var pemContent = System.Text.Encoding.UTF8.GetString(pemBytes); + + var rsa = RSA.Create(); + rsa.ImportFromPem(pemContent.AsSpan()); + + var securityKey = new RsaSecurityKey(rsa); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.RsaSha256); + + var now = DateTime.UtcNow; + var token = new JwtSecurityToken( + issuer: AppId, + claims: new[] + { + new Claim("iat", new DateTimeOffset(now.AddSeconds(-60)).ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) + }, + expires: now.AddMinutes(10), + signingCredentials: credentials + ); + + return new JwtSecurityTokenHandler().WriteToken(token); + } + + private HttpClient CreateClient() + { + var client = _httpClientFactory.CreateClient("GitHubApp"); + client.BaseAddress = new Uri("https://api.github.com"); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OpenDeepWiki", "1.0")); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + return client; + } + + public async Task> ListInstallationsAsync(CancellationToken cancellationToken = default) + { + var jwt = GenerateJwt(); + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + + var response = await client.GetAsync("/app/installations", cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var installations = JsonSerializer.Deserialize>(json, JsonOptions) ?? new(); + + return installations.Select(i => new GitHubInstallationInfo + { + Id = i.Id, + AccountLogin = i.Account?.Login ?? string.Empty, + AccountType = i.Account?.Type ?? string.Empty, + AccountId = i.Account?.Id ?? 0, + AvatarUrl = i.Account?.AvatarUrl + }).ToList(); + } + + public async Task GetInstallationTokenAsync(long installationId, CancellationToken cancellationToken = default) + { + // Check cache in DB first + var installation = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.InstallationId == installationId, cancellationToken); + + if (installation?.CachedAccessToken != null && + installation.TokenExpiresAt.HasValue && + installation.TokenExpiresAt.Value > DateTime.UtcNow.AddMinutes(5)) + { + _logger.LogDebug("Using cached installation token for installation {InstallationId}", installationId); + return installation.CachedAccessToken; + } + + // Generate fresh token + var jwt = GenerateJwt(); + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + + var response = await client.PostAsync( + $"/app/installations/{installationId}/access_tokens", + null, + cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var tokenResponse = JsonSerializer.Deserialize(json, JsonOptions); + + if (tokenResponse?.Token == null) + throw new InvalidOperationException($"Failed to get installation token for installation {installationId}"); + + // Cache the token + if (installation != null) + { + installation.CachedAccessToken = tokenResponse.Token; + installation.TokenExpiresAt = tokenResponse.ExpiresAt; + installation.UpdateTimestamp(); + await _context.SaveChangesAsync(cancellationToken); + } + + _logger.LogInformation("Generated fresh installation token for installation {InstallationId}, expires at {ExpiresAt}", + installationId, tokenResponse.ExpiresAt); + + return tokenResponse.Token; + } + + public async Task ListInstallationReposAsync( + long installationId, int page = 1, int perPage = 30, CancellationToken cancellationToken = default) + { + var token = await GetInstallationTokenAsync(installationId, cancellationToken); + + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + + var response = await client.GetAsync( + $"/installation/repositories?per_page={perPage}&page={page}", + cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var result = JsonSerializer.Deserialize(json, JsonOptions); + + return new GitHubRepoListResult + { + TotalCount = result?.TotalCount ?? 0, + Repositories = result?.Repositories?.Select(r => new GitHubInstallationRepo + { + Id = r.Id, + FullName = r.FullName ?? string.Empty, + Name = r.Name ?? string.Empty, + Owner = r.Owner?.Login ?? string.Empty, + Private = r.Private, + Description = r.Description, + Language = r.Language, + StargazersCount = r.StargazersCount, + ForksCount = r.ForksCount, + DefaultBranch = r.DefaultBranch ?? "main", + CloneUrl = r.CloneUrl ?? string.Empty, + HtmlUrl = r.HtmlUrl ?? string.Empty + }).ToList() ?? new() + }; + } + + // Internal DTOs for GitHub API responses + private class GitHubInstallationResponse + { + public long Id { get; set; } + public GitHubAccountResponse? Account { get; set; } + } + + private class GitHubAccountResponse + { + public long Id { get; set; } + public string? Login { get; set; } + public string? Type { get; set; } + public string? AvatarUrl { get; set; } + } + + private class GitHubTokenResponse + { + public string? Token { get; set; } + public DateTime? ExpiresAt { get; set; } + } + + private class GitHubRepoListResponse + { + public int TotalCount { get; set; } + public List? Repositories { get; set; } + } + + private class GitHubRepoResponse + { + public long Id { get; set; } + public string? FullName { get; set; } + public string? Name { get; set; } + public GitHubOwnerResponse? Owner { get; set; } + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string? DefaultBranch { get; set; } + public string? CloneUrl { get; set; } + public string? HtmlUrl { get; set; } + } + + private class GitHubOwnerResponse + { + public string? Login { get; set; } + } +} diff --git a/web/app/admin/github-import/page.tsx b/web/app/admin/github-import/page.tsx new file mode 100644 index 00000000..c456d4f9 --- /dev/null +++ b/web/app/admin/github-import/page.tsx @@ -0,0 +1,967 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useState } from "react"; +import { useTranslations } from "@/hooks/use-translations"; +import { toast } from "sonner"; +import { + getGitHubStatus, + getGitHubInstallUrl, + getGitHubConfig, + saveGitHubConfig, + resetGitHubConfig, + getInstallationRepos, + batchImportRepos, + getDepartments, + disconnectGitHubInstallation, + GitHubStatus, + GitHubConfig, + GitHubRepo, + GitHubInstallation, + BatchImportResult, + AdminDepartment, +} from "@/lib/admin-api"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Label } from "@/components/ui/label"; +import { + RefreshCw, + ExternalLink, + GitBranch, + Star, + GitFork, + Lock, + Globe, + Loader2, + Building2, + CheckCircle2, + AlertCircle, + Plus, + Download, + Search, + Key, + Save, + HelpCircle, + Info, + Unlink, + RotateCcw, + Upload, +} from "lucide-react"; + +const PAGE_SIZE = 30; +const FETCH_BATCH_SIZE = 100; + +export default function GitHubImportPage() { + const t = useTranslations(); + + // Status + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + + // Selected installation + const [selectedInstallation, setSelectedInstallation] = useState(null); + + // All repos (fetched in full) + const [allRepos, setAllRepos] = useState([]); + const [repoTotalCount, setRepoTotalCount] = useState(0); + const [repoLoading, setRepoLoading] = useState(false); + const [loadProgress, setLoadProgress] = useState(""); + + // Selection + const [selectedRepos, setSelectedRepos] = useState>(new Set()); + + // Filters + const [searchQuery, setSearchQuery] = useState(""); + const [languageFilter, setLanguageFilter] = useState("all"); + + // Client-side pagination + const [page, setPage] = useState(1); + + // Import + const [departments, setDepartments] = useState([]); + const [selectedDepartmentId, setSelectedDepartmentId] = useState(""); + const [languageCode, setLanguageCode] = useState("en"); + const [importing, setImporting] = useState(false); + const [importResult, setImportResult] = useState(null); + + // GitHub App configuration form + const [config, setConfig] = useState(null); + const [configForm, setConfigForm] = useState({ + appId: "", + appName: "", + privateKey: "", + }); + const [saving, setSaving] = useState(false); + const [disconnecting, setDisconnecting] = useState(null); + const [resetting, setResetting] = useState(false); + + const fetchConfig = useCallback(async () => { + try { + const result = await getGitHubConfig(); + setConfig(result); + if (result.appId) setConfigForm((f) => ({ ...f, appId: result.appId || "" })); + if (result.appName) setConfigForm((f) => ({ ...f, appName: result.appName || "" })); + } catch { + // Config endpoint may not be available + } + }, []); + + const handleSaveConfig = async () => { + if (!configForm.appId.trim()) { + toast.error(t("admin.githubImport.validationAppIdRequired")); + return; + } + // When updating existing config, privateKey can be empty (keep existing) + if (!config?.hasPrivateKey && !configForm.privateKey.trim()) { + toast.error(t("admin.githubImport.validationPrivateKeyRequired")); + return; + } + if (configForm.privateKey.trim() && !configForm.privateKey.includes("-----BEGIN")) { + toast.error(t("admin.githubImport.validationPrivateKeyInvalid")); + return; + } + + setSaving(true); + try { + await saveGitHubConfig({ + appId: configForm.appId.trim(), + appName: configForm.appName.trim(), + privateKey: configForm.privateKey, + }); + toast.success(t("admin.githubImport.configSaved")); + setConfigForm((f) => ({ ...f, privateKey: "" })); + // Refresh status and config to show the configured state + await fetchStatus(); + await fetchConfig(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.configSaveFailed") + ); + } finally { + setSaving(false); + } + }; + + const fetchStatus = useCallback(async () => { + setLoading(true); + try { + const result = await getGitHubStatus(); + setStatus(result); + if (result.installations.length > 0 && !selectedInstallation) { + setSelectedInstallation(result.installations[0]); + } + } catch (error) { + toast.error(t("admin.githubImport.fetchStatusFailed")); + } finally { + setLoading(false); + } + }, [t, selectedInstallation]); + + const fetchDepartments = useCallback(async () => { + try { + const result = await getDepartments(); + setDepartments(result); + if (result.length > 0 && !selectedDepartmentId) { + setSelectedDepartmentId(result[0].id); + } + } catch { + // Departments may not be available + } + }, [selectedDepartmentId]); + + // Fetch ALL repos from the installation (paginated API calls in background) + const fetchAllRepos = useCallback(async () => { + if (!selectedInstallation) return; + setRepoLoading(true); + setAllRepos([]); + setRepoTotalCount(0); + setLoadProgress(""); + + try { + // First request to get total count + const firstResult = await getInstallationRepos( + selectedInstallation.installationId, + 1, + FETCH_BATCH_SIZE + ); + const total = firstResult.totalCount; + setRepoTotalCount(total); + + let accumulated = [...firstResult.repositories]; + setAllRepos(accumulated); + setLoadProgress(`${accumulated.length} / ${total}`); + + // Fetch remaining pages + const totalPages = Math.ceil(total / FETCH_BATCH_SIZE); + for (let p = 2; p <= totalPages; p++) { + const result = await getInstallationRepos( + selectedInstallation.installationId, + p, + FETCH_BATCH_SIZE + ); + accumulated = [...accumulated, ...result.repositories]; + setAllRepos(accumulated); + setLoadProgress(`${accumulated.length} / ${total}`); + } + } catch (error) { + toast.error(t("admin.githubImport.fetchReposFailed")); + } finally { + setRepoLoading(false); + setLoadProgress(""); + } + }, [selectedInstallation, t]); + + useEffect(() => { + fetchStatus(); + fetchDepartments(); + fetchConfig(); + }, []); + + useEffect(() => { + if (selectedInstallation) { + setSelectedRepos(new Set()); + setImportResult(null); + setSearchQuery(""); + setLanguageFilter("all"); + setPage(1); + fetchAllRepos(); + } + }, [selectedInstallation]); + + const handleConnectNew = async () => { + try { + const { url } = await getGitHubInstallUrl(); + window.location.href = url; + } catch (error) { + toast.error("Failed to get install URL"); + } + }; + + const handleDisconnect = async (inst: GitHubInstallation) => { + const confirmed = window.confirm( + t("admin.githubImport.disconnectConfirm").replace("{org}", inst.accountLogin) + ); + if (!confirmed) return; + + setDisconnecting(inst.id); + try { + await disconnectGitHubInstallation(inst.id); + toast.success(t("admin.githubImport.disconnectSuccess")); + if (selectedInstallation?.id === inst.id) { + setSelectedInstallation(null); + setAllRepos([]); + } + await fetchStatus(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.disconnectFailed") + ); + } finally { + setDisconnecting(null); + } + }; + + const handleResetConfig = async () => { + const confirmed = window.confirm( + t("admin.githubImport.resetConfigConfirm") + ); + if (!confirmed) return; + + setResetting(true); + try { + await resetGitHubConfig(); + toast.success(t("admin.githubImport.resetConfigSuccess")); + setSelectedInstallation(null); + setAllRepos([]); + setConfig(null); + setConfigForm({ appId: "", appName: "", privateKey: "" }); + await fetchStatus(); + await fetchConfig(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.resetConfigFailed") + ); + } finally { + setResetting(false); + } + }; + + const toggleRepo = (fullName: string) => { + setSelectedRepos((prev) => { + const next = new Set(prev); + if (next.has(fullName)) { + next.delete(fullName); + } else { + next.add(fullName); + } + return next; + }); + }; + + // Apply filters across ALL repos + const filteredRepos = useMemo(() => { + return allRepos.filter((r) => { + if (searchQuery) { + const q = searchQuery.toLowerCase(); + const matchesName = r.fullName.toLowerCase().includes(q); + const matchesDesc = r.description?.toLowerCase().includes(q); + if (!matchesName && !matchesDesc) return false; + } + if (languageFilter !== "all" && (r.language || "").toLowerCase() !== languageFilter.toLowerCase()) { + return false; + } + return true; + }); + }, [allRepos, searchQuery, languageFilter]); + + const importableFiltered = useMemo( + () => filteredRepos.filter((r) => !r.alreadyImported), + [filteredRepos] + ); + + // Client-side pagination of filtered results + const totalFilteredPages = Math.ceil(filteredRepos.length / PAGE_SIZE); + const pagedRepos = useMemo( + () => filteredRepos.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE), + [filteredRepos, page] + ); + + // Collect unique languages from ALL repos for the filter dropdown + const repoLanguages = useMemo( + () => Array.from(new Set(allRepos.map((r) => r.language).filter(Boolean) as string[])).sort(), + [allRepos] + ); + + // Reset page when filters change + useEffect(() => { + setPage(1); + }, [searchQuery, languageFilter]); + + const toggleSelectAll = () => { + if (selectedRepos.size === importableFiltered.length && importableFiltered.length > 0) { + setSelectedRepos(new Set()); + } else { + setSelectedRepos(new Set(importableFiltered.map((r) => r.fullName))); + } + }; + + const handleImport = async () => { + if (!selectedInstallation || selectedRepos.size === 0 || !selectedDepartmentId) return; + + setImporting(true); + setImportResult(null); + try { + const selectedRepoData = allRepos + .filter((r) => selectedRepos.has(r.fullName)) + .map((r) => ({ + fullName: r.fullName, + name: r.name, + owner: r.owner, + cloneUrl: r.cloneUrl, + defaultBranch: r.defaultBranch, + private: r.private, + language: r.language, + stargazersCount: r.stargazersCount, + forksCount: r.forksCount, + })); + + const result = await batchImportRepos({ + installationId: selectedInstallation.installationId, + departmentId: selectedDepartmentId, + languageCode, + repos: selectedRepoData, + }); + + setImportResult(result); + setSelectedRepos(new Set()); + toast.success( + t("admin.githubImport.importSuccess") + .replace("{imported}", result.imported.toString()) + .replace("{skipped}", result.skipped.toString()) + ); + + // Refresh repo list to update "already imported" flags + fetchAllRepos(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.importFailed") + ); + } finally { + setImporting(false); + } + }; + + if (loading) { + return ( +
+ +
+ ); + } + + return ( +
+ {/* Header */} +
+
+

{t("admin.githubImport.title")}

+

+ {t("admin.githubImport.description")} +

+
+ +
+ + {/* Status Card */} + + + {t("admin.githubImport.connectionStatus")} + + + {!status?.configured ? ( +
+ + {t("admin.githubImport.credentialSourceNone")} +
+ ) : ( +
+
+ + {t("admin.githubImport.configured")} + {config && ( + + ({config.source === "database" + ? t("admin.githubImport.credentialSourceDb") + : t("admin.githubImport.credentialSourceEnv")}) + + )} +
+ + {/* Connected Organizations */} + {status.installations.length > 0 && ( +
+

+ {t("admin.githubImport.connectedOrgs")} +

+
+ {status.installations.map((inst) => ( +
setSelectedInstallation(inst)} + > +
+ {inst.avatarUrl && ( + {inst.accountLogin} + )} +
+ {inst.accountLogin} + + {inst.accountType} + +
+
+
+ {inst.departmentName && ( + + + {inst.departmentName} + + )} + +
+
+ ))} +
+
+ )} + +
+ + +
+
+ )} + + + + {/* Repository Selection */} + {selectedInstallation && ( + + + + {t("admin.githubImport.importFrom").replace( + "{org}", + selectedInstallation.accountLogin + )} + + + {repoLoading && loadProgress + ? `${t("admin.githubImport.loadingRepos")} ${loadProgress}...` + : t("admin.githubImport.totalRepos").replace( + "{count}", + repoTotalCount.toString() + )} + + + + {/* Import Options */} +
+
+ + +
+ +
+ + +
+
+ + {/* Search and Filter */} +
+
+ + setSearchQuery(e.target.value)} + className="pl-9" + /> +
+ +
+ + {/* Select All */} +
+ 0 && + selectedRepos.size === importableFiltered.length + } + onCheckedChange={toggleSelectAll} + /> + + {t("admin.githubImport.selectAll")} + {searchQuery || languageFilter !== "all" + ? ` (${filteredRepos.length} ${t("admin.githubImport.matching")})` + : ""} + {selectedRepos.size > 0 && + ` - ${selectedRepos.size} ${t("admin.githubImport.selected")}`} + +
+ + {/* Repository List */} + {repoLoading && allRepos.length === 0 ? ( +
+ +
+ ) : ( +
+ {pagedRepos.length === 0 && searchQuery && ( +

+ {t("admin.githubImport.noResults")} +

+ )} + {pagedRepos.map((repo) => ( +
+ toggleRepo(repo.fullName)} + /> +
+
+ + {repo.fullName} + {repo.private ? ( + + ) : ( + + )} + {repo.alreadyImported && ( + + {t("admin.githubImport.alreadyImported")} + + )} +
+ {repo.description && ( +

+ {repo.description} +

+ )} +
+
+ {repo.language && ( + + {repo.language} + + )} + + + {repo.stargazersCount} + + + + {repo.forksCount} + + e.stopPropagation()} + > + + +
+
+ ))} +
+ )} + + {/* Pagination */} + {totalFilteredPages > 1 && ( +
+ + + {page} / {totalFilteredPages} + + +
+ )} + + {/* Import Button */} +
+ + {selectedRepos.size > 0 + ? t("admin.githubImport.readyToImport").replace( + "{count}", + selectedRepos.size.toString() + ) + : t("admin.githubImport.selectReposPrompt")} + + +
+ + {/* Import Result */} + {importResult && ( + + +
+ +
+

+ {t("admin.githubImport.importComplete")} +

+

+ {importResult.imported} {t("admin.githubImport.imported")},{" "} + {importResult.skipped} {t("admin.githubImport.skipped")} +

+ {importResult.skippedRepos.length > 0 && ( +

+ {t("admin.githubImport.skippedList")}:{" "} + {importResult.skippedRepos.join(", ")} +

+ )} +
+
+ + + )} + + + )} + + {/* Configuration Form -- shown when not configured */} + {status && !status.configured && ( + + + + + {status.configured + ? t("admin.githubImport.updateConfig") + : t("admin.githubImport.configTitle")} + + + {status.configured + ? t("admin.githubImport.updateConfigDescription") + : t("admin.githubImport.configDescription")} + + + + {config?.source === "environment" && !status.configured && ( +
+ + {t("admin.githubImport.overrideEnvWarning")} +
+ )} + +
+
+ + + setConfigForm((f) => ({ ...f, appId: e.target.value })) + } + /> +

+ {t("admin.githubImport.appIdHelp")} +

+
+ +
+ + + setConfigForm((f) => ({ ...f, appName: e.target.value })) + } + /> +

+ {t("admin.githubImport.appNameHelp")} +

+
+ +
+
+ + + { + const file = e.target.files?.[0]; + if (!file) return; + const reader = new FileReader(); + reader.onload = (ev) => { + const content = ev.target?.result as string; + setConfigForm((f) => ({ ...f, privateKey: content })); + }; + reader.readAsText(file); + e.target.value = ""; + }} + /> +
+