diff --git a/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs b/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs new file mode 100644 index 00000000..23c6363d --- /dev/null +++ b/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs @@ -0,0 +1,148 @@ +using OpenDeepWiki.Models.Admin; +using OpenDeepWiki.Services.Admin; +using OpenDeepWiki.Services.Auth; +using OpenDeepWiki.Services.GitHub; + +namespace OpenDeepWiki.Endpoints.Admin; + +public static class AdminGitHubImportEndpoints +{ + public static RouteGroupBuilder MapAdminGitHubImportEndpoints(this RouteGroupBuilder group) + { + var github = group.MapGroup("/github") + .WithTags("Admin - GitHub Import"); + + github.MapGet("/status", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + var result = await service.GetStatusAsync(cancellationToken); + return Results.Ok(new { success = true, data = result }); + }); + + github.MapGet("/config", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + var result = await service.GetGitHubConfigAsync(cancellationToken); + return Results.Ok(new { success = true, data = result }); + }); + + github.MapPost("/config", async ( + SaveGitHubConfigRequest request, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.SaveGitHubConfigAsync(request, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapDelete("/config", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + await service.ResetGitHubConfigAsync(cancellationToken); + return Results.Ok(new { success = true }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapGet("/install-url", async ( + Microsoft.Extensions.Configuration.IConfiguration configuration, + GitHubAppCredentialCache cache) => + { + var appName = cache.AppName + ?? configuration["GitHub:App:Name"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Name") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_NAME") + ?? "deepwiki-keboola"; + var url = $"https://github.com/apps/{appName}/installations/new"; + return Results.Ok(new { success = true, data = new { url, appName } }); + }); + + github.MapPost("/installations", async ( + StoreInstallationRequest request, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.StoreInstallationAsync(request.InstallationId, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapDelete("/installations/{installationId}", async ( + string installationId, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + await service.DisconnectInstallationAsync(installationId, cancellationToken); + return Results.Ok(new { success = true }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapGet("/installations/{installationId:long}/repos", async ( + long installationId, + int page, + int perPage, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.ListInstallationReposAsync(installationId, page, perPage, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (Exception ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapPost("/batch-import", async ( + BatchImportRequest request, + IAdminGitHubImportService service, + IUserContext userContext, + CancellationToken cancellationToken) => + { + try + { + var userId = userContext.UserId; + if (string.IsNullOrEmpty(userId)) + return Results.Unauthorized(); + + var result = await service.BatchImportAsync(request, userId, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + return group; + } +} diff --git a/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs b/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs new file mode 100644 index 00000000..0d2fd0d3 --- /dev/null +++ b/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs @@ -0,0 +1,100 @@ +namespace OpenDeepWiki.Models.Admin; + +public class GitHubStatusResponse +{ + public bool Configured { get; set; } + public string? AppName { get; set; } + public List Installations { get; set; } = new(); +} + +public class GitHubInstallationDto +{ + public string? Id { get; set; } + public long InstallationId { get; set; } + public string AccountLogin { get; set; } = string.Empty; + public string AccountType { get; set; } = string.Empty; + public long AccountId { get; set; } + public string? AvatarUrl { get; set; } + public string? DepartmentId { get; set; } + public string? DepartmentName { get; set; } + public DateTime CreatedAt { get; set; } +} + +public class StoreInstallationRequest +{ + public long InstallationId { get; set; } +} + +public class GitHubRepoDto +{ + public long Id { get; set; } + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string DefaultBranch { get; set; } = "main"; + public string CloneUrl { get; set; } = string.Empty; + public string HtmlUrl { get; set; } = string.Empty; + public bool AlreadyImported { get; set; } +} + +public class GitHubRepoListDto +{ + public int TotalCount { get; set; } + public List Repositories { get; set; } = new(); + public int Page { get; set; } + public int PerPage { get; set; } +} + +public class BatchImportRequest +{ + public long InstallationId { get; set; } + public string DepartmentId { get; set; } = string.Empty; + public string LanguageCode { get; set; } = "en"; + public List Repos { get; set; } = new(); +} + +public class BatchImportRepo +{ + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public string CloneUrl { get; set; } = string.Empty; + public string DefaultBranch { get; set; } = "main"; + public bool Private { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } +} + +public class BatchImportResult +{ + public int TotalRequested { get; set; } + public int Imported { get; set; } + public int Skipped { get; set; } + public List SkippedRepos { get; set; } = new(); + public List ImportedRepos { get; set; } = new(); +} + +public class SaveGitHubConfigRequest +{ + public string AppId { get; set; } = string.Empty; + public string AppName { get; set; } = string.Empty; + public string PrivateKey { get; set; } = string.Empty; +} + +public class GitHubConfigResponse +{ + public bool HasAppId { get; set; } + public bool HasPrivateKey { get; set; } + public string? AppId { get; set; } + public string? AppName { get; set; } + /// + /// Credential source: "database", "environment", or "none". + /// + public string Source { get; set; } = "none"; +} diff --git a/src/OpenDeepWiki/Program.cs b/src/OpenDeepWiki/Program.cs index f22e89c1..19a8746c 100644 --- a/src/OpenDeepWiki/Program.cs +++ b/src/OpenDeepWiki/Program.cs @@ -11,6 +11,7 @@ using OpenDeepWiki.Infrastructure; using OpenDeepWiki.Services.Admin; using OpenDeepWiki.Services.Auth; +using OpenDeepWiki.Services.GitHub; using OpenDeepWiki.Services.Chat; using OpenDeepWiki.Services.MindMap; using OpenDeepWiki.Services.Notifications; @@ -302,6 +303,11 @@ // 注册处理日志服务（使用 Singleton，因为它内部使用 IServiceScopeFactory 创建独立 scope） builder.Services.AddSingleton(); + // 注册 GitHub App 服务 + builder.Services.AddSingleton(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + // 注册管理端服务 builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -397,6 +403,10 @@ var settingsService = scope.ServiceProvider.GetRequiredService(); var wikiOptions = scope.ServiceProvider.GetRequiredService>(); await SystemSettingDefaults.ApplyToWikiGeneratorOptions(wikiOptions.Value, settingsService); + + // Load GitHub App credentials from DB into the in-memory cache + var githubCache = app.Services.GetRequiredService(); + await githubCache.LoadFromDbAsync(settingsService); } // 启用 CORS @@ -457,8 +467,7 @@ static void LoadEnvFile(IConfigurationBuilder configuration) var envPaths = new[] { Path.Combine(Directory.GetCurrentDirectory(), ".env"), - Path.Combine(AppContext.BaseDirectory, ".env"), - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".env"), + Path.Combine(AppContext.BaseDirectory, ".env") }; foreach (var envPath in envPaths) diff --git a/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs b/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs new file mode 100644 index 00000000..e0f9a7fa --- /dev/null +++ b/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs @@ -0,0 +1,504 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Security.Claims; +using System.Security.Cryptography; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using OpenDeepWiki.EFCore; +using OpenDeepWiki.Entities; +using OpenDeepWiki.Models.Admin; +using OpenDeepWiki.Services.GitHub; + +namespace OpenDeepWiki.Services.Admin; + +public class AdminGitHubImportService : IAdminGitHubImportService +{ + private readonly IContext _context; + private readonly IGitHubAppService _gitHubAppService; + private readonly IAdminSettingsService _settingsService; + private readonly IHttpClientFactory _httpClientFactory; + private readonly GitHubAppCredentialCache _cache; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + + public AdminGitHubImportService( + IContext context, + IGitHubAppService gitHubAppService, + IAdminSettingsService settingsService, + IHttpClientFactory httpClientFactory, + GitHubAppCredentialCache cache, + IConfiguration configuration, + ILogger logger) + { + _context = context; + _gitHubAppService = gitHubAppService; + _settingsService = settingsService; + _httpClientFactory = httpClientFactory; + _cache = cache; + _configuration = configuration; + _logger = logger; + } + + public async Task GetStatusAsync(CancellationToken cancellationToken = default) + { + var response = new GitHubStatusResponse + { + Configured = _gitHubAppService.IsConfigured + }; + + if (!response.Configured) + return response; + + // Load stored installations from DB + var installations = await _context.GitHubAppInstallations + .Where(i => !i.IsDeleted) + .ToListAsync(cancellationToken); + + // Load department names for linked installations + var departmentIds = installations + .Where(i => i.DepartmentId != null) + .Select(i => i.DepartmentId!) + .Distinct() + .ToList(); + + var departments = departmentIds.Count > 0 + ? await _context.Departments + .Where(d => departmentIds.Contains(d.Id)) + .ToDictionaryAsync(d => d.Id, d => d.Name, cancellationToken) + : new Dictionary(); + + response.Installations = installations.Select(i => new GitHubInstallationDto + { + Id = i.Id, + InstallationId = i.InstallationId, + AccountLogin = i.AccountLogin, + AccountType = i.AccountType, + AccountId = i.AccountId, + AvatarUrl = i.AvatarUrl, + DepartmentId = i.DepartmentId, + DepartmentName = i.DepartmentId != null && departments.ContainsKey(i.DepartmentId) + ? departments[i.DepartmentId] + : null, + CreatedAt = i.CreatedAt + }).ToList(); + + return response; + } + + public async Task StoreInstallationAsync(long installationId, CancellationToken cancellationToken = default) + { + // Check if already stored (including soft-deleted, to avoid UNIQUE constraint violation) + var existing = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.InstallationId == installationId, cancellationToken); + + if (existing != null) + { + // Re-activate if soft-deleted + if (existing.IsDeleted) + { + existing.IsDeleted = false; + existing.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + _logger.LogInformation("Re-activated GitHub App installation {InstallationId} for {AccountLogin}", + installationId, existing.AccountLogin); + } + + return new GitHubInstallationDto + { + Id = existing.Id, + InstallationId = existing.InstallationId, + AccountLogin = existing.AccountLogin, + AccountType = existing.AccountType, + AccountId = existing.AccountId, + AvatarUrl = existing.AvatarUrl, + DepartmentId = existing.DepartmentId, + CreatedAt = existing.CreatedAt + }; + } + + // Fetch installation details from GitHub + var allInstallations = await _gitHubAppService.ListInstallationsAsync(cancellationToken); + var info = allInstallations.FirstOrDefault(i => i.Id == installationId); + + if (info == null) + throw new InvalidOperationException($"Installation {installationId} not found on GitHub. Ensure the app is installed."); + + var entity = new GitHubAppInstallation + { + Id = Guid.NewGuid().ToString(), + InstallationId = installationId, + AccountLogin = info.AccountLogin, + AccountType = info.AccountType, + AccountId = info.AccountId, + AvatarUrl = info.AvatarUrl, + }; + + _context.GitHubAppInstallations.Add(entity); + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("Stored GitHub App installation {InstallationId} for {AccountLogin}", + installationId, info.AccountLogin); + + return new GitHubInstallationDto + { + Id = entity.Id, + InstallationId = entity.InstallationId, + AccountLogin = entity.AccountLogin, + AccountType = entity.AccountType, + AccountId = entity.AccountId, + AvatarUrl = entity.AvatarUrl, + CreatedAt = entity.CreatedAt + }; + } + + public async Task DisconnectInstallationAsync(string installationId, CancellationToken cancellationToken = default) + { + var entity = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.Id == installationId && !i.IsDeleted, cancellationToken); + + if (entity == null) + throw new InvalidOperationException($"Installation {installationId} not found."); + + entity.IsDeleted = true; + entity.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("Disconnected GitHub App installation {InstallationId} for {AccountLogin}", + entity.InstallationId, entity.AccountLogin); + } + + public async Task ListInstallationReposAsync( + long installationId, int page, int perPage, CancellationToken cancellationToken = default) + { + var result = await _gitHubAppService.ListInstallationReposAsync(installationId, page, perPage, cancellationToken); + + // Check which repos are already imported + var repoUrls = result.Repositories.Select(r => r.CloneUrl).ToList(); + var existingUrls = await _context.Repositories + .Where(r => repoUrls.Contains(r.GitUrl) && !r.IsDeleted) + .Select(r => r.GitUrl) + .ToListAsync(cancellationToken); + + var existingUrlSet = new HashSet(existingUrls, StringComparer.OrdinalIgnoreCase); + + return new GitHubRepoListDto + { + TotalCount = result.TotalCount, + Page = page, + PerPage = perPage, + Repositories = result.Repositories.Select(r => new GitHubRepoDto + { + Id = r.Id, + FullName = r.FullName, + Name = r.Name, + Owner = r.Owner, + Private = r.Private, + Description = r.Description, + Language = r.Language, + StargazersCount = r.StargazersCount, + ForksCount = r.ForksCount, + DefaultBranch = r.DefaultBranch, + CloneUrl = r.CloneUrl, + HtmlUrl = r.HtmlUrl, + AlreadyImported = existingUrlSet.Contains(r.CloneUrl) + }).ToList() + }; + } + + public async Task BatchImportAsync( + BatchImportRequest request, string ownerUserId, CancellationToken cancellationToken = default) + { + var result = new BatchImportResult + { + TotalRequested = request.Repos.Count + }; + + // Verify department exists + var department = await _context.Departments + .FirstOrDefaultAsync(d => d.Id == request.DepartmentId && !d.IsDeleted, cancellationToken); + + if (department == null) + throw new InvalidOperationException($"Department {request.DepartmentId} not found."); + + // Get existing repos by clone URL to avoid duplicates + var requestUrls = request.Repos.Select(r => r.CloneUrl).ToList(); + var existingRepos = await _context.Repositories + .Where(r => requestUrls.Contains(r.GitUrl) && !r.IsDeleted) + .ToDictionaryAsync(r => r.GitUrl, r => r, StringComparer.OrdinalIgnoreCase, cancellationToken); + + foreach (var repo in request.Repos) + { + if (existingRepos.ContainsKey(repo.CloneUrl)) + { + result.Skipped++; + result.SkippedRepos.Add(repo.FullName); + _logger.LogDebug("Skipping {Repo}: already imported", repo.FullName); + continue; + } + + // Create Repository entity + var repoEntity = new Repository + { + Id = Guid.NewGuid().ToString(), + OwnerUserId = ownerUserId, + GitUrl = repo.CloneUrl, + RepoName = repo.Name, + OrgName = repo.Owner, + IsPublic = !repo.Private, + Status = RepositoryStatus.Pending, + PrimaryLanguage = repo.Language, + StarCount = repo.StargazersCount, + ForkCount = repo.ForksCount + }; + + _context.Repositories.Add(repoEntity); + + // Create default branch + var branch = new RepositoryBranch + { + Id = Guid.NewGuid().ToString(), + RepositoryId = repoEntity.Id, + BranchName = repo.DefaultBranch + }; + + _context.RepositoryBranches.Add(branch); + + // Create branch language + var branchLanguage = new BranchLanguage + { + Id = Guid.NewGuid().ToString(), + RepositoryBranchId = branch.Id, + LanguageCode = request.LanguageCode, + IsDefault = true + }; + + _context.BranchLanguages.Add(branchLanguage); + + // Assign to department + var assignment = new RepositoryAssignment + { + Id = Guid.NewGuid().ToString(), + RepositoryId = repoEntity.Id, + DepartmentId = request.DepartmentId, + AssigneeUserId = ownerUserId + }; + + _context.RepositoryAssignments.Add(assignment); + + result.Imported++; + result.ImportedRepos.Add(repo.FullName); + } + + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation( + "Batch import complete: {Imported} imported, {Skipped} skipped out of {Total} requested", + result.Imported, result.Skipped, result.TotalRequested); + + return result; + } + + public async Task GetGitHubConfigAsync(CancellationToken cancellationToken = default) + { + var dbAppId = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_ID"); + var dbPrivateKey = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + var dbAppName = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_NAME"); + + var hasDbAppId = !string.IsNullOrWhiteSpace(dbAppId?.Value); + var hasDbPrivateKey = !string.IsNullOrWhiteSpace(dbPrivateKey?.Value); + + if (hasDbAppId || hasDbPrivateKey) + { + return new GitHubConfigResponse + { + HasAppId = hasDbAppId, + HasPrivateKey = hasDbPrivateKey, + AppId = dbAppId?.Value, + AppName = dbAppName?.Value, + Source = "database" + }; + } + + // Check environment variables / configuration + var envAppId = _configuration["GitHub:App:Id"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Id") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_ID"); + var envPrivateKey = _configuration["GitHub:App:PrivateKey"] + ?? Environment.GetEnvironmentVariable("GitHub__App__PrivateKey") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_PRIVATE_KEY"); + var envAppName = _configuration["GitHub:App:Name"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Name") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_NAME"); + + if (!string.IsNullOrWhiteSpace(envAppId) || !string.IsNullOrWhiteSpace(envPrivateKey)) + { + return new GitHubConfigResponse + { + HasAppId = !string.IsNullOrWhiteSpace(envAppId), + HasPrivateKey = !string.IsNullOrWhiteSpace(envPrivateKey), + AppId = envAppId, + AppName = envAppName, + Source = "environment" + }; + } + + return new GitHubConfigResponse { Source = "none" }; + } + + public async Task SaveGitHubConfigAsync(SaveGitHubConfigRequest request, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(request.AppId)) + throw new InvalidOperationException("App ID is required."); + + string privateKeyBase64; + + if (!string.IsNullOrWhiteSpace(request.PrivateKey)) + { + // New private key provided -- validate and encode + if (!request.PrivateKey.Contains("-----BEGIN")) + throw new InvalidOperationException("Private Key must be a valid PEM file (should contain '-----BEGIN')."); + + privateKeyBase64 = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(request.PrivateKey)); + } + else + { + // No new private key -- use existing from DB or cache + var existingKey = _cache.PrivateKeyBase64; + if (string.IsNullOrWhiteSpace(existingKey)) + { + var dbKey = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + existingKey = dbKey?.Value; + } + if (string.IsNullOrWhiteSpace(existingKey)) + throw new InvalidOperationException("Private Key is required (no existing key found)."); + + privateKeyBase64 = existingKey; + } + + // Validate credentials by attempting to generate JWT and call GitHub API + await ValidateGitHubCredentialsAsync(request.AppId, privateKeyBase64, cancellationToken); + + // Upsert settings into SystemSettings with category "github" + await UpsertSettingAsync("GITHUB_APP_ID", request.AppId, "github", "GitHub App ID"); + await UpsertSettingAsync("GITHUB_APP_PRIVATE_KEY", privateKeyBase64, "github", "GitHub App private key (base64-encoded PEM)"); + await UpsertSettingAsync("GITHUB_APP_NAME", request.AppName, "github", "GitHub App name (URL slug)"); + await _context.SaveChangesAsync(cancellationToken); + + // Update the in-memory cache so IsConfigured returns true immediately + _cache.Update(request.AppId, privateKeyBase64, request.AppName); + + _logger.LogInformation("GitHub App credentials saved to database for App ID {AppId}", request.AppId); + + return new GitHubConfigResponse + { + HasAppId = true, + HasPrivateKey = true, + AppId = request.AppId, + AppName = request.AppName, + Source = "database" + }; + } + + public async Task ResetGitHubConfigAsync(CancellationToken cancellationToken = default) + { + // Soft-delete all GitHub config settings from SystemSettings + var githubSettings = await _context.SystemSettings + .Where(s => s.Category == "github" && !s.IsDeleted) + .ToListAsync(cancellationToken); + + foreach (var setting in githubSettings) + { + setting.IsDeleted = true; + setting.UpdatedAt = DateTime.UtcNow; + } + + await _context.SaveChangesAsync(cancellationToken); + + // Clear the in-memory cache + _cache.Clear(); + + _logger.LogInformation("GitHub App configuration has been reset"); + } + + private async Task ValidateGitHubCredentialsAsync(string appId, string privateKeyBase64, CancellationToken cancellationToken) + { + try + { + var pemBytes = Convert.FromBase64String(privateKeyBase64); + var pemContent = System.Text.Encoding.UTF8.GetString(pemBytes); + + var rsa = RSA.Create(); + rsa.ImportFromPem(pemContent.AsSpan()); + + var securityKey = new RsaSecurityKey(rsa); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.RsaSha256); + + var now = DateTime.UtcNow; + var token = new JwtSecurityToken( + issuer: appId, + claims: new[] + { + new Claim("iat", new DateTimeOffset(now.AddSeconds(-60)).ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) + }, + expires: now.AddMinutes(10), + signingCredentials: credentials + ); + + var jwt = new JwtSecurityTokenHandler().WriteToken(token); + + // Call GitHub API to verify the credentials + using var client = _httpClientFactory.CreateClient("GitHubApp"); + client.BaseAddress = new Uri("https://api.github.com"); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OpenDeepWiki", "1.0")); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + client.Timeout = TimeSpan.FromSeconds(15); + + var response = await client.GetAsync("/app", cancellationToken); + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"GitHub API returned {(int)response.StatusCode}: {body}"); + } + } + catch (InvalidOperationException) + { + throw; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to validate GitHub App credentials: {ex.Message}", ex); + } + } + + private async Task UpsertSettingAsync(string key, string? value, string category, string description) + { + // Check for any existing row (including soft-deleted) to avoid UNIQUE constraint violation + var existing = await _context.SystemSettings + .FirstOrDefaultAsync(s => s.Key == key); + + if (existing != null) + { + existing.Value = value; + existing.Description = description; + existing.Category = category; + existing.IsDeleted = false; + existing.UpdatedAt = DateTime.UtcNow; + } + else + { + _context.SystemSettings.Add(new SystemSetting + { + Id = Guid.NewGuid().ToString(), + Key = key, + Value = value, + Description = description, + Category = category, + CreatedAt = DateTime.UtcNow + }); + } + } +} diff --git a/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs b/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs new file mode 100644 index 00000000..fb8dc5a0 --- /dev/null +++ b/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs @@ -0,0 +1,15 @@ +using OpenDeepWiki.Models.Admin; + +namespace OpenDeepWiki.Services.Admin; + +public interface IAdminGitHubImportService +{ + Task GetStatusAsync(CancellationToken cancellationToken = default); + Task StoreInstallationAsync(long installationId, CancellationToken cancellationToken = default); + Task ListInstallationReposAsync(long installationId, int page, int perPage, CancellationToken cancellationToken = default); + Task BatchImportAsync(BatchImportRequest request, string ownerUserId, CancellationToken cancellationToken = default); + Task GetGitHubConfigAsync(CancellationToken cancellationToken = default); + Task SaveGitHubConfigAsync(SaveGitHubConfigRequest request, CancellationToken cancellationToken = default); + Task DisconnectInstallationAsync(string installationId, CancellationToken cancellationToken = default); + Task ResetGitHubConfigAsync(CancellationToken cancellationToken = default); +} diff --git a/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs b/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs index e227291c..e0d7b278 100644 --- a/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs +++ b/src/OpenDeepWiki/Services/Admin/SystemSettingDefaults.cs @@ -9,64 +9,86 @@ namespace OpenDeepWiki.Services.Admin; /// -/// 系统设置帮助类，用于从环境变量初始化默认设置 +/// Helper class for initializing default system settings from environment variables. /// public static class SystemSettingDefaults { /// - /// Wiki生成器相关的默认设置 + /// Default settings for the wiki generator. /// - public static readonly (string Key, string Category, string Description)[] WikiGeneratorDefaults = + public static readonly (string Key, string Category, string Description)[] WikiGeneratorDefaults = [ - ("WIKI_CATALOG_MODEL", "ai", "目录生成使用的AI模型"), - ("WIKI_CATALOG_ENDPOINT", "ai", "目录生成API端点"), - ("WIKI_CATALOG_API_KEY", "ai", "目录生成API密钥"), - ("WIKI_CATALOG_REQUEST_TYPE", "ai", "目录生成请求类型"), - ("WIKI_CONTENT_MODEL", "ai", "内容生成使用的AI模型"), - ("WIKI_CONTENT_ENDPOINT", "ai", "内容生成API端点"), - ("WIKI_CONTENT_API_KEY", "ai", "内容生成API密钥"), - ("WIKI_CONTENT_REQUEST_TYPE", "ai", "内容生成请求类型"), - ("WIKI_TRANSLATION_MODEL", "ai", "翻译使用的AI模型"), - ("WIKI_TRANSLATION_ENDPOINT", "ai", "翻译API端点"), - ("WIKI_TRANSLATION_API_KEY", "ai", "翻译API密钥"), - ("WIKI_TRANSLATION_REQUEST_TYPE", "ai", "翻译请求类型"), - ("WIKI_LANGUAGES", "ai", "支持的语言列表（逗号分隔）"), - ("WIKI_PARALLEL_COUNT", "ai", "并行生成文档数量"), - ("WIKI_MAX_OUTPUT_TOKENS", "ai", "最大输出Token数量"), - ("WIKI_DOCUMENT_GENERATION_TIMEOUT_MINUTES", "ai", "文档生成超时时间（分钟）"), - ("WIKI_TRANSLATION_TIMEOUT_MINUTES", "ai", "翻译超时时间（分钟）"), - ("WIKI_TITLE_TRANSLATION_TIMEOUT_MINUTES", "ai", "标题翻译超时时间（分钟）"), - ("WIKI_README_MAX_LENGTH", "ai", "README内容最大长度"), - ("WIKI_DIRECTORY_TREE_MAX_DEPTH", "ai", "目录树最大深度"), - ("WIKI_MAX_RETRY_ATTEMPTS", "ai", "最大重试次数"), - ("WIKI_RETRY_DELAY_MS", "ai", "重试延迟时间（毫秒）"), - ("WIKI_PROMPTS_DIRECTORY", "ai", "提示模板目录") + ("WIKI_CATALOG_MODEL", "ai", "AI model used for catalog generation"), + ("WIKI_CATALOG_ENDPOINT", "ai", "API endpoint for catalog generation"), + ("WIKI_CATALOG_API_KEY", "ai", "API key for catalog generation"), + ("WIKI_CATALOG_REQUEST_TYPE", "ai", "Request type for catalog generation"), + ("WIKI_CONTENT_MODEL", "ai", "AI model used for content generation"), + ("WIKI_CONTENT_ENDPOINT", "ai", "API endpoint for content generation"), + ("WIKI_CONTENT_API_KEY", "ai", "API key for content generation"), + ("WIKI_CONTENT_REQUEST_TYPE", "ai", "Request type for content generation"), + ("WIKI_TRANSLATION_MODEL", "ai", "AI model used for translation"), + ("WIKI_TRANSLATION_ENDPOINT", "ai", "API endpoint for translation"), + ("WIKI_TRANSLATION_API_KEY", "ai", "API key for translation"), + ("WIKI_TRANSLATION_REQUEST_TYPE", "ai", "Request type for translation"), + ("WIKI_LANGUAGES", "ai", "Supported languages (comma-separated)"), + ("WIKI_PARALLEL_COUNT", "ai", "Number of parallel document generation tasks"), + ("WIKI_MAX_OUTPUT_TOKENS", "ai", "Maximum output token count"), + ("WIKI_DOCUMENT_GENERATION_TIMEOUT_MINUTES", "ai", "Document generation timeout (minutes)"), + ("WIKI_TRANSLATION_TIMEOUT_MINUTES", "ai", "Translation timeout (minutes)"), + ("WIKI_TITLE_TRANSLATION_TIMEOUT_MINUTES", "ai", "Title translation timeout (minutes)"), + ("WIKI_README_MAX_LENGTH", "ai", "Maximum README content length"), + ("WIKI_DIRECTORY_TREE_MAX_DEPTH", "ai", "Maximum directory tree depth"), + ("WIKI_MAX_RETRY_ATTEMPTS", "ai", "Maximum retry attempts"), + ("WIKI_RETRY_DELAY_MS", "ai", "Retry delay (milliseconds)"), + ("WIKI_PROMPTS_DIRECTORY", "ai", "Prompt templates directory") ]; /// - /// 初始化系统设置默认值（仅在数据库中不存在对应设置时生效） + /// Default settings for the GitHub App integration. + /// Only seeded if env vars are present (unlike wiki settings which always seed). + /// + public static readonly (string Key, string Category, string Description)[] GitHubAppDefaults = + [ + ("GITHUB_APP_ID", "github", "GitHub App ID"), + ("GITHUB_APP_NAME", "github", "GitHub App name (URL slug)"), + ("GITHUB_APP_PRIVATE_KEY", "github", "GitHub App private key (base64-encoded PEM)") + ]; + + /// + /// Initialize default system settings (only for keys not already in the database). /// public static async Task InitializeDefaultsAsync(IConfiguration configuration, IContext context) { - var existingKeys = await context.SystemSettings + var existingSettings = await context.SystemSettings .Where(s => !s.IsDeleted) - .Select(s => s.Key) .ToListAsync(); + var existingByKey = existingSettings.ToDictionary(s => s.Key); + var settingsToAdd = new List(); + var hasChanges = false; - // 准备 WikiGeneratorOptions 的默认值，方便在没有环境变量时也能写入 + // Prepare WikiGeneratorOptions defaults so values can be written even without env vars var wikiOptionDefaults = new WikiGeneratorOptions(); var wikiSection = configuration.GetSection(WikiGeneratorOptions.SectionName); wikiSection.Bind(wikiOptionDefaults); - // 将 PostConfigure 内的环境变量覆盖也应用到默认选项，确保与运行时一致 + // Apply environment variable overrides to match runtime behavior ApplyEnvironmentOverrides(wikiOptionDefaults, configuration); - // 处理Wiki生成器相关设置 + // Process wiki generator settings foreach (var (key, category, description) in WikiGeneratorDefaults) { - if (!existingKeys.Contains(key)) + if (existingByKey.TryGetValue(key, out var existing)) + { + // Update description if it changed (e.g. translated from Chinese to English) + if (existing.Description != description) + { + existing.Description = description; + hasChanges = true; + } + } + else { var envValue = GetEnvironmentOrConfigurationValue(configuration, key); var fallbackValue = GetOptionDefaultValue(wikiOptionDefaults, key); @@ -84,31 +106,65 @@ public static async Task InitializeDefaultsAsync(IConfiguration configuration, I } } + // Process GitHub App settings (only seed if env var has a value) + foreach (var (key, category, description) in GitHubAppDefaults) + { + if (existingByKey.TryGetValue(key, out var existing)) + { + if (existing.Description != description) + { + existing.Description = description; + hasChanges = true; + } + } + else + { + var envValue = GetEnvironmentOrConfigurationValue(configuration, key); + if (!string.IsNullOrWhiteSpace(envValue)) + { + settingsToAdd.Add(new SystemSetting + { + Id = Guid.NewGuid().ToString(), + Key = key, + Value = envValue, + Description = description, + Category = category, + CreatedAt = DateTime.UtcNow + }); + } + } + } + if (settingsToAdd.Count > 0) { context.SystemSettings.AddRange(settingsToAdd); + hasChanges = true; + } + + if (hasChanges) + { await context.SaveChangesAsync(); } } /// - /// 从环境变量或配置中获取值 + /// Get value from environment variable or configuration. /// private static string? GetEnvironmentOrConfigurationValue(IConfiguration configuration, string key) { - // 优先从环境变量获取 + // Prefer environment variable var envValue = Environment.GetEnvironmentVariable(key); if (!string.IsNullOrWhiteSpace(envValue)) { return envValue; } - // 从配置中获取（支持appsettings.json等） + // Fall back to configuration (appsettings.json etc.) return configuration[key]; } /// - /// 将环境变量中的值应用到 WikiGeneratorOptions，以保持与运行时一致 + /// Apply environment variable values to WikiGeneratorOptions for runtime consistency. /// private static void ApplyEnvironmentOverrides(WikiGeneratorOptions options, IConfiguration configuration) { @@ -123,7 +179,7 @@ private static void ApplyEnvironmentOverrides(WikiGeneratorOptions options, ICon } /// - /// 将系统设置应用到WikiGeneratorOptions + /// Apply system settings to WikiGeneratorOptions. /// public static async Task ApplyToWikiGeneratorOptions(WikiGeneratorOptions options, IAdminSettingsService settingsService) { @@ -138,7 +194,7 @@ public static async Task ApplyToWikiGeneratorOptions(WikiGeneratorOptions option } /// - /// 从 WikiGeneratorOptions 中获取默认值（字符串） + /// Get the default value from WikiGeneratorOptions as a string. /// private static string? GetOptionDefaultValue(WikiGeneratorOptions options, string key) { @@ -172,7 +228,7 @@ public static async Task ApplyToWikiGeneratorOptions(WikiGeneratorOptions option } /// - /// 将单个设置应用到WikiGeneratorOptions + /// Apply a single setting to WikiGeneratorOptions. /// public static void ApplySettingToOption(WikiGeneratorOptions options, string key, string value) { diff --git a/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs b/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs new file mode 100644 index 00000000..273a3f6d --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs @@ -0,0 +1,69 @@ +using OpenDeepWiki.Services.Admin; + +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Thread-safe singleton cache for GitHub App credentials loaded from the database. +/// Checked first by GitHubAppService before falling back to env vars/config. +/// +public class GitHubAppCredentialCache +{ + private readonly object _lock = new(); + private string? _appId; + private string? _privateKeyBase64; + private string? _appName; + + public string? AppId + { + get { lock (_lock) return _appId; } + } + + public string? PrivateKeyBase64 + { + get { lock (_lock) return _privateKeyBase64; } + } + + public string? AppName + { + get { lock (_lock) return _appName; } + } + + /// + /// Update all cached credentials atomically. + /// + public void Update(string? appId, string? privateKeyBase64, string? appName) + { + lock (_lock) + { + _appId = string.IsNullOrWhiteSpace(appId) ? null : appId; + _privateKeyBase64 = string.IsNullOrWhiteSpace(privateKeyBase64) ? null : privateKeyBase64; + _appName = string.IsNullOrWhiteSpace(appName) ? null : appName; + } + } + + /// + /// Clear all cached credentials atomically. + /// + public void Clear() + { + lock (_lock) + { + _appId = null; + _privateKeyBase64 = null; + _appName = null; + } + } + + /// + /// Load credentials from the SystemSettings database table. + /// Called at startup and after saving new credentials via the admin UI. + /// + public async Task LoadFromDbAsync(IAdminSettingsService settingsService) + { + var appId = await settingsService.GetSettingByKeyAsync("GITHUB_APP_ID"); + var privateKey = await settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + var appName = await settingsService.GetSettingByKeyAsync("GITHUB_APP_NAME"); + + Update(appId?.Value, privateKey?.Value, appName?.Value); + } +} diff --git a/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs b/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs new file mode 100644 index 00000000..88e51245 --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs @@ -0,0 +1,255 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using OpenDeepWiki.EFCore; +using OpenDeepWiki.Entities; + +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Implements GitHub App authentication and API operations. +/// Uses JWT signed with the app's private key to generate installation tokens. +/// +public class GitHubAppService : IGitHubAppService +{ + private readonly IConfiguration _configuration; + private readonly GitHubAppCredentialCache _cache; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IContext _context; + private readonly ILogger _logger; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true + }; + + public GitHubAppService( + IConfiguration configuration, + GitHubAppCredentialCache cache, + IHttpClientFactory httpClientFactory, + IContext context, + ILogger logger) + { + _configuration = configuration; + _cache = cache; + _httpClientFactory = httpClientFactory; + _context = context; + _logger = logger; + } + + private string? AppId => + _cache.AppId + ?? _configuration["GitHub:App:Id"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Id") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_ID"); + + private string? PrivateKeyBase64 => + _cache.PrivateKeyBase64 + ?? _configuration["GitHub:App:PrivateKey"] + ?? Environment.GetEnvironmentVariable("GitHub__App__PrivateKey") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_PRIVATE_KEY"); + + public bool IsConfigured => !string.IsNullOrWhiteSpace(AppId) && !string.IsNullOrWhiteSpace(PrivateKeyBase64); + + /// + /// Generate a JWT for authenticating as the GitHub App. + /// JWT is valid for 10 minutes (GitHub maximum). + /// + private string GenerateJwt() + { + if (!IsConfigured) + throw new InvalidOperationException("GitHub App is not configured. Set GitHub:App:Id and GitHub:App:PrivateKey."); + + var pemBytes = Convert.FromBase64String(PrivateKeyBase64!); + var pemContent = System.Text.Encoding.UTF8.GetString(pemBytes); + + var rsa = RSA.Create(); + rsa.ImportFromPem(pemContent.AsSpan()); + + var securityKey = new RsaSecurityKey(rsa); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.RsaSha256); + + var now = DateTime.UtcNow; + var token = new JwtSecurityToken( + issuer: AppId, + claims: new[] + { + new Claim("iat", new DateTimeOffset(now.AddSeconds(-60)).ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) + }, + expires: now.AddMinutes(10), + signingCredentials: credentials + ); + + return new JwtSecurityTokenHandler().WriteToken(token); + } + + private HttpClient CreateClient() + { + var client = _httpClientFactory.CreateClient("GitHubApp"); + client.BaseAddress = new Uri("https://api.github.com"); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OpenDeepWiki", "1.0")); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + return client; + } + + public async Task> ListInstallationsAsync(CancellationToken cancellationToken = default) + { + var jwt = GenerateJwt(); + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + + var response = await client.GetAsync("/app/installations", cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var installations = JsonSerializer.Deserialize>(json, JsonOptions) ?? new(); + + return installations.Select(i => new GitHubInstallationInfo + { + Id = i.Id, + AccountLogin = i.Account?.Login ?? string.Empty, + AccountType = i.Account?.Type ?? string.Empty, + AccountId = i.Account?.Id ?? 0, + AvatarUrl = i.Account?.AvatarUrl + }).ToList(); + } + + public async Task GetInstallationTokenAsync(long installationId, CancellationToken cancellationToken = default) + { + // Check cache in DB first + var installation = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.InstallationId == installationId, cancellationToken); + + if (installation?.CachedAccessToken != null && + installation.TokenExpiresAt.HasValue && + installation.TokenExpiresAt.Value > DateTime.UtcNow.AddMinutes(5)) + { + _logger.LogDebug("Using cached installation token for installation {InstallationId}", installationId); + return installation.CachedAccessToken; + } + + // Generate fresh token + var jwt = GenerateJwt(); + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + + var response = await client.PostAsync( + $"/app/installations/{installationId}/access_tokens", + null, + cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var tokenResponse = JsonSerializer.Deserialize(json, JsonOptions); + + if (tokenResponse?.Token == null) + throw new InvalidOperationException($"Failed to get installation token for installation {installationId}"); + + // Cache the token + if (installation != null) + { + installation.CachedAccessToken = tokenResponse.Token; + installation.TokenExpiresAt = tokenResponse.ExpiresAt; + installation.UpdateTimestamp(); + await _context.SaveChangesAsync(cancellationToken); + } + + _logger.LogInformation("Generated fresh installation token for installation {InstallationId}, expires at {ExpiresAt}", + installationId, tokenResponse.ExpiresAt); + + return tokenResponse.Token; + } + + public async Task ListInstallationReposAsync( + long installationId, int page = 1, int perPage = 30, CancellationToken cancellationToken = default) + { + var token = await GetInstallationTokenAsync(installationId, cancellationToken); + + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + + var response = await client.GetAsync( + $"/installation/repositories?per_page={perPage}&page={page}", + cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var result = JsonSerializer.Deserialize(json, JsonOptions); + + return new GitHubRepoListResult + { + TotalCount = result?.TotalCount ?? 0, + Repositories = result?.Repositories?.Select(r => new GitHubInstallationRepo + { + Id = r.Id, + FullName = r.FullName ?? string.Empty, + Name = r.Name ?? string.Empty, + Owner = r.Owner?.Login ?? string.Empty, + Private = r.Private, + Description = r.Description, + Language = r.Language, + StargazersCount = r.StargazersCount, + ForksCount = r.ForksCount, + DefaultBranch = r.DefaultBranch ?? "main", + CloneUrl = r.CloneUrl ?? string.Empty, + HtmlUrl = r.HtmlUrl ?? string.Empty + }).ToList() ?? new() + }; + } + + // Internal DTOs for GitHub API responses + private class GitHubInstallationResponse + { + public long Id { get; set; } + public GitHubAccountResponse? Account { get; set; } + } + + private class GitHubAccountResponse + { + public long Id { get; set; } + public string? Login { get; set; } + public string? Type { get; set; } + public string? AvatarUrl { get; set; } + } + + private class GitHubTokenResponse + { + public string? Token { get; set; } + public DateTime? ExpiresAt { get; set; } + } + + private class GitHubRepoListResponse + { + public int TotalCount { get; set; } + public List? Repositories { get; set; } + } + + private class GitHubRepoResponse + { + public long Id { get; set; } + public string? FullName { get; set; } + public string? Name { get; set; } + public GitHubOwnerResponse? Owner { get; set; } + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string? DefaultBranch { get; set; } + public string? CloneUrl { get; set; } + public string? HtmlUrl { get; set; } + } + + private class GitHubOwnerResponse + { + public string? Login { get; set; } + } +} diff --git a/web/app/admin/github-import/page.tsx b/web/app/admin/github-import/page.tsx new file mode 100644 index 00000000..c456d4f9 --- /dev/null +++ b/web/app/admin/github-import/page.tsx @@ -0,0 +1,967 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useState } from "react"; +import { useTranslations } from "@/hooks/use-translations"; +import { toast } from "sonner"; +import { + getGitHubStatus, + getGitHubInstallUrl, + getGitHubConfig, + saveGitHubConfig, + resetGitHubConfig, + getInstallationRepos, + batchImportRepos, + getDepartments, + disconnectGitHubInstallation, + GitHubStatus, + GitHubConfig, + GitHubRepo, + GitHubInstallation, + BatchImportResult, + AdminDepartment, +} from "@/lib/admin-api"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Label } from "@/components/ui/label"; +import { + RefreshCw, + ExternalLink, + GitBranch, + Star, + GitFork, + Lock, + Globe, + Loader2, + Building2, + CheckCircle2, + AlertCircle, + Plus, + Download, + Search, + Key, + Save, + HelpCircle, + Info, + Unlink, + RotateCcw, + Upload, +} from "lucide-react"; + +const PAGE_SIZE = 30; +const FETCH_BATCH_SIZE = 100; + +export default function GitHubImportPage() { + const t = useTranslations(); + + // Status + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + + // Selected installation + const [selectedInstallation, setSelectedInstallation] = useState(null); + + // All repos (fetched in full) + const [allRepos, setAllRepos] = useState([]); + const [repoTotalCount, setRepoTotalCount] = useState(0); + const [repoLoading, setRepoLoading] = useState(false); + const [loadProgress, setLoadProgress] = useState(""); + + // Selection + const [selectedRepos, setSelectedRepos] = useState>(new Set()); + + // Filters + const [searchQuery, setSearchQuery] = useState(""); + const [languageFilter, setLanguageFilter] = useState("all"); + + // Client-side pagination + const [page, setPage] = useState(1); + + // Import + const [departments, setDepartments] = useState([]); + const [selectedDepartmentId, setSelectedDepartmentId] = useState(""); + const [languageCode, setLanguageCode] = useState("en"); + const [importing, setImporting] = useState(false); + const [importResult, setImportResult] = useState(null); + + // GitHub App configuration form + const [config, setConfig] = useState(null); + const [configForm, setConfigForm] = useState({ + appId: "", + appName: "", + privateKey: "", + }); + const [saving, setSaving] = useState(false); + const [disconnecting, setDisconnecting] = useState(null); + const [resetting, setResetting] = useState(false); + + const fetchConfig = useCallback(async () => { + try { + const result = await getGitHubConfig(); + setConfig(result); + if (result.appId) setConfigForm((f) => ({ ...f, appId: result.appId || "" })); + if (result.appName) setConfigForm((f) => ({ ...f, appName: result.appName || "" })); + } catch { + // Config endpoint may not be available + } + }, []); + + const handleSaveConfig = async () => { + if (!configForm.appId.trim()) { + toast.error(t("admin.githubImport.validationAppIdRequired")); + return; + } + // When updating existing config, privateKey can be empty (keep existing) + if (!config?.hasPrivateKey && !configForm.privateKey.trim()) { + toast.error(t("admin.githubImport.validationPrivateKeyRequired")); + return; + } + if (configForm.privateKey.trim() && !configForm.privateKey.includes("-----BEGIN")) { + toast.error(t("admin.githubImport.validationPrivateKeyInvalid")); + return; + } + + setSaving(true); + try { + await saveGitHubConfig({ + appId: configForm.appId.trim(), + appName: configForm.appName.trim(), + privateKey: configForm.privateKey, + }); + toast.success(t("admin.githubImport.configSaved")); + setConfigForm((f) => ({ ...f, privateKey: "" })); + // Refresh status and config to show the configured state + await fetchStatus(); + await fetchConfig(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.configSaveFailed") + ); + } finally { + setSaving(false); + } + }; + + const fetchStatus = useCallback(async () => { + setLoading(true); + try { + const result = await getGitHubStatus(); + setStatus(result); + if (result.installations.length > 0 && !selectedInstallation) { + setSelectedInstallation(result.installations[0]); + } + } catch (error) { + toast.error(t("admin.githubImport.fetchStatusFailed")); + } finally { + setLoading(false); + } + }, [t, selectedInstallation]); + + const fetchDepartments = useCallback(async () => { + try { + const result = await getDepartments(); + setDepartments(result); + if (result.length > 0 && !selectedDepartmentId) { + setSelectedDepartmentId(result[0].id); + } + } catch { + // Departments may not be available + } + }, [selectedDepartmentId]); + + // Fetch ALL repos from the installation (paginated API calls in background) + const fetchAllRepos = useCallback(async () => { + if (!selectedInstallation) return; + setRepoLoading(true); + setAllRepos([]); + setRepoTotalCount(0); + setLoadProgress(""); + + try { + // First request to get total count + const firstResult = await getInstallationRepos( + selectedInstallation.installationId, + 1, + FETCH_BATCH_SIZE + ); + const total = firstResult.totalCount; + setRepoTotalCount(total); + + let accumulated = [...firstResult.repositories]; + setAllRepos(accumulated); + setLoadProgress(`${accumulated.length} / ${total}`); + + // Fetch remaining pages + const totalPages = Math.ceil(total / FETCH_BATCH_SIZE); + for (let p = 2; p <= totalPages; p++) { + const result = await getInstallationRepos( + selectedInstallation.installationId, + p, + FETCH_BATCH_SIZE + ); + accumulated = [...accumulated, ...result.repositories]; + setAllRepos(accumulated); + setLoadProgress(`${accumulated.length} / ${total}`); + } + } catch (error) { + toast.error(t("admin.githubImport.fetchReposFailed")); + } finally { + setRepoLoading(false); + setLoadProgress(""); + } + }, [selectedInstallation, t]); + + useEffect(() => { + fetchStatus(); + fetchDepartments(); + fetchConfig(); + }, []); + + useEffect(() => { + if (selectedInstallation) { + setSelectedRepos(new Set()); + setImportResult(null); + setSearchQuery(""); + setLanguageFilter("all"); + setPage(1); + fetchAllRepos(); + } + }, [selectedInstallation]); + + const handleConnectNew = async () => { + try { + const { url } = await getGitHubInstallUrl(); + window.location.href = url; + } catch (error) { + toast.error("Failed to get install URL"); + } + }; + + const handleDisconnect = async (inst: GitHubInstallation) => { + const confirmed = window.confirm( + t("admin.githubImport.disconnectConfirm").replace("{org}", inst.accountLogin) + ); + if (!confirmed) return; + + setDisconnecting(inst.id); + try { + await disconnectGitHubInstallation(inst.id); + toast.success(t("admin.githubImport.disconnectSuccess")); + if (selectedInstallation?.id === inst.id) { + setSelectedInstallation(null); + setAllRepos([]); + } + await fetchStatus(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.disconnectFailed") + ); + } finally { + setDisconnecting(null); + } + }; + + const handleResetConfig = async () => { + const confirmed = window.confirm( + t("admin.githubImport.resetConfigConfirm") + ); + if (!confirmed) return; + + setResetting(true); + try { + await resetGitHubConfig(); + toast.success(t("admin.githubImport.resetConfigSuccess")); + setSelectedInstallation(null); + setAllRepos([]); + setConfig(null); + setConfigForm({ appId: "", appName: "", privateKey: "" }); + await fetchStatus(); + await fetchConfig(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.resetConfigFailed") + ); + } finally { + setResetting(false); + } + }; + + const toggleRepo = (fullName: string) => { + setSelectedRepos((prev) => { + const next = new Set(prev); + if (next.has(fullName)) { + next.delete(fullName); + } else { + next.add(fullName); + } + return next; + }); + }; + + // Apply filters across ALL repos + const filteredRepos = useMemo(() => { + return allRepos.filter((r) => { + if (searchQuery) { + const q = searchQuery.toLowerCase(); + const matchesName = r.fullName.toLowerCase().includes(q); + const matchesDesc = r.description?.toLowerCase().includes(q); + if (!matchesName && !matchesDesc) return false; + } + if (languageFilter !== "all" && (r.language || "").toLowerCase() !== languageFilter.toLowerCase()) { + return false; + } + return true; + }); + }, [allRepos, searchQuery, languageFilter]); + + const importableFiltered = useMemo( + () => filteredRepos.filter((r) => !r.alreadyImported), + [filteredRepos] + ); + + // Client-side pagination of filtered results + const totalFilteredPages = Math.ceil(filteredRepos.length / PAGE_SIZE); + const pagedRepos = useMemo( + () => filteredRepos.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE), + [filteredRepos, page] + ); + + // Collect unique languages from ALL repos for the filter dropdown + const repoLanguages = useMemo( + () => Array.from(new Set(allRepos.map((r) => r.language).filter(Boolean) as string[])).sort(), + [allRepos] + ); + + // Reset page when filters change + useEffect(() => { + setPage(1); + }, [searchQuery, languageFilter]); + + const toggleSelectAll = () => { + if (selectedRepos.size === importableFiltered.length && importableFiltered.length > 0) { + setSelectedRepos(new Set()); + } else { + setSelectedRepos(new Set(importableFiltered.map((r) => r.fullName))); + } + }; + + const handleImport = async () => { + if (!selectedInstallation || selectedRepos.size === 0 || !selectedDepartmentId) return; + + setImporting(true); + setImportResult(null); + try { + const selectedRepoData = allRepos + .filter((r) => selectedRepos.has(r.fullName)) + .map((r) => ({ + fullName: r.fullName, + name: r.name, + owner: r.owner, + cloneUrl: r.cloneUrl, + defaultBranch: r.defaultBranch, + private: r.private, + language: r.language, + stargazersCount: r.stargazersCount, + forksCount: r.forksCount, + })); + + const result = await batchImportRepos({ + installationId: selectedInstallation.installationId, + departmentId: selectedDepartmentId, + languageCode, + repos: selectedRepoData, + }); + + setImportResult(result); + setSelectedRepos(new Set()); + toast.success( + t("admin.githubImport.importSuccess") + .replace("{imported}", result.imported.toString()) + .replace("{skipped}", result.skipped.toString()) + ); + + // Refresh repo list to update "already imported" flags + fetchAllRepos(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.importFailed") + ); + } finally { + setImporting(false); + } + }; + + if (loading) { + return ( +
+ +
+ ); + } + + return ( +
+ {/* Header */} +
+
+

{t("admin.githubImport.title")}

+

+ {t("admin.githubImport.description")} +

+
+ +
+ + {/* Status Card */} + + + {t("admin.githubImport.connectionStatus")} + + + {!status?.configured ? ( +
+ + {t("admin.githubImport.credentialSourceNone")} +
+ ) : ( +
+
+ + {t("admin.githubImport.configured")} + {config && ( + + ({config.source === "database" + ? t("admin.githubImport.credentialSourceDb") + : t("admin.githubImport.credentialSourceEnv")}) + + )} +
+ + {/* Connected Organizations */} + {status.installations.length > 0 && ( +
+

+ {t("admin.githubImport.connectedOrgs")} +

+
+ {status.installations.map((inst) => ( +
setSelectedInstallation(inst)} + > +
+ {inst.avatarUrl && ( + {inst.accountLogin} + )} +
+ {inst.accountLogin} + + {inst.accountType} + +
+
+
+ {inst.departmentName && ( + + + {inst.departmentName} + + )} + +
+
+ ))} +
+
+ )} + +
+ + +
+
+ )} + + + + {/* Repository Selection */} + {selectedInstallation && ( + + + + {t("admin.githubImport.importFrom").replace( + "{org}", + selectedInstallation.accountLogin + )} + + + {repoLoading && loadProgress + ? `${t("admin.githubImport.loadingRepos")} ${loadProgress}...` + : t("admin.githubImport.totalRepos").replace( + "{count}", + repoTotalCount.toString() + )} + + + + {/* Import Options */} +
+
+ + +
+ +
+ + +
+
+ + {/* Search and Filter */} +
+
+ + setSearchQuery(e.target.value)} + className="pl-9" + /> +
+ +
+ + {/* Select All */} +
+ 0 && + selectedRepos.size === importableFiltered.length + } + onCheckedChange={toggleSelectAll} + /> + + {t("admin.githubImport.selectAll")} + {searchQuery || languageFilter !== "all" + ? ` (${filteredRepos.length} ${t("admin.githubImport.matching")})` + : ""} + {selectedRepos.size > 0 && + ` - ${selectedRepos.size} ${t("admin.githubImport.selected")}`} + +
+ + {/* Repository List */} + {repoLoading && allRepos.length === 0 ? ( +
+ +
+ ) : ( +
+ {pagedRepos.length === 0 && searchQuery && ( +

+ {t("admin.githubImport.noResults")} +

+ )} + {pagedRepos.map((repo) => ( +
+ toggleRepo(repo.fullName)} + /> +
+
+ + {repo.fullName} + {repo.private ? ( + + ) : ( + + )} + {repo.alreadyImported && ( + + {t("admin.githubImport.alreadyImported")} + + )} +
+ {repo.description && ( +

+ {repo.description} +

+ )} +
+
+ {repo.language && ( + + {repo.language} + + )} + + + {repo.stargazersCount} + + + + {repo.forksCount} + + e.stopPropagation()} + > + + +
+
+ ))} +
+ )} + + {/* Pagination */} + {totalFilteredPages > 1 && ( +
+ + + {page} / {totalFilteredPages} + + +
+ )} + + {/* Import Button */} +
+ + {selectedRepos.size > 0 + ? t("admin.githubImport.readyToImport").replace( + "{count}", + selectedRepos.size.toString() + ) + : t("admin.githubImport.selectReposPrompt")} + + +
+ + {/* Import Result */} + {importResult && ( + + +
+ +
+

+ {t("admin.githubImport.importComplete")} +

+

+ {importResult.imported} {t("admin.githubImport.imported")},{" "} + {importResult.skipped} {t("admin.githubImport.skipped")} +

+ {importResult.skippedRepos.length > 0 && ( +

+ {t("admin.githubImport.skippedList")}:{" "} + {importResult.skippedRepos.join(", ")} +

+ )} +
+
+ + + )} + + + )} + + {/* Configuration Form -- shown when not configured */} + {status && !status.configured && ( + + + + + {status.configured + ? t("admin.githubImport.updateConfig") + : t("admin.githubImport.configTitle")} + + + {status.configured + ? t("admin.githubImport.updateConfigDescription") + : t("admin.githubImport.configDescription")} + + + + {config?.source === "environment" && !status.configured && ( +
+ + {t("admin.githubImport.overrideEnvWarning")} +
+ )} + +
+
+ + + setConfigForm((f) => ({ ...f, appId: e.target.value })) + } + /> +

+ {t("admin.githubImport.appIdHelp")} +

+
+ +
+ + + setConfigForm((f) => ({ ...f, appName: e.target.value })) + } + /> +

+ {t("admin.githubImport.appNameHelp")} +

+
+ +
+
+ + + { + const file = e.target.files?.[0]; + if (!file) return; + const reader = new FileReader(); + reader.onload = (ev) => { + const content = ev.target?.result as string; + setConfigForm((f) => ({ ...f, privateKey: content })); + }; + reader.readAsText(file); + e.target.value = ""; + }} + /> +
+