diff --git a/src/OpenDeepWiki.EFCore/MasterDbContext.cs b/src/OpenDeepWiki.EFCore/MasterDbContext.cs index e9f21013..8ee153a7 100644 --- a/src/OpenDeepWiki.EFCore/MasterDbContext.cs +++ b/src/OpenDeepWiki.EFCore/MasterDbContext.cs @@ -47,6 +47,7 @@ public interface IContext : IDisposable DbSet McpProviders { get; set; } DbSet McpUsageLogs { get; set; } DbSet McpDailyStatistics { get; set; } + DbSet GitHubAppInstallations { get; set; } Task SaveChangesAsync(CancellationToken cancellationToken = default); } @@ -97,6 +98,7 @@ protected MasterDbContext(DbContextOptions options) public DbSet McpProviders { get; set; } = null!; public DbSet McpUsageLogs { get; set; } = null!; public DbSet McpDailyStatistics { get; set; } = null!; + public DbSet GitHubAppInstallations { get; set; } = null!; protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -382,5 +384,15 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) // 日期索引 builder.HasIndex(s => s.Date); }); + + // GitHubAppInstallation 表配置 + modelBuilder.Entity(entity => + { + entity.HasIndex(e => e.InstallationId).IsUnique(); + entity.HasOne(e => e.Department) + .WithMany() + .HasForeignKey(e => e.DepartmentId) + .OnDelete(DeleteBehavior.SetNull); + }); } } diff --git a/src/OpenDeepWiki.Entities/GitHub/GitHubAppInstallation.cs b/src/OpenDeepWiki.Entities/GitHub/GitHubAppInstallation.cs new file mode 100644 index 00000000..030d058b --- /dev/null +++ b/src/OpenDeepWiki.Entities/GitHub/GitHubAppInstallation.cs @@ -0,0 +1,62 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace OpenDeepWiki.Entities; + +/// +/// Represents a GitHub App installation on an organization or user account. +/// +public class GitHubAppInstallation : AggregateRoot +{ + /// + /// GitHub's installation ID. + /// + public long InstallationId { get; set; } + + /// + /// Organization or user login name (e.g., "keboola"). + /// + [Required] + [StringLength(100)] + public string AccountLogin { get; set; } = string.Empty; + + /// + /// Account type: "Organization" or "User". + /// + [Required] + [StringLength(20)] + public string AccountType { get; set; } = "Organization"; + + /// + /// GitHub account ID. + /// + public long AccountId { get; set; } + + /// + /// Avatar URL of the account. + /// + [StringLength(500)] + public string? AvatarUrl { get; set; } + + /// + /// Optional link to a Department for auto-assignment. + /// + [StringLength(36)] + public string? DepartmentId { get; set; } + + /// + /// Cached installation access token (short-lived, ~1 hour). + /// + public string? CachedAccessToken { get; set; } + + /// + /// When the cached access token expires. + /// + public DateTime? TokenExpiresAt { get; set; } + + /// + /// Navigation property to the linked department. + /// + [ForeignKey("DepartmentId")] + public virtual Department? Department { get; set; } +} diff --git a/src/OpenDeepWiki/Endpoints/Admin/AdminEndpoints.cs b/src/OpenDeepWiki/Endpoints/Admin/AdminEndpoints.cs index d02e1868..f555da0b 100644 --- a/src/OpenDeepWiki/Endpoints/Admin/AdminEndpoints.cs +++ b/src/OpenDeepWiki/Endpoints/Admin/AdminEndpoints.cs @@ -21,6 +21,7 @@ public static IEndpointRouteBuilder MapAdminEndpoints(this IEndpointRouteBuilder adminGroup.MapAdminSettingsEndpoints(); adminGroup.MapAdminChatAssistantEndpoints(); adminGroup.MapAdminMcpProviderEndpoints(); + adminGroup.MapAdminGitHubImportEndpoints(); return app; } diff --git a/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs b/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs new file mode 100644 index 00000000..23c6363d --- /dev/null +++ b/src/OpenDeepWiki/Endpoints/Admin/AdminGitHubImportEndpoints.cs @@ -0,0 +1,148 @@ +using OpenDeepWiki.Models.Admin; +using OpenDeepWiki.Services.Admin; +using OpenDeepWiki.Services.Auth; +using OpenDeepWiki.Services.GitHub; + +namespace OpenDeepWiki.Endpoints.Admin; + +public static class AdminGitHubImportEndpoints +{ + public static RouteGroupBuilder MapAdminGitHubImportEndpoints(this RouteGroupBuilder group) + { + var github = group.MapGroup("/github") + .WithTags("Admin - GitHub Import"); + + github.MapGet("/status", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + var result = await service.GetStatusAsync(cancellationToken); + return Results.Ok(new { success = true, data = result }); + }); + + github.MapGet("/config", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + var result = await service.GetGitHubConfigAsync(cancellationToken); + return Results.Ok(new { success = true, data = result }); + }); + + github.MapPost("/config", async ( + SaveGitHubConfigRequest request, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.SaveGitHubConfigAsync(request, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapDelete("/config", async ( + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + await service.ResetGitHubConfigAsync(cancellationToken); + return Results.Ok(new { success = true }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapGet("/install-url", async ( + Microsoft.Extensions.Configuration.IConfiguration configuration, + GitHubAppCredentialCache cache) => + { + var appName = cache.AppName + ?? configuration["GitHub:App:Name"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Name") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_NAME") + ?? "deepwiki-keboola"; + var url = $"https://github.com/apps/{appName}/installations/new"; + return Results.Ok(new { success = true, data = new { url, appName } }); + }); + + github.MapPost("/installations", async ( + StoreInstallationRequest request, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.StoreInstallationAsync(request.InstallationId, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapDelete("/installations/{installationId}", async ( + string installationId, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + await service.DisconnectInstallationAsync(installationId, cancellationToken); + return Results.Ok(new { success = true }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapGet("/installations/{installationId:long}/repos", async ( + long installationId, + int page, + int perPage, + IAdminGitHubImportService service, + CancellationToken cancellationToken) => + { + try + { + var result = await service.ListInstallationReposAsync(installationId, page, perPage, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (Exception ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + github.MapPost("/batch-import", async ( + BatchImportRequest request, + IAdminGitHubImportService service, + IUserContext userContext, + CancellationToken cancellationToken) => + { + try + { + var userId = userContext.UserId; + if (string.IsNullOrEmpty(userId)) + return Results.Unauthorized(); + + var result = await service.BatchImportAsync(request, userId, cancellationToken); + return Results.Ok(new { success = true, data = result }); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { success = false, message = ex.Message }); + } + }); + + return group; + } +} diff --git a/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs b/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs new file mode 100644 index 00000000..0d2fd0d3 --- /dev/null +++ b/src/OpenDeepWiki/Models/Admin/GitHubImportModels.cs @@ -0,0 +1,100 @@ +namespace OpenDeepWiki.Models.Admin; + +public class GitHubStatusResponse +{ + public bool Configured { get; set; } + public string? AppName { get; set; } + public List Installations { get; set; } = new(); +} + +public class GitHubInstallationDto +{ + public string? Id { get; set; } + public long InstallationId { get; set; } + public string AccountLogin { get; set; } = string.Empty; + public string AccountType { get; set; } = string.Empty; + public long AccountId { get; set; } + public string? AvatarUrl { get; set; } + public string? DepartmentId { get; set; } + public string? DepartmentName { get; set; } + public DateTime CreatedAt { get; set; } +} + +public class StoreInstallationRequest +{ + public long InstallationId { get; set; } +} + +public class GitHubRepoDto +{ + public long Id { get; set; } + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string DefaultBranch { get; set; } = "main"; + public string CloneUrl { get; set; } = string.Empty; + public string HtmlUrl { get; set; } = string.Empty; + public bool AlreadyImported { get; set; } +} + +public class GitHubRepoListDto +{ + public int TotalCount { get; set; } + public List Repositories { get; set; } = new(); + public int Page { get; set; } + public int PerPage { get; set; } +} + +public class BatchImportRequest +{ + public long InstallationId { get; set; } + public string DepartmentId { get; set; } = string.Empty; + public string LanguageCode { get; set; } = "en"; + public List Repos { get; set; } = new(); +} + +public class BatchImportRepo +{ + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public string CloneUrl { get; set; } = string.Empty; + public string DefaultBranch { get; set; } = "main"; + public bool Private { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } +} + +public class BatchImportResult +{ + public int TotalRequested { get; set; } + public int Imported { get; set; } + public int Skipped { get; set; } + public List SkippedRepos { get; set; } = new(); + public List ImportedRepos { get; set; } = new(); +} + +public class SaveGitHubConfigRequest +{ + public string AppId { get; set; } = string.Empty; + public string AppName { get; set; } = string.Empty; + public string PrivateKey { get; set; } = string.Empty; +} + +public class GitHubConfigResponse +{ + public bool HasAppId { get; set; } + public bool HasPrivateKey { get; set; } + public string? AppId { get; set; } + public string? AppName { get; set; } + /// + /// Credential source: "database", "environment", or "none". + /// + public string Source { get; set; } = "none"; +} diff --git a/src/OpenDeepWiki/Program.cs b/src/OpenDeepWiki/Program.cs index 5bf2e68a..c0d034ea 100644 --- a/src/OpenDeepWiki/Program.cs +++ b/src/OpenDeepWiki/Program.cs @@ -21,6 +21,7 @@ using OpenDeepWiki.Services.Repositories; using OpenDeepWiki.Services.Translation; using OpenDeepWiki.Services.Mcp; +using OpenDeepWiki.Services.GitHub; using OpenDeepWiki.Services.UserProfile; using OpenDeepWiki.Services.Wiki; using Scalar.AspNetCore; @@ -288,6 +289,11 @@ // 注册处理日志服务（使用 Singleton，因为它内部使用 IServiceScopeFactory 创建独立 scope） builder.Services.AddSingleton(); + // Register GitHub App services + builder.Services.AddSingleton(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + // 注册管理端服务 builder.Services.AddScoped(); builder.Services.AddScoped(); @@ -445,6 +451,22 @@ await DbInitializer.InitializeAsync(scope.ServiceProvider); } + // Load GitHub App credentials from DB into cache + using (var scope = app.Services.CreateScope()) + { + var credCache = scope.ServiceProvider.GetRequiredService(); + var settingsService = scope.ServiceProvider.GetRequiredService(); + try + { + await credCache.LoadFromDbAsync(settingsService); + } + catch (Exception ex) + { + var logger = scope.ServiceProvider.GetRequiredService>(); + logger.LogWarning(ex, "Failed to load GitHub App credentials from database at startup"); + } + } + app.Run(); } catch (Exception ex) diff --git a/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs b/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs new file mode 100644 index 00000000..e0f9a7fa --- /dev/null +++ b/src/OpenDeepWiki/Services/Admin/AdminGitHubImportService.cs @@ -0,0 +1,504 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Security.Claims; +using System.Security.Cryptography; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using OpenDeepWiki.EFCore; +using OpenDeepWiki.Entities; +using OpenDeepWiki.Models.Admin; +using OpenDeepWiki.Services.GitHub; + +namespace OpenDeepWiki.Services.Admin; + +public class AdminGitHubImportService : IAdminGitHubImportService +{ + private readonly IContext _context; + private readonly IGitHubAppService _gitHubAppService; + private readonly IAdminSettingsService _settingsService; + private readonly IHttpClientFactory _httpClientFactory; + private readonly GitHubAppCredentialCache _cache; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + + public AdminGitHubImportService( + IContext context, + IGitHubAppService gitHubAppService, + IAdminSettingsService settingsService, + IHttpClientFactory httpClientFactory, + GitHubAppCredentialCache cache, + IConfiguration configuration, + ILogger logger) + { + _context = context; + _gitHubAppService = gitHubAppService; + _settingsService = settingsService; + _httpClientFactory = httpClientFactory; + _cache = cache; + _configuration = configuration; + _logger = logger; + } + + public async Task GetStatusAsync(CancellationToken cancellationToken = default) + { + var response = new GitHubStatusResponse + { + Configured = _gitHubAppService.IsConfigured + }; + + if (!response.Configured) + return response; + + // Load stored installations from DB + var installations = await _context.GitHubAppInstallations + .Where(i => !i.IsDeleted) + .ToListAsync(cancellationToken); + + // Load department names for linked installations + var departmentIds = installations + .Where(i => i.DepartmentId != null) + .Select(i => i.DepartmentId!) + .Distinct() + .ToList(); + + var departments = departmentIds.Count > 0 + ? await _context.Departments + .Where(d => departmentIds.Contains(d.Id)) + .ToDictionaryAsync(d => d.Id, d => d.Name, cancellationToken) + : new Dictionary(); + + response.Installations = installations.Select(i => new GitHubInstallationDto + { + Id = i.Id, + InstallationId = i.InstallationId, + AccountLogin = i.AccountLogin, + AccountType = i.AccountType, + AccountId = i.AccountId, + AvatarUrl = i.AvatarUrl, + DepartmentId = i.DepartmentId, + DepartmentName = i.DepartmentId != null && departments.ContainsKey(i.DepartmentId) + ? departments[i.DepartmentId] + : null, + CreatedAt = i.CreatedAt + }).ToList(); + + return response; + } + + public async Task StoreInstallationAsync(long installationId, CancellationToken cancellationToken = default) + { + // Check if already stored (including soft-deleted, to avoid UNIQUE constraint violation) + var existing = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.InstallationId == installationId, cancellationToken); + + if (existing != null) + { + // Re-activate if soft-deleted + if (existing.IsDeleted) + { + existing.IsDeleted = false; + existing.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + _logger.LogInformation("Re-activated GitHub App installation {InstallationId} for {AccountLogin}", + installationId, existing.AccountLogin); + } + + return new GitHubInstallationDto + { + Id = existing.Id, + InstallationId = existing.InstallationId, + AccountLogin = existing.AccountLogin, + AccountType = existing.AccountType, + AccountId = existing.AccountId, + AvatarUrl = existing.AvatarUrl, + DepartmentId = existing.DepartmentId, + CreatedAt = existing.CreatedAt + }; + } + + // Fetch installation details from GitHub + var allInstallations = await _gitHubAppService.ListInstallationsAsync(cancellationToken); + var info = allInstallations.FirstOrDefault(i => i.Id == installationId); + + if (info == null) + throw new InvalidOperationException($"Installation {installationId} not found on GitHub. Ensure the app is installed."); + + var entity = new GitHubAppInstallation + { + Id = Guid.NewGuid().ToString(), + InstallationId = installationId, + AccountLogin = info.AccountLogin, + AccountType = info.AccountType, + AccountId = info.AccountId, + AvatarUrl = info.AvatarUrl, + }; + + _context.GitHubAppInstallations.Add(entity); + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("Stored GitHub App installation {InstallationId} for {AccountLogin}", + installationId, info.AccountLogin); + + return new GitHubInstallationDto + { + Id = entity.Id, + InstallationId = entity.InstallationId, + AccountLogin = entity.AccountLogin, + AccountType = entity.AccountType, + AccountId = entity.AccountId, + AvatarUrl = entity.AvatarUrl, + CreatedAt = entity.CreatedAt + }; + } + + public async Task DisconnectInstallationAsync(string installationId, CancellationToken cancellationToken = default) + { + var entity = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.Id == installationId && !i.IsDeleted, cancellationToken); + + if (entity == null) + throw new InvalidOperationException($"Installation {installationId} not found."); + + entity.IsDeleted = true; + entity.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation("Disconnected GitHub App installation {InstallationId} for {AccountLogin}", + entity.InstallationId, entity.AccountLogin); + } + + public async Task ListInstallationReposAsync( + long installationId, int page, int perPage, CancellationToken cancellationToken = default) + { + var result = await _gitHubAppService.ListInstallationReposAsync(installationId, page, perPage, cancellationToken); + + // Check which repos are already imported + var repoUrls = result.Repositories.Select(r => r.CloneUrl).ToList(); + var existingUrls = await _context.Repositories + .Where(r => repoUrls.Contains(r.GitUrl) && !r.IsDeleted) + .Select(r => r.GitUrl) + .ToListAsync(cancellationToken); + + var existingUrlSet = new HashSet(existingUrls, StringComparer.OrdinalIgnoreCase); + + return new GitHubRepoListDto + { + TotalCount = result.TotalCount, + Page = page, + PerPage = perPage, + Repositories = result.Repositories.Select(r => new GitHubRepoDto + { + Id = r.Id, + FullName = r.FullName, + Name = r.Name, + Owner = r.Owner, + Private = r.Private, + Description = r.Description, + Language = r.Language, + StargazersCount = r.StargazersCount, + ForksCount = r.ForksCount, + DefaultBranch = r.DefaultBranch, + CloneUrl = r.CloneUrl, + HtmlUrl = r.HtmlUrl, + AlreadyImported = existingUrlSet.Contains(r.CloneUrl) + }).ToList() + }; + } + + public async Task BatchImportAsync( + BatchImportRequest request, string ownerUserId, CancellationToken cancellationToken = default) + { + var result = new BatchImportResult + { + TotalRequested = request.Repos.Count + }; + + // Verify department exists + var department = await _context.Departments + .FirstOrDefaultAsync(d => d.Id == request.DepartmentId && !d.IsDeleted, cancellationToken); + + if (department == null) + throw new InvalidOperationException($"Department {request.DepartmentId} not found."); + + // Get existing repos by clone URL to avoid duplicates + var requestUrls = request.Repos.Select(r => r.CloneUrl).ToList(); + var existingRepos = await _context.Repositories + .Where(r => requestUrls.Contains(r.GitUrl) && !r.IsDeleted) + .ToDictionaryAsync(r => r.GitUrl, r => r, StringComparer.OrdinalIgnoreCase, cancellationToken); + + foreach (var repo in request.Repos) + { + if (existingRepos.ContainsKey(repo.CloneUrl)) + { + result.Skipped++; + result.SkippedRepos.Add(repo.FullName); + _logger.LogDebug("Skipping {Repo}: already imported", repo.FullName); + continue; + } + + // Create Repository entity + var repoEntity = new Repository + { + Id = Guid.NewGuid().ToString(), + OwnerUserId = ownerUserId, + GitUrl = repo.CloneUrl, + RepoName = repo.Name, + OrgName = repo.Owner, + IsPublic = !repo.Private, + Status = RepositoryStatus.Pending, + PrimaryLanguage = repo.Language, + StarCount = repo.StargazersCount, + ForkCount = repo.ForksCount + }; + + _context.Repositories.Add(repoEntity); + + // Create default branch + var branch = new RepositoryBranch + { + Id = Guid.NewGuid().ToString(), + RepositoryId = repoEntity.Id, + BranchName = repo.DefaultBranch + }; + + _context.RepositoryBranches.Add(branch); + + // Create branch language + var branchLanguage = new BranchLanguage + { + Id = Guid.NewGuid().ToString(), + RepositoryBranchId = branch.Id, + LanguageCode = request.LanguageCode, + IsDefault = true + }; + + _context.BranchLanguages.Add(branchLanguage); + + // Assign to department + var assignment = new RepositoryAssignment + { + Id = Guid.NewGuid().ToString(), + RepositoryId = repoEntity.Id, + DepartmentId = request.DepartmentId, + AssigneeUserId = ownerUserId + }; + + _context.RepositoryAssignments.Add(assignment); + + result.Imported++; + result.ImportedRepos.Add(repo.FullName); + } + + await _context.SaveChangesAsync(cancellationToken); + + _logger.LogInformation( + "Batch import complete: {Imported} imported, {Skipped} skipped out of {Total} requested", + result.Imported, result.Skipped, result.TotalRequested); + + return result; + } + + public async Task GetGitHubConfigAsync(CancellationToken cancellationToken = default) + { + var dbAppId = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_ID"); + var dbPrivateKey = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + var dbAppName = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_NAME"); + + var hasDbAppId = !string.IsNullOrWhiteSpace(dbAppId?.Value); + var hasDbPrivateKey = !string.IsNullOrWhiteSpace(dbPrivateKey?.Value); + + if (hasDbAppId || hasDbPrivateKey) + { + return new GitHubConfigResponse + { + HasAppId = hasDbAppId, + HasPrivateKey = hasDbPrivateKey, + AppId = dbAppId?.Value, + AppName = dbAppName?.Value, + Source = "database" + }; + } + + // Check environment variables / configuration + var envAppId = _configuration["GitHub:App:Id"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Id") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_ID"); + var envPrivateKey = _configuration["GitHub:App:PrivateKey"] + ?? Environment.GetEnvironmentVariable("GitHub__App__PrivateKey") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_PRIVATE_KEY"); + var envAppName = _configuration["GitHub:App:Name"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Name") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_NAME"); + + if (!string.IsNullOrWhiteSpace(envAppId) || !string.IsNullOrWhiteSpace(envPrivateKey)) + { + return new GitHubConfigResponse + { + HasAppId = !string.IsNullOrWhiteSpace(envAppId), + HasPrivateKey = !string.IsNullOrWhiteSpace(envPrivateKey), + AppId = envAppId, + AppName = envAppName, + Source = "environment" + }; + } + + return new GitHubConfigResponse { Source = "none" }; + } + + public async Task SaveGitHubConfigAsync(SaveGitHubConfigRequest request, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(request.AppId)) + throw new InvalidOperationException("App ID is required."); + + string privateKeyBase64; + + if (!string.IsNullOrWhiteSpace(request.PrivateKey)) + { + // New private key provided -- validate and encode + if (!request.PrivateKey.Contains("-----BEGIN")) + throw new InvalidOperationException("Private Key must be a valid PEM file (should contain '-----BEGIN')."); + + privateKeyBase64 = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(request.PrivateKey)); + } + else + { + // No new private key -- use existing from DB or cache + var existingKey = _cache.PrivateKeyBase64; + if (string.IsNullOrWhiteSpace(existingKey)) + { + var dbKey = await _settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + existingKey = dbKey?.Value; + } + if (string.IsNullOrWhiteSpace(existingKey)) + throw new InvalidOperationException("Private Key is required (no existing key found)."); + + privateKeyBase64 = existingKey; + } + + // Validate credentials by attempting to generate JWT and call GitHub API + await ValidateGitHubCredentialsAsync(request.AppId, privateKeyBase64, cancellationToken); + + // Upsert settings into SystemSettings with category "github" + await UpsertSettingAsync("GITHUB_APP_ID", request.AppId, "github", "GitHub App ID"); + await UpsertSettingAsync("GITHUB_APP_PRIVATE_KEY", privateKeyBase64, "github", "GitHub App private key (base64-encoded PEM)"); + await UpsertSettingAsync("GITHUB_APP_NAME", request.AppName, "github", "GitHub App name (URL slug)"); + await _context.SaveChangesAsync(cancellationToken); + + // Update the in-memory cache so IsConfigured returns true immediately + _cache.Update(request.AppId, privateKeyBase64, request.AppName); + + _logger.LogInformation("GitHub App credentials saved to database for App ID {AppId}", request.AppId); + + return new GitHubConfigResponse + { + HasAppId = true, + HasPrivateKey = true, + AppId = request.AppId, + AppName = request.AppName, + Source = "database" + }; + } + + public async Task ResetGitHubConfigAsync(CancellationToken cancellationToken = default) + { + // Soft-delete all GitHub config settings from SystemSettings + var githubSettings = await _context.SystemSettings + .Where(s => s.Category == "github" && !s.IsDeleted) + .ToListAsync(cancellationToken); + + foreach (var setting in githubSettings) + { + setting.IsDeleted = true; + setting.UpdatedAt = DateTime.UtcNow; + } + + await _context.SaveChangesAsync(cancellationToken); + + // Clear the in-memory cache + _cache.Clear(); + + _logger.LogInformation("GitHub App configuration has been reset"); + } + + private async Task ValidateGitHubCredentialsAsync(string appId, string privateKeyBase64, CancellationToken cancellationToken) + { + try + { + var pemBytes = Convert.FromBase64String(privateKeyBase64); + var pemContent = System.Text.Encoding.UTF8.GetString(pemBytes); + + var rsa = RSA.Create(); + rsa.ImportFromPem(pemContent.AsSpan()); + + var securityKey = new RsaSecurityKey(rsa); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.RsaSha256); + + var now = DateTime.UtcNow; + var token = new JwtSecurityToken( + issuer: appId, + claims: new[] + { + new Claim("iat", new DateTimeOffset(now.AddSeconds(-60)).ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) + }, + expires: now.AddMinutes(10), + signingCredentials: credentials + ); + + var jwt = new JwtSecurityTokenHandler().WriteToken(token); + + // Call GitHub API to verify the credentials + using var client = _httpClientFactory.CreateClient("GitHubApp"); + client.BaseAddress = new Uri("https://api.github.com"); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OpenDeepWiki", "1.0")); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + client.Timeout = TimeSpan.FromSeconds(15); + + var response = await client.GetAsync("/app", cancellationToken); + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"GitHub API returned {(int)response.StatusCode}: {body}"); + } + } + catch (InvalidOperationException) + { + throw; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to validate GitHub App credentials: {ex.Message}", ex); + } + } + + private async Task UpsertSettingAsync(string key, string? value, string category, string description) + { + // Check for any existing row (including soft-deleted) to avoid UNIQUE constraint violation + var existing = await _context.SystemSettings + .FirstOrDefaultAsync(s => s.Key == key); + + if (existing != null) + { + existing.Value = value; + existing.Description = description; + existing.Category = category; + existing.IsDeleted = false; + existing.UpdatedAt = DateTime.UtcNow; + } + else + { + _context.SystemSettings.Add(new SystemSetting + { + Id = Guid.NewGuid().ToString(), + Key = key, + Value = value, + Description = description, + Category = category, + CreatedAt = DateTime.UtcNow + }); + } + } +} diff --git a/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs b/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs new file mode 100644 index 00000000..fb8dc5a0 --- /dev/null +++ b/src/OpenDeepWiki/Services/Admin/IAdminGitHubImportService.cs @@ -0,0 +1,15 @@ +using OpenDeepWiki.Models.Admin; + +namespace OpenDeepWiki.Services.Admin; + +public interface IAdminGitHubImportService +{ + Task GetStatusAsync(CancellationToken cancellationToken = default); + Task StoreInstallationAsync(long installationId, CancellationToken cancellationToken = default); + Task ListInstallationReposAsync(long installationId, int page, int perPage, CancellationToken cancellationToken = default); + Task BatchImportAsync(BatchImportRequest request, string ownerUserId, CancellationToken cancellationToken = default); + Task GetGitHubConfigAsync(CancellationToken cancellationToken = default); + Task SaveGitHubConfigAsync(SaveGitHubConfigRequest request, CancellationToken cancellationToken = default); + Task DisconnectInstallationAsync(string installationId, CancellationToken cancellationToken = default); + Task ResetGitHubConfigAsync(CancellationToken cancellationToken = default); +} diff --git a/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs b/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs new file mode 100644 index 00000000..273a3f6d --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/GitHubAppCredentialCache.cs @@ -0,0 +1,69 @@ +using OpenDeepWiki.Services.Admin; + +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Thread-safe singleton cache for GitHub App credentials loaded from the database. +/// Checked first by GitHubAppService before falling back to env vars/config. +/// +public class GitHubAppCredentialCache +{ + private readonly object _lock = new(); + private string? _appId; + private string? _privateKeyBase64; + private string? _appName; + + public string? AppId + { + get { lock (_lock) return _appId; } + } + + public string? PrivateKeyBase64 + { + get { lock (_lock) return _privateKeyBase64; } + } + + public string? AppName + { + get { lock (_lock) return _appName; } + } + + /// + /// Update all cached credentials atomically. + /// + public void Update(string? appId, string? privateKeyBase64, string? appName) + { + lock (_lock) + { + _appId = string.IsNullOrWhiteSpace(appId) ? null : appId; + _privateKeyBase64 = string.IsNullOrWhiteSpace(privateKeyBase64) ? null : privateKeyBase64; + _appName = string.IsNullOrWhiteSpace(appName) ? null : appName; + } + } + + /// + /// Clear all cached credentials atomically. + /// + public void Clear() + { + lock (_lock) + { + _appId = null; + _privateKeyBase64 = null; + _appName = null; + } + } + + /// + /// Load credentials from the SystemSettings database table. + /// Called at startup and after saving new credentials via the admin UI. + /// + public async Task LoadFromDbAsync(IAdminSettingsService settingsService) + { + var appId = await settingsService.GetSettingByKeyAsync("GITHUB_APP_ID"); + var privateKey = await settingsService.GetSettingByKeyAsync("GITHUB_APP_PRIVATE_KEY"); + var appName = await settingsService.GetSettingByKeyAsync("GITHUB_APP_NAME"); + + Update(appId?.Value, privateKey?.Value, appName?.Value); + } +} diff --git a/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs b/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs new file mode 100644 index 00000000..88e51245 --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/GitHubAppService.cs @@ -0,0 +1,255 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using OpenDeepWiki.EFCore; +using OpenDeepWiki.Entities; + +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Implements GitHub App authentication and API operations. +/// Uses JWT signed with the app's private key to generate installation tokens. +/// +public class GitHubAppService : IGitHubAppService +{ + private readonly IConfiguration _configuration; + private readonly GitHubAppCredentialCache _cache; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IContext _context; + private readonly ILogger _logger; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = true + }; + + public GitHubAppService( + IConfiguration configuration, + GitHubAppCredentialCache cache, + IHttpClientFactory httpClientFactory, + IContext context, + ILogger logger) + { + _configuration = configuration; + _cache = cache; + _httpClientFactory = httpClientFactory; + _context = context; + _logger = logger; + } + + private string? AppId => + _cache.AppId + ?? _configuration["GitHub:App:Id"] + ?? Environment.GetEnvironmentVariable("GitHub__App__Id") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_ID"); + + private string? PrivateKeyBase64 => + _cache.PrivateKeyBase64 + ?? _configuration["GitHub:App:PrivateKey"] + ?? Environment.GetEnvironmentVariable("GitHub__App__PrivateKey") + ?? Environment.GetEnvironmentVariable("GITHUB_APP_PRIVATE_KEY"); + + public bool IsConfigured => !string.IsNullOrWhiteSpace(AppId) && !string.IsNullOrWhiteSpace(PrivateKeyBase64); + + /// + /// Generate a JWT for authenticating as the GitHub App. + /// JWT is valid for 10 minutes (GitHub maximum). + /// + private string GenerateJwt() + { + if (!IsConfigured) + throw new InvalidOperationException("GitHub App is not configured. Set GitHub:App:Id and GitHub:App:PrivateKey."); + + var pemBytes = Convert.FromBase64String(PrivateKeyBase64!); + var pemContent = System.Text.Encoding.UTF8.GetString(pemBytes); + + var rsa = RSA.Create(); + rsa.ImportFromPem(pemContent.AsSpan()); + + var securityKey = new RsaSecurityKey(rsa); + var credentials = new SigningCredentials(securityKey, SecurityAlgorithms.RsaSha256); + + var now = DateTime.UtcNow; + var token = new JwtSecurityToken( + issuer: AppId, + claims: new[] + { + new Claim("iat", new DateTimeOffset(now.AddSeconds(-60)).ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) + }, + expires: now.AddMinutes(10), + signingCredentials: credentials + ); + + return new JwtSecurityTokenHandler().WriteToken(token); + } + + private HttpClient CreateClient() + { + var client = _httpClientFactory.CreateClient("GitHubApp"); + client.BaseAddress = new Uri("https://api.github.com"); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OpenDeepWiki", "1.0")); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + return client; + } + + public async Task> ListInstallationsAsync(CancellationToken cancellationToken = default) + { + var jwt = GenerateJwt(); + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + + var response = await client.GetAsync("/app/installations", cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var installations = JsonSerializer.Deserialize>(json, JsonOptions) ?? new(); + + return installations.Select(i => new GitHubInstallationInfo + { + Id = i.Id, + AccountLogin = i.Account?.Login ?? string.Empty, + AccountType = i.Account?.Type ?? string.Empty, + AccountId = i.Account?.Id ?? 0, + AvatarUrl = i.Account?.AvatarUrl + }).ToList(); + } + + public async Task GetInstallationTokenAsync(long installationId, CancellationToken cancellationToken = default) + { + // Check cache in DB first + var installation = await _context.GitHubAppInstallations + .FirstOrDefaultAsync(i => i.InstallationId == installationId, cancellationToken); + + if (installation?.CachedAccessToken != null && + installation.TokenExpiresAt.HasValue && + installation.TokenExpiresAt.Value > DateTime.UtcNow.AddMinutes(5)) + { + _logger.LogDebug("Using cached installation token for installation {InstallationId}", installationId); + return installation.CachedAccessToken; + } + + // Generate fresh token + var jwt = GenerateJwt(); + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", jwt); + + var response = await client.PostAsync( + $"/app/installations/{installationId}/access_tokens", + null, + cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var tokenResponse = JsonSerializer.Deserialize(json, JsonOptions); + + if (tokenResponse?.Token == null) + throw new InvalidOperationException($"Failed to get installation token for installation {installationId}"); + + // Cache the token + if (installation != null) + { + installation.CachedAccessToken = tokenResponse.Token; + installation.TokenExpiresAt = tokenResponse.ExpiresAt; + installation.UpdateTimestamp(); + await _context.SaveChangesAsync(cancellationToken); + } + + _logger.LogInformation("Generated fresh installation token for installation {InstallationId}, expires at {ExpiresAt}", + installationId, tokenResponse.ExpiresAt); + + return tokenResponse.Token; + } + + public async Task ListInstallationReposAsync( + long installationId, int page = 1, int perPage = 30, CancellationToken cancellationToken = default) + { + var token = await GetInstallationTokenAsync(installationId, cancellationToken); + + using var client = CreateClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + + var response = await client.GetAsync( + $"/installation/repositories?per_page={perPage}&page={page}", + cancellationToken); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken); + var result = JsonSerializer.Deserialize(json, JsonOptions); + + return new GitHubRepoListResult + { + TotalCount = result?.TotalCount ?? 0, + Repositories = result?.Repositories?.Select(r => new GitHubInstallationRepo + { + Id = r.Id, + FullName = r.FullName ?? string.Empty, + Name = r.Name ?? string.Empty, + Owner = r.Owner?.Login ?? string.Empty, + Private = r.Private, + Description = r.Description, + Language = r.Language, + StargazersCount = r.StargazersCount, + ForksCount = r.ForksCount, + DefaultBranch = r.DefaultBranch ?? "main", + CloneUrl = r.CloneUrl ?? string.Empty, + HtmlUrl = r.HtmlUrl ?? string.Empty + }).ToList() ?? new() + }; + } + + // Internal DTOs for GitHub API responses + private class GitHubInstallationResponse + { + public long Id { get; set; } + public GitHubAccountResponse? Account { get; set; } + } + + private class GitHubAccountResponse + { + public long Id { get; set; } + public string? Login { get; set; } + public string? Type { get; set; } + public string? AvatarUrl { get; set; } + } + + private class GitHubTokenResponse + { + public string? Token { get; set; } + public DateTime? ExpiresAt { get; set; } + } + + private class GitHubRepoListResponse + { + public int TotalCount { get; set; } + public List? Repositories { get; set; } + } + + private class GitHubRepoResponse + { + public long Id { get; set; } + public string? FullName { get; set; } + public string? Name { get; set; } + public GitHubOwnerResponse? Owner { get; set; } + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string? DefaultBranch { get; set; } + public string? CloneUrl { get; set; } + public string? HtmlUrl { get; set; } + } + + private class GitHubOwnerResponse + { + public string? Login { get; set; } + } +} diff --git a/src/OpenDeepWiki/Services/GitHub/IGitHubAppService.cs b/src/OpenDeepWiki/Services/GitHub/IGitHubAppService.cs new file mode 100644 index 00000000..4b52b663 --- /dev/null +++ b/src/OpenDeepWiki/Services/GitHub/IGitHubAppService.cs @@ -0,0 +1,67 @@ +namespace OpenDeepWiki.Services.GitHub; + +/// +/// Information about a GitHub App installation. +/// +public class GitHubInstallationInfo +{ + public long Id { get; set; } + public string AccountLogin { get; set; } = string.Empty; + public string AccountType { get; set; } = string.Empty; + public long AccountId { get; set; } + public string? AvatarUrl { get; set; } +} + +/// +/// A repository accessible via a GitHub App installation. +/// +public class GitHubInstallationRepo +{ + public long Id { get; set; } + public string FullName { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string Owner { get; set; } = string.Empty; + public bool Private { get; set; } + public string? Description { get; set; } + public string? Language { get; set; } + public int StargazersCount { get; set; } + public int ForksCount { get; set; } + public string DefaultBranch { get; set; } = "main"; + public string CloneUrl { get; set; } = string.Empty; + public string HtmlUrl { get; set; } = string.Empty; +} + +/// +/// Result of listing installation repositories (paginated). +/// +public class GitHubRepoListResult +{ + public int TotalCount { get; set; } + public List Repositories { get; set; } = new(); +} + +/// +/// Service for GitHub App operations: JWT generation, installation tokens, repo listing. +/// +public interface IGitHubAppService +{ + /// + /// Whether the GitHub App is configured (App ID and Private Key are set). + /// + bool IsConfigured { get; } + + /// + /// List all installations of the GitHub App. + /// + Task> ListInstallationsAsync(CancellationToken cancellationToken = default); + + /// + /// Get or refresh an installation access token. + /// + Task GetInstallationTokenAsync(long installationId, CancellationToken cancellationToken = default); + + /// + /// List repositories accessible by a specific installation. + /// + Task ListInstallationReposAsync(long installationId, int page = 1, int perPage = 30, CancellationToken cancellationToken = default); +} diff --git a/src/OpenDeepWiki/Services/Repositories/RepositoryService.cs b/src/OpenDeepWiki/Services/Repositories/RepositoryService.cs index aaad052e..0f78ae7f 100644 --- a/src/OpenDeepWiki/Services/Repositories/RepositoryService.cs +++ b/src/OpenDeepWiki/Services/Repositories/RepositoryService.cs @@ -4,12 +4,13 @@ using OpenDeepWiki.Entities; using OpenDeepWiki.Models; using OpenDeepWiki.Services.Auth; +using OpenDeepWiki.Services.GitHub; namespace OpenDeepWiki.Services.Repositories; [MiniApi(Route = "/api/v1/repositories")] [Tags("仓库")] -public class RepositoryService(IContext context, IGitPlatformService gitPlatformService, IUserContext userContext) +public class RepositoryService(IContext context, IGitPlatformService gitPlatformService, IUserContext userContext, IGitHubAppService gitHubAppService) { [HttpPost("/submit")] public async Task SubmitAsync([FromBody] RepositorySubmitRequest request) @@ -22,7 +23,16 @@ public async Task SubmitAsync([FromBody] RepositorySubmitRequest req if (!request.IsPublic && string.IsNullOrWhiteSpace(request.AuthAccount) && string.IsNullOrWhiteSpace(request.AuthPassword)) { - throw new InvalidOperationException("仓库凭据为空时不允许设置为私有"); + // Allow private repos without credentials if a GitHub App installation exists for the org + var hasAppInstallation = gitHubAppService.IsConfigured && + !string.IsNullOrWhiteSpace(request.OrgName) && + await context.GitHubAppInstallations.AnyAsync( + i => i.AccountLogin == request.OrgName && !i.IsDeleted); + + if (!hasAppInstallation) + { + throw new InvalidOperationException("Private repositories require credentials or a GitHub App installation for the organization"); + } } // 校验是否已存在相同仓库（相同 GitUrl + BranchName） @@ -261,16 +271,24 @@ public async Task UpdateVisibilityAsync([FromBody] UpdateVisibilityRequ }, statusCode: StatusCodes.Status403Forbidden); } - // 无密码仓库不能设为私有 + // Allow private repos without credentials if a GitHub App installation exists for the org if (!request.IsPublic && string.IsNullOrWhiteSpace(repository.AuthPassword)) { - return Results.BadRequest(new UpdateVisibilityResponse + var hasAppInstallation = gitHubAppService.IsConfigured && + !string.IsNullOrWhiteSpace(repository.OrgName) && + await context.GitHubAppInstallations.AnyAsync( + i => i.AccountLogin == repository.OrgName && !i.IsDeleted); + + if (!hasAppInstallation) { - Id = request.RepositoryId, - IsPublic = repository.IsPublic, - Success = false, - ErrorMessage = "仓库凭据为空时不允许设置为私有" - }); + return Results.BadRequest(new UpdateVisibilityResponse + { + Id = request.RepositoryId, + IsPublic = repository.IsPublic, + Success = false, + ErrorMessage = "Private repositories require credentials or a GitHub App installation for the organization" + }); + } } // 更新可见性 diff --git a/web/app/admin/github-import/callback/page.tsx b/web/app/admin/github-import/callback/page.tsx new file mode 100644 index 00000000..70a6d66c --- /dev/null +++ b/web/app/admin/github-import/callback/page.tsx @@ -0,0 +1,54 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; +import { storeGitHubInstallation } from "@/lib/admin-api"; +import { Loader2 } from "lucide-react"; + +export default function GitHubImportCallbackPage() { + const router = useRouter(); + const searchParams = useSearchParams(); + const [error, setError] = useState(null); + + useEffect(() => { + const installationId = searchParams.get("installation_id"); + const setupAction = searchParams.get("setup_action"); + + if (!installationId) { + setError("Missing installation_id parameter"); + return; + } + + const storeAndRedirect = async () => { + try { + await storeGitHubInstallation(parseInt(installationId, 10)); + router.push("/admin/github-import"); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to store installation"); + } + }; + + storeAndRedirect(); + }, [searchParams, router]); + + if (error) { + return ( +
+

Error: {error}

+ +
+ ); + } + + return ( +
+ +

Connecting GitHub installation...

+
+ ); +} diff --git a/web/app/admin/github-import/page.tsx b/web/app/admin/github-import/page.tsx new file mode 100644 index 00000000..bf510bd9 --- /dev/null +++ b/web/app/admin/github-import/page.tsx @@ -0,0 +1,1038 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useState } from "react"; +import { useTranslations } from "@/hooks/use-translations"; +import { toast } from "sonner"; +import { + getGitHubStatus, + getGitHubInstallUrl, + getGitHubConfig, + saveGitHubConfig, + resetGitHubConfig, + getInstallationRepos, + batchImportRepos, + getDepartments, + disconnectGitHubInstallation, + GitHubStatus, + GitHubConfig, + GitHubRepo, + GitHubInstallation, + BatchImportResult, + AdminDepartment, +} from "@/lib/admin-api"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Label } from "@/components/ui/label"; +import { + RefreshCw, + ExternalLink, + GitBranch, + Star, + GitFork, + Lock, + Globe, + Loader2, + Building2, + CheckCircle2, + AlertCircle, + Plus, + Download, + Search, + Key, + Save, + HelpCircle, + Info, + Unlink, + RotateCcw, + Upload, +} from "lucide-react"; + +const PAGE_SIZE = 30; +const FETCH_BATCH_SIZE = 100; + +export default function GitHubImportPage() { + const t = useTranslations(); + + // Status + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + + // Selected installation + const [selectedInstallation, setSelectedInstallation] = useState(null); + + // All repos (fetched in full) + const [allRepos, setAllRepos] = useState([]); + const [repoTotalCount, setRepoTotalCount] = useState(0); + const [repoLoading, setRepoLoading] = useState(false); + const [loadProgress, setLoadProgress] = useState(""); + + // Selection + const [selectedRepos, setSelectedRepos] = useState>(new Set()); + + // Filters + const [searchQuery, setSearchQuery] = useState(""); + const [languageFilter, setLanguageFilter] = useState("all"); + const [importStatusFilter, setImportStatusFilter] = useState<"all" | "not_imported" | "imported">("all"); + + // Gmail-style select scope + const [selectAllScope, setSelectAllScope] = useState<"page" | "all">("page"); + + // Client-side pagination + const [page, setPage] = useState(1); + + // Import + const [departments, setDepartments] = useState([]); + const [selectedDepartmentId, setSelectedDepartmentId] = useState(""); + const [languageCode, setLanguageCode] = useState("en"); + const [importing, setImporting] = useState(false); + const [importResult, setImportResult] = useState(null); + + // GitHub App configuration form + const [config, setConfig] = useState(null); + const [configForm, setConfigForm] = useState({ + appId: "", + appName: "", + privateKey: "", + }); + const [saving, setSaving] = useState(false); + const [disconnecting, setDisconnecting] = useState(null); + const [resetting, setResetting] = useState(false); + + const fetchConfig = useCallback(async () => { + try { + const result = await getGitHubConfig(); + setConfig(result); + if (result.appId) setConfigForm((f) => ({ ...f, appId: result.appId || "" })); + if (result.appName) setConfigForm((f) => ({ ...f, appName: result.appName || "" })); + } catch { + // Config endpoint may not be available + } + }, []); + + const handleSaveConfig = async () => { + if (!configForm.appId.trim()) { + toast.error(t("admin.githubImport.validationAppIdRequired")); + return; + } + // When updating existing config, privateKey can be empty (keep existing) + if (!config?.hasPrivateKey && !configForm.privateKey.trim()) { + toast.error(t("admin.githubImport.validationPrivateKeyRequired")); + return; + } + if (configForm.privateKey.trim() && !configForm.privateKey.includes("-----BEGIN")) { + toast.error(t("admin.githubImport.validationPrivateKeyInvalid")); + return; + } + + setSaving(true); + try { + await saveGitHubConfig({ + appId: configForm.appId.trim(), + appName: configForm.appName.trim(), + privateKey: configForm.privateKey, + }); + toast.success(t("admin.githubImport.configSaved")); + setConfigForm((f) => ({ ...f, privateKey: "" })); + // Refresh status and config to show the configured state + await fetchStatus(); + await fetchConfig(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.configSaveFailed") + ); + } finally { + setSaving(false); + } + }; + + const fetchStatus = useCallback(async () => { + setLoading(true); + try { + const result = await getGitHubStatus(); + setStatus(result); + if (result.installations.length > 0 && !selectedInstallation) { + setSelectedInstallation(result.installations[0]); + } + } catch (error) { + toast.error(t("admin.githubImport.fetchStatusFailed")); + } finally { + setLoading(false); + } + }, [t, selectedInstallation]); + + const fetchDepartments = useCallback(async () => { + try { + const result = await getDepartments(); + setDepartments(result); + if (result.length > 0 && !selectedDepartmentId) { + setSelectedDepartmentId(result[0].id); + } + } catch { + // Departments may not be available + } + }, [selectedDepartmentId]); + + // Fetch ALL repos from the installation (paginated API calls in background) + const fetchAllRepos = useCallback(async () => { + if (!selectedInstallation) return; + setRepoLoading(true); + setAllRepos([]); + setRepoTotalCount(0); + setLoadProgress(""); + + try { + // First request to get total count + const firstResult = await getInstallationRepos( + selectedInstallation.installationId, + 1, + FETCH_BATCH_SIZE + ); + const total = firstResult.totalCount; + setRepoTotalCount(total); + + let accumulated = [...firstResult.repositories]; + setAllRepos(accumulated); + setLoadProgress(`${accumulated.length} / ${total}`); + + // Fetch remaining pages + const totalPages = Math.ceil(total / FETCH_BATCH_SIZE); + for (let p = 2; p <= totalPages; p++) { + const result = await getInstallationRepos( + selectedInstallation.installationId, + p, + FETCH_BATCH_SIZE + ); + accumulated = [...accumulated, ...result.repositories]; + setAllRepos(accumulated); + setLoadProgress(`${accumulated.length} / ${total}`); + } + } catch (error) { + toast.error(t("admin.githubImport.fetchReposFailed")); + } finally { + setRepoLoading(false); + setLoadProgress(""); + } + }, [selectedInstallation, t]); + + useEffect(() => { + fetchStatus(); + fetchDepartments(); + fetchConfig(); + }, []); + + useEffect(() => { + if (selectedInstallation) { + setSelectedRepos(new Set()); + setImportResult(null); + setSearchQuery(""); + setLanguageFilter("all"); + setImportStatusFilter("all"); + setSelectAllScope("page"); + setPage(1); + fetchAllRepos(); + } + }, [selectedInstallation]); + + const handleConnectNew = async () => { + try { + const { url } = await getGitHubInstallUrl(); + window.location.href = url; + } catch (error) { + toast.error("Failed to get install URL"); + } + }; + + const handleDisconnect = async (inst: GitHubInstallation) => { + const confirmed = window.confirm( + t("admin.githubImport.disconnectConfirm").replace("{org}", inst.accountLogin) + ); + if (!confirmed) return; + + setDisconnecting(inst.id); + try { + await disconnectGitHubInstallation(inst.id); + toast.success(t("admin.githubImport.disconnectSuccess")); + if (selectedInstallation?.id === inst.id) { + setSelectedInstallation(null); + setAllRepos([]); + } + await fetchStatus(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.disconnectFailed") + ); + } finally { + setDisconnecting(null); + } + }; + + const handleResetConfig = async () => { + const confirmed = window.confirm( + t("admin.githubImport.resetConfigConfirm") + ); + if (!confirmed) return; + + setResetting(true); + try { + await resetGitHubConfig(); + toast.success(t("admin.githubImport.resetConfigSuccess")); + setSelectedInstallation(null); + setAllRepos([]); + setConfig(null); + setConfigForm({ appId: "", appName: "", privateKey: "" }); + await fetchStatus(); + await fetchConfig(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.resetConfigFailed") + ); + } finally { + setResetting(false); + } + }; + + const toggleRepo = (fullName: string) => { + setSelectAllScope("page"); + setSelectedRepos((prev) => { + const next = new Set(prev); + if (next.has(fullName)) { + next.delete(fullName); + } else { + next.add(fullName); + } + return next; + }); + }; + + // Apply filters across ALL repos + const filteredRepos = useMemo(() => { + return allRepos.filter((r) => { + if (searchQuery) { + const q = searchQuery.toLowerCase(); + const matchesName = r.fullName.toLowerCase().includes(q); + const matchesDesc = r.description?.toLowerCase().includes(q); + if (!matchesName && !matchesDesc) return false; + } + if (languageFilter !== "all" && (r.language || "").toLowerCase() !== languageFilter.toLowerCase()) { + return false; + } + if (importStatusFilter === "not_imported" && r.alreadyImported) return false; + if (importStatusFilter === "imported" && !r.alreadyImported) return false; + return true; + }); + }, [allRepos, searchQuery, languageFilter, importStatusFilter]); + + const importableFiltered = useMemo( + () => filteredRepos.filter((r) => !r.alreadyImported), + [filteredRepos] + ); + + // Client-side pagination of filtered results + const totalFilteredPages = Math.ceil(filteredRepos.length / PAGE_SIZE); + const pagedRepos = useMemo( + () => filteredRepos.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE), + [filteredRepos, page] + ); + + // Collect unique languages from ALL repos for the filter dropdown + const repoLanguages = useMemo( + () => Array.from(new Set(allRepos.map((r) => r.language).filter(Boolean) as string[])).sort(), + [allRepos] + ); + + // Reset page and selection scope when filters change + useEffect(() => { + setPage(1); + setSelectAllScope("page"); + }, [searchQuery, languageFilter, importStatusFilter]); + + // Page-level selection helpers + const importableOnPage = useMemo( + () => pagedRepos.filter((r) => !r.alreadyImported), + [pagedRepos] + ); + const allPageSelected = importableOnPage.length > 0 && importableOnPage.every((r) => selectedRepos.has(r.fullName)); + const somePageSelected = importableOnPage.some((r) => selectedRepos.has(r.fullName)); + const hasMoreBeyondPage = importableFiltered.length > importableOnPage.length; + + const toggleSelectAll = () => { + if (allPageSelected) { + setSelectedRepos(new Set()); + setSelectAllScope("page"); + } else { + const pageNames = new Set(importableOnPage.map((r) => r.fullName)); + setSelectedRepos((prev) => { + const next = new Set(prev); + pageNames.forEach((name) => next.add(name)); + return next; + }); + setSelectAllScope("page"); + } + }; + + const handleSelectAllMatching = () => { + setSelectedRepos(new Set(importableFiltered.map((r) => r.fullName))); + setSelectAllScope("all"); + }; + + const handleClearSelection = () => { + setSelectedRepos(new Set()); + setSelectAllScope("page"); + }; + + const handleImport = async () => { + if (!selectedInstallation || selectedRepos.size === 0 || !selectedDepartmentId) return; + + setImporting(true); + setImportResult(null); + try { + const selectedRepoData = allRepos + .filter((r) => selectedRepos.has(r.fullName)) + .map((r) => ({ + fullName: r.fullName, + name: r.name, + owner: r.owner, + cloneUrl: r.cloneUrl, + defaultBranch: r.defaultBranch, + private: r.private, + language: r.language, + stargazersCount: r.stargazersCount, + forksCount: r.forksCount, + })); + + const result = await batchImportRepos({ + installationId: selectedInstallation.installationId, + departmentId: selectedDepartmentId, + languageCode, + repos: selectedRepoData, + }); + + setImportResult(result); + setSelectedRepos(new Set()); + toast.success( + t("admin.githubImport.importSuccess") + .replace("{imported}", result.imported.toString()) + .replace("{skipped}", result.skipped.toString()) + ); + + // Refresh repo list to update "already imported" flags + fetchAllRepos(); + } catch (error) { + toast.error( + error instanceof Error ? error.message : t("admin.githubImport.importFailed") + ); + } finally { + setImporting(false); + } + }; + + if (loading) { + return ( +
+ +
+ ); + } + + return ( +
+ {/* Header */} +
+
+

{t("admin.githubImport.title")}

+

+ {t("admin.githubImport.description")} +

+
+ +
+ + {/* Status Card */} + + + {t("admin.githubImport.connectionStatus")} + + + {!status?.configured ? ( +
+ + {t("admin.githubImport.credentialSourceNone")} +
+ ) : ( +
+
+ + {t("admin.githubImport.configured")} + {config && ( + + ({config.source === "database" + ? t("admin.githubImport.credentialSourceDb") + : t("admin.githubImport.credentialSourceEnv")}) + + )} +
+ + {/* Connected Organizations */} + {status.installations.length > 0 && ( +
+

+ {t("admin.githubImport.connectedOrgs")} +

+
+ {status.installations.map((inst) => ( +
setSelectedInstallation(inst)} + > +
+ {inst.avatarUrl && ( + {inst.accountLogin} + )} +
+ {inst.accountLogin} + + {inst.accountType} + +
+
+
+ {inst.departmentName && ( + + + {inst.departmentName} + + )} + +
+
+ ))} +
+
+ )} + +
+ + +
+
+ )} + + + + {/* Repository Selection */} + {selectedInstallation && ( + + + + {t("admin.githubImport.importFrom").replace( + "{org}", + selectedInstallation.accountLogin + )} + + + {repoLoading && loadProgress + ? `${t("admin.githubImport.loadingRepos")} ${loadProgress}...` + : t("admin.githubImport.totalRepos").replace( + "{count}", + repoTotalCount.toString() + )} + + + + {/* Import Options */} +
+
+ + +
+ +
+ + +
+
+ + {/* Search and Filter */} +
+
+ + setSearchQuery(e.target.value)} + className="pl-9" + /> +
+ + +
+ + {/* Select All */} +
+
+ + + {t("admin.githubImport.selectAll")} + {selectedRepos.size > 0 && + ` - ${selectedRepos.size} ${t("admin.githubImport.selected")}`} + +
+ {/* Gmail-style banner */} + {allPageSelected && hasMoreBeyondPage && selectAllScope === "page" && ( +
+ + {t("admin.githubImport.allPageSelected").replace("{count}", importableOnPage.length.toString())} + {" "} + +
+ )} + {selectAllScope === "all" && ( +
+ + {t("admin.githubImport.allMatchingSelected").replace("{count}", importableFiltered.length.toString())} + {" "} + +
+ )} +
+ + {/* Repository List */} + {repoLoading && allRepos.length === 0 ? ( +
+ +
+ ) : ( +
+ {pagedRepos.length === 0 && (searchQuery || languageFilter !== "all" || importStatusFilter !== "all") && ( +

+ {t("admin.githubImport.noResults")} +

+ )} + {pagedRepos.map((repo) => ( +
+ toggleRepo(repo.fullName)} + /> +
+
+ + {repo.fullName} + {repo.private ? ( + + ) : ( + + )} + {repo.alreadyImported && ( + + {t("admin.githubImport.alreadyImported")} + + )} +
+ {repo.description && ( +

+ {repo.description} +

+ )} +
+
+ {repo.language && ( + + {repo.language} + + )} + + + {repo.stargazersCount} + + + + {repo.forksCount} + + e.stopPropagation()} + > + + +
+
+ ))} +
+ )} + + {/* Pagination */} + {totalFilteredPages > 1 && ( +
+ + + {page} / {totalFilteredPages} + + +
+ )} + + {/* Import Button */} +
+ + {selectedRepos.size > 0 + ? t("admin.githubImport.readyToImport").replace( + "{count}", + selectedRepos.size.toString() + ) + : t("admin.githubImport.selectReposPrompt")} + + +
+ + {/* Import Result */} + {importResult && ( + + +
+ +
+

+ {t("admin.githubImport.importComplete")} +

+

+ {importResult.imported} {t("admin.githubImport.imported")},{" "} + {importResult.skipped} {t("admin.githubImport.skipped")} +

+ {importResult.skippedRepos.length > 0 && ( +

+ {t("admin.githubImport.skippedList")}:{" "} + {importResult.skippedRepos.join(", ")} +

+ )} +
+
+ + + )} + + + )} + + {/* Configuration Form -- shown when not configured */} + {status && !status.configured && ( + + + + + {status.configured + ? t("admin.githubImport.updateConfig") + : t("admin.githubImport.configTitle")} + + + {status.configured + ? t("admin.githubImport.updateConfigDescription") + : t("admin.githubImport.configDescription")} + + + + {config?.source === "environment" && !status.configured && ( +
+ + {t("admin.githubImport.overrideEnvWarning")} +
+ )} + +
+
+ + + setConfigForm((f) => ({ ...f, appId: e.target.value })) + } + /> +

+ {t("admin.githubImport.appIdHelp")} +

+
+ +
+ + + setConfigForm((f) => ({ ...f, appName: e.target.value })) + } + /> +

+ {t("admin.githubImport.appNameHelp")} +

+
+ +
+
+ + + { + const file = e.target.files?.[0]; + if (!file) return; + const reader = new FileReader(); + reader.onload = (ev) => { + const content = ev.target?.result as string; + setConfigForm((f) => ({ ...f, privateKey: content })); + }; + reader.readAsText(file); + e.target.value = ""; + }} + /> +
+