From 39c00bfd6f0ba50d2562ab6688a830f18f22719a Mon Sep 17 00:00:00 2001 From: zhangyangyu Date: Mon, 15 Jun 2026 02:01:32 +0800 Subject: [PATCH] feat: add client aggregate APIs --- docs/design/client-aggregate-apis.md | 511 +++++++++++ e2e/README.md | 20 + e2e/benchmark/client-aggregate-performance.sh | 355 ++++++++ internal/rest/handlers_agent.go | 40 +- internal/rest/handlers_aggregate.go | 830 ++++++++++++++++++ internal/rest/handlers_aggregate_test.go | 207 +++++ internal/rest/handlers_issue_list.go | 160 +++- internal/router/router.go | 7 + internal/service/issue_list_page.go | 70 +- 9 files changed, 2135 insertions(+), 65 deletions(-) create mode 100644 docs/design/client-aggregate-apis.md create mode 100755 e2e/benchmark/client-aggregate-performance.sh create mode 100644 internal/rest/handlers_aggregate.go create mode 100644 internal/rest/handlers_aggregate_test.go diff --git a/docs/design/client-aggregate-apis.md b/docs/design/client-aggregate-apis.md new file mode 100644 index 0000000..3a8895e --- /dev/null +++ b/docs/design/client-aggregate-apis.md @@ -0,0 +1,511 @@ +# Design: Client Aggregate APIs + +Status: Draft + +## Summary + +Console and Agent Team Service currently build product views by composing many GitHub-compatible REST calls in the browser or in the local ATS process. That keeps the API surface familiar, but it pushes fan-out, pagination, role normalization, and N+1 joins into clients. This document proposes a small set of generic read aggregation APIs for AGS. The goal is not to create console-only or ATS-only shortcuts; the goal is to expose reusable snapshots that future products can also use. + +The current conclusion is that one aggregate endpoint is not enough. `GET /api/v3/orgs/{org}/management-summary` is useful for the console organization page, but the larger wins are viewer bootstrap, issue thread loading, issue list filtering, and explicit wiki page batch reads. + +## Goals + +- Reduce client-side fan-out for common first-render paths. +- Keep mutations on the existing GitHub-compatible endpoints. +- Keep aggregate responses bounded, paginated, and explicit about included sections. +- Make viewer permissions and row-level capabilities explicit instead of forcing clients to infer them. +- Reuse existing GitHub-compatible response transforms where possible, while allowing custom aggregate envelopes. +- Support console, ATS, and future AGS-backed products with the same endpoints. + +## Non-goals + +- Replacing the GitHub-compatible REST API. +- Adding new mutation semantics. +- Returning unbounded repository, organization, wiki, or issue history. +- Adding product-specific endpoints such as `/console/...` or `/agent-team/...`. +- Hiding authorization differences. Each section must still obey the same authorization rules as the underlying resource. +- Adding recursive "return every wiki body" APIs. Full wiki body scans should stay outside the default client path. + +## Reuse Criteria + +An aggregate API should be added only when it is reusable by more than one current product, or when the resource shape is generic enough that the next AGS-backed product is likely to need it. Console-specific and ATS-specific flows should be expressed as generic AGS resource reads: viewer summaries, repository summaries, issue threads, issue list filters, wiki catalogs, and explicit wiki page batches. + +This means the API should not expose concepts like "agent profiles", "chat spaces", or "runs" directly. Those are product conventions layered on top of issues and wiki pages. The API can make issue and wiki access efficient, but product interpretation stays in the client. + +## Current Client Hotspots + +### Console + +Console bootstrap currently needs `/user`, `/user/agents`, `/user/orgs`, `/user/repos`, and then `/orgs/{org}/repos` for each organization. This is the broadest fan-out path and affects every signed-in session. + +Repository selection currently starts with `/repos/{owner}/{repo}` and then individual pages load labels, issues, wiki data, collaborators, teams, invitations, or organization context later. This is correct but makes page transitions show multiple loading phases. + +The organization page composes `/orgs/{org}`, `/orgs/{org}/repos`, `/orgs/{org}/teams`, `/orgs/{org}/members`, `/orgs/{org}/outside_collaborators`, and `/orgs/{org}/invitations`. Team drill-down can add more calls for members, repos, and invitations. + +The memories page pages through `/repos/{owner}/{repo}/issues`, then filters pull requests client-side. It often needs labels as a separate request. + +The wiki page composes page list/tree/search/detail/history/backlinks/labels depending on the view. The first render needs a catalog-like snapshot more than it needs each detail endpoint. + +### Agent Team Service + +ATS stores metadata in AGS. `list_agents` calls wiki page list under `agents/`, then reads each agent profile page one by one. That is a clear N+1 pattern and will get worse as teams grow. + +ATS chat uses AGS issues as durable chat spaces and issue comments as messages. Opening a chat needs the issue plus comments. Worker processing also repeatedly loads issue plus comments when it sees a mention. + +ATS runs are represented as AGS issues. `list_runs` lists all issues and filters `[run]` titles client-side. If a listed issue does not include the body, ATS fetches each run issue again to parse `agent` and `source`. + +ATS worker polling uses notifications, then loads issue and comments for each mention notification. It also scans chat issues and comments with a human token to detect mentions. That gives durable behavior, but it is call-heavy. + +## Call Chain Notation + +Each proposed API below lists current and future call chains for console and ATS: + +- "Current console" means the call pattern visible in the current console client. +- "Current ATS" means the call pattern visible in the current Agent Team Service backend or worker. +- "Future console" and "Future ATS" describe how the client should change after the aggregate exists. +- "No first-milestone use" means the endpoint is still generic, but that product should not adopt it until it has a real page or workflow that benefits from it. + +## Proposed API Set + +### 1. Viewer Summary + +```text +GET /api/v3/viewer/summary +``` + +This endpoint returns the current viewer and the workspace navigation data needed for first render. + +Default `include`: + +```text +include=user,orgs,repositories,invitations,agent_bindings +``` + +Optional query parameters: + +```text +repo_affiliation=owner,collaborator,organization_member +per_page=100 +page=1 +``` + +Response sketch: + +```json +{ + "user": { "login": "alice", "type": "User", "user_kind": "human" }, + "organizations": [ + { "login": "acme", "role": "admin", "permissions": { "manage_members": true } } + ], + "repositories": [ + { + "full_name": "acme/metadata", + "owner": { "login": "acme", "type": "Organization" }, + "permission": "admin", + "private": true + } + ], + "invitations": { + "repository_count": 1, + "organization_count": 0, + "repository_items": [], + "organization_items": [] + }, + "agent_bindings": { + "count": 2, + "items": [] + }, + "pagination": { "next": null } +} +``` + +This replaces console bootstrap fan-out and is also useful for any future client that needs a repo picker, workspace switcher, or invitation badge. It should be one of the first endpoints implemented. + +Call chain: + +- Current console: `AppStore.bootstrap()` -> `GET /user` -> `GET /user/agents` -> `GET /user/orgs` -> `GET /user/repos` -> `GET /orgs/{org}/repos` once per organization. Inbox badges later call `GET /user/repository_invitations` and `GET /user/organization_invitations`. +- Current ATS: login/session creation calls `GET /user`; workspace setup separately probes `GET /repos/{human}/metadata`. +- Future console: `AppStore.bootstrap()` -> `GET /viewer/summary?include=user,orgs,repositories,invitations,agent_bindings`, then only drill down with existing endpoints when the user opens a specific repo, org, or invitation view. +- Future ATS: session bootstrap can use `GET /viewer/summary?include=user,repositories` if it needs a workspace picker or metadata repo status. The minimal token login flow can keep `GET /user` if it only needs identity. + +### 2. Repository Summary + +```text +GET /api/v3/repos/{owner}/{repo}/summary +``` + +This endpoint returns a bounded snapshot for entering a repository workspace. + +Default `include`: + +```text +include=repo,viewer,counts,labels,wiki,agents +``` + +Optional sections: + +```text +include=access,invitations,teams,milestones +``` + +Response sketch: + +```json +{ + "repository": { "full_name": "acme/metadata", "private": true }, + "viewer": { + "permission": "admin", + "capabilities": { + "read": true, + "write": true, + "admin": true, + "manage_access": true + } + }, + "counts": { + "open_issues": 12, + "wiki_pages": 8, + "labels": 24, + "pending_invitations": 1 + }, + "labels": [], + "wiki": { "root_pages": [], "updated_at": "2026-06-12T00:00:00Z" }, + "agents": { "bound_count": 1, "items": [] } +} +``` + +This endpoint is useful but less urgent than viewer bootstrap and issue/wiki aggregates. It becomes valuable when console wants smoother repo transitions without loading every repo subview. + +Call chain: + +- Current console: `setRepo(owner, repo)` -> `GET /repos/{owner}/{repo}`. Individual pages then load labels, issues, wiki metadata, repo collaborators, repo invitations, and organization context through separate calls. +- Current ATS: `workspace()` and `bootstrap_workspace()` probe or create the metadata repository with `GET /repos/{human}/metadata` and `POST /user/repos`; other ATS reads go directly to issues and wiki pages. +- Future console: repo switch -> `GET /repos/{owner}/{repo}/summary?include=repo,viewer,counts,labels,wiki,agents`. Detail views still call dedicated issue, wiki, or access endpoints when opened. +- Future ATS: metadata workspace status can call `GET /repos/{human}/metadata/summary?include=repo,viewer,counts,wiki` if the UI wants a richer readiness check. No first-milestone use is required for worker execution. + +### 3. Organization Management Summary + +```text +GET /api/v3/orgs/{org}/management-summary +``` + +This endpoint returns the first-render state for organization management: organization profile, viewer role, repositories, members, pending invitations, teams, outside collaborators, and row-level capabilities. + +Default `include`: + +```text +include=repos,members,invitations,teams,outside_collaborators +``` + +Optional query parameters: + +```text +per_page=50 +team_detail_limit=20 +include=audit +audit_limit=20 +``` + +Role semantics should be GitHub-compatible at the REST boundary: + +- Organization membership role: `admin` or `member`. +- Organization invitation role: `admin` or `member`. +- Team membership role: `maintainer` or `member`. +- Repository permission: `admin`, `maintain`, `write`, `triage`, `read`, or `none`. + +This endpoint is still worth implementing for console, but it is not the only aggregation surface. It should be treated as one P0 item because the organization page is currently slow and role inference has caused correctness bugs. + +Call chain: + +- Current console: `loadOrgContext(ownerLogin)` -> parallel `GET /orgs/{org}`, `GET /orgs/{org}/repos`, `GET /orgs/{org}/teams`, `GET /orgs/{org}/members`, `GET /orgs/{org}/outside_collaborators`, and `GET /orgs/{org}/invitations`. Team detail pages may add `GET /orgs/{org}/teams/{team_slug}`, `/members`, `/repos`, and `/invitations`. +- Current ATS: no first-class organization management view. Agent binding uses repo grants against the human metadata repo and does not need the organization management snapshot. +- Future console: organization page first render -> `GET /orgs/{org}/management-summary`. Mutations still use existing membership, invitation, team, and repo grant endpoints, then refetch the summary or patch local state. +- Future ATS: no first-milestone use. A future team-admin product can reuse this endpoint when it needs an AGS organization management screen. + +### 4. Issue Thread + +```text +GET /api/v3/repos/{owner}/{repo}/issues/{issue_number}/thread +``` + +This endpoint returns an issue and its comments in one authorization-checked response. + +Default `include`: + +```text +include=issue,comments,viewer +``` + +Optional query parameters: + +```text +comments_per_page=100 +comments_page=1 +comment_sort=created +comment_direction=asc +``` + +Response sketch: + +```json +{ + "issue": { "number": 1, "title": "[chat] general", "state": "open" }, + "comments": [ + { "id": 101, "body": "hello", "user": { "login": "alice" } } + ], + "viewer": { + "capabilities": { + "comment": true, + "edit_own_comments": true, + "delete_own_comments": true + } + }, + "pagination": { "comments_next": null } +} +``` + +This is generic and high value. Console memory detail and ATS chat both need issue plus comments. The ATS worker also needs this when handling notifications or chat mentions. + +Call chain: + +- Current console: issue or memory detail views load `GET /repos/{owner}/{repo}/issues/{number}` and `GET /repos/{owner}/{repo}/issues/{number}/comments` separately when they need full thread context. +- Current ATS: opening chat messages uses `GET /repos/{owner}/{repo}/issues/{number}/comments`. Worker notification handling calls `GET /repos/{owner}/{repo}/issues/{number}` and then `GET /repos/{owner}/{repo}/issues/{number}/comments`. Chat mention scanning also loads issue plus comments for each chat issue. +- Future console: memory detail -> `GET /repos/{owner}/{repo}/issues/{number}/thread?include=issue,comments,viewer`. +- Future ATS: chat open and worker source loading -> `GET /repos/{owner}/{repo}/issues/{number}/thread?include=issue,comments,viewer`. The worker can then build prompts without two separate network round trips. + +### 5. Issue List Filters And Lightweight Views + +This can be implemented as extensions to the existing list endpoint rather than a new endpoint: + +```text +GET /api/v3/repos/{owner}/{repo}/issues?kind=issue&title_prefix=%5Bchat%5D&include=body&fields=number,title,state,body,updated_at +``` + +Useful additions: + +- `kind=issue|pull|all` so clients do not fetch PRs only to filter them out. +- `title_prefix=` for clients that encode typed records in issue titles, such as `[chat]`, `[run]`, `[incident]`, or `[decision]`. +- `include=body` so clients can parse lightweight issue metadata without fetching every issue. +- `fields=` to keep list responses small when only metadata is needed. + +This is more general than an ATS-specific `/chats` or `/runs` API. It benefits console memory lists, ATS chat lists, ATS run lists, and any future product that stores typed records as issues. + +Call chain: + +- Current console: `loadRepoMemories()` repeatedly calls `GET /repos/{owner}/{repo}/issues?state=all&per_page=100&page=N`, then filters out pull requests client-side. Labels are loaded through `GET /repos/{owner}/{repo}/labels` when needed. +- Current ATS: `list_chats()` calls `GET /repos/{human}/metadata/issues?state=all` and filters titles starting with `[chat]`. `list_runs()` calls the same all-issues list, filters `[run]`, and may call `GET /repos/{human}/metadata/issues/{number}` per run if the body is absent. +- Future console: memories list -> `GET /repos/{owner}/{repo}/issues?kind=issue&fields=number,title,state,user,labels,updated_at&state=all`. Detail still uses issue thread. +- Future ATS: chat list -> `GET /repos/{human}/metadata/issues?kind=issue&title_prefix=%5Bchat%5D&fields=number,title,state,updated_at`. Run list -> `GET /repos/{human}/metadata/issues?kind=issue&title_prefix=%5Brun%5D&include=body&fields=number,title,state,body,updated_at`. + +### 6. Wiki Page Batch Read + +```text +POST /api/v3/repos/{owner}/{repo}/wiki/pages/batch +``` + +This endpoint fetches explicitly selected wiki pages. It does not recursively expand a path and it does not mean "return every wiki page body". Callers must first choose a bounded set of slugs from existing list, tree, or search endpoints. + +Request sketch: + +```json +{ + "slugs": ["docs/getting-started", "agents/python-backend"], + "include": ["body", "labels", "backlink_count"], + "body_limit": 20000, + "ref": "main" +} +``` + +Response sketch: + +```json +{ + "items": [ + { + "slug": "docs/getting-started", + "title": "Getting Started", + "updated_at": "2026-06-12T00:00:00Z", + "body": "# Getting Started\n", + "body_truncated": false, + "labels": [] + } + ], + "missing": ["agents/python-backend"], + "limits": { + "max_slugs": 50, + "body_limit": 20000 + } +} +``` + +Bounds: + +- Reject empty `slugs`. +- Default `max_slugs` should be 50, with a hard maximum of 100. +- Default `body_limit` should be conservative. Large bodies must return `body_truncated: true`. +- No path recursion inside this endpoint. Path recursion stays in `GET /wiki/pages` and `GET /wiki/tree`, which return metadata by default. + +This is reusable across products. Console can batch-load selected wiki pages from a tree or search result. ATS can batch-load selected metadata records after listing their slugs. Future products can use the same shape for structured wiki-backed records. + +Call chain: + +- Current console: wiki views use `GET /wiki/pages`, `GET /wiki/tree`, or `GET /wiki/search` to discover pages, then call `GET /wiki/pages/{slug}`, `/history`, `/backlinks`, and `/labels` separately for detail data. +- Current ATS: `list_agents()` calls `GET /wiki/pages?path=agents&recursive=true`, then calls `GET /wiki/pages/{slug}` once per returned page to parse each profile. +- Future console: wiki tree/search returns selected slugs -> `POST /wiki/pages/batch` with `slugs` and `include=["body","labels","backlink_count"]` for the visible or selected pages only. +- Future ATS: `list_agents()` keeps using `GET /wiki/pages?path=agents&recursive=true` for metadata-only slug discovery, then calls `POST /wiki/pages/batch` with those explicit slugs and `include=["body"]`. It never asks AGS to recursively return every body. + +### 7. Wiki Catalog + +```text +GET /api/v3/repos/{owner}/{repo}/wiki/catalog +``` + +This endpoint returns the wiki navigation catalog for first render: tree, page metadata, labels, backlink counts, and latest update metadata. It should not return full bodies by default. + +Default `include`: + +```text +include=tree,pages,labels,backlink_counts +``` + +This is mainly a first-render wiki improvement. It is lower priority than explicit wiki page batch read because batch read removes a broader class of N+1 problems without encouraging full wiki body scans. + +Call chain: + +- Current console: wiki first render composes `GET /wiki/tree`, `GET /wiki/pages`, optional `GET /wiki/search`, and detail-only calls for selected pages. +- Current ATS: no first-milestone use. ATS treats wiki pages as metadata records and benefits more from explicit batch reads than from a navigation catalog. +- Future console: wiki first render -> `GET /wiki/catalog?include=tree,pages,labels,backlink_counts`, then selected page details use `POST /wiki/pages/batch` or existing detail endpoints. +- Future ATS: no first-milestone use. A future metadata browser could use `GET /wiki/catalog` for navigation while still using `POST /wiki/pages/batch` for selected record bodies. + +### 8. Notification Summary + +```text +GET /api/v3/notifications/summary +``` + +This endpoint returns notifications with enough subject context to avoid follow-up calls for the common worker case. + +Default `include`: + +```text +include=subject,latest_comments +``` + +Optional query parameters: + +```text +all=false +reason=mention +latest_comments_limit=10 +``` + +Response sketch: + +```json +[ + { + "id": "notification-1", + "reason": "mention", + "repository": { "full_name": "acme/metadata" }, + "subject": { + "type": "Issue", + "number": 1, + "title": "[chat] general", + "body": "type: agent-team-chat" + }, + "latest_comments": [] + } +] +``` + +This is useful for notification-driven clients. The current ATS worker is one example, but the endpoint shape should stay generic: notifications plus bounded subject context. It should be implemented carefully so it does not make notification reads expensive for users with many notifications. + +Call chain: + +- Current console: no first-milestone use. Console currently uses invitation-specific endpoints for inbox-like UI, not the generic notifications endpoint. +- Current ATS: worker calls `GET /notifications?all=false`, filters mention notifications, then calls `GET /repos/{owner}/{repo}/issues/{number}` and `GET /repos/{owner}/{repo}/issues/{number}/comments` for each relevant notification. +- Future console: no first-milestone use. A future notification inbox can call `GET /notifications/summary?include=subject`. +- Future ATS: worker calls `GET /notifications/summary?reason=mention&include=subject,latest_comments&latest_comments_limit=10`; if the returned context is insufficient, it falls back to `GET /issues/{number}/thread`. + +## Priority Recommendation + +P0 endpoints/extensions: + +- `GET /api/v3/viewer/summary` +- `GET /api/v3/orgs/{org}/management-summary` +- `GET /api/v3/repos/{owner}/{repo}/issues/{issue_number}/thread` +- issue list filters: `kind`, `title_prefix`, `include=body`, and `fields` +- `POST /api/v3/repos/{owner}/{repo}/wiki/pages/batch` + +P1 endpoints/extensions: + +- `GET /api/v3/repos/{owner}/{repo}/summary` +- `GET /api/v3/repos/{owner}/{repo}/wiki/catalog` +- `GET /api/v3/notifications/summary` + +The org management endpoint is worth implementing, but it should not be the only aggregate API. If only one endpoint can be built first, `viewer/summary` has the broadest product reuse. If the immediate pain is console organization management, build `orgs/{org}/management-summary` first. If the immediate pain is AGS-backed collaboration clients, build issue thread, issue list filters, and wiki page batch read first. + +## Authorization Model + +Each aggregate section must use the same authorization rules as the underlying resource. A viewer who cannot read an underlying list must not receive that list through an aggregate. For optional sections, the preferred behavior is to omit unauthorized sections and include a capability flag that explains the viewer cannot manage or view that section. Return `403` only when the entire aggregate endpoint is forbidden. + +Aggregate responses should include viewer capabilities where the UI would otherwise infer them. Examples: + +```json +{ + "viewer": { + "login": "alice", + "role": "admin", + "capabilities": { + "manage_members": true, + "manage_invitations": true, + "comment": true + } + } +} +``` + +## Data Bounds + +Every aggregate endpoint must have explicit bounds: + +- `per_page` and `page` for top-level lists. +- Section-specific limits such as `team_detail_limit`, `comments_per_page`, `latest_comments_limit`, and `body_limit`. +- A default `include` set that is useful but not explosive. +- A documented fallback path to the existing detailed endpoints for drill-down. +- Explicit selection for body-heavy batch reads. For wiki bodies, clients must provide `slugs`; the backend must not infer "all pages under this path" and return all bodies. + +Aggregates should not silently return all data for large organizations, repositories, or wikis. + +## Implementation Notes + +Prefer grouped service methods over calling REST handlers internally. REST handlers should remain thin wrappers around service APIs. + +Use grouped queries for nested data. For example, organization summary should load team member counts and team repository counts for all returned teams in grouped queries, not one query per team. + +Keep response structs separate from GitHub-compatible endpoint structs when the aggregate needs capability flags, counts, or section envelopes. Reuse transformation helpers for embedded GitHub-like resources. + +Use short TTL caching only after measuring. Correctness and bounded query count are more important for the first implementation. + +## Rollout Plan + +1. Add backend service DTOs and tests for `viewer/summary`. +2. Update console bootstrap to use `viewer/summary`, with fallback to current endpoints while staging verifies the response. +3. Add `orgs/{org}/management-summary` and move the organization page to it. +4. Add issue thread, issue list filters, and wiki page batch read for console and AGS-backed collaboration clients. +5. Add repo summary, wiki catalog, and notification summary as follow-up optimizations. + +## Test Plan + +Backend tests should cover authorization, omitted sections, role normalization, pagination bounds, and large-list truncation for every aggregate. + +Console tests should cover bootstrap with one summary request, organization page first render with management summary, and fallback behavior when an aggregate endpoint is missing. + +ATS tests should cover metadata record loading through wiki page batch read, chat loading through issue thread, and worker notification processing through notification summary if that endpoint is implemented. + +## Open Questions + +- Should `viewer/summary` include full invitation records by default, or only counts unless `include=invitations:items` is requested? +- Should issue list `title_prefix` be limited to exact prefix matching, or should it support a more general server-side query language? +- Should wiki page batch read use `POST` for request body size, or support a `GET` variant for small slug lists? +- Should `notifications/summary` mark notifications as read, or should it preserve the existing notification read semantics exactly? diff --git a/e2e/README.md b/e2e/README.md index 1ef06a6..eb8ad4d 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -45,6 +45,20 @@ Override base URL (if not on port 80) or curl flags: make test-e2e E2E_BASE_URL="https://github.localhost:8080" ``` +Run the aggregate API performance benchmark against any environment: + +```bash +E2E_TOKEN="$TOKEN" make test-e2e SCRIPT=benchmark/client-aggregate-performance E2E_BASE_URL="http://127.0.0.1:8080" +E2E_TOKEN="$TOKEN" make test-e2e SCRIPT=benchmark/client-aggregate-performance E2E_BASE_URL="https://github.example.com" +``` + +Optional benchmark inputs: +- `E2E_PERF_REPO=owner/repo` chooses the repository target. Otherwise the first visible repo is used. +- `E2E_PERF_ORG=org` chooses the org management target. Otherwise the first visible org is used. +- `E2E_PERF_ISSUE_NUMBER=123` chooses the issue thread target. Otherwise the first issue in the repo is used. +- `E2E_PERF_WIKI_SLUGS=home,guides/setup` chooses wiki pages for batch comparison. Otherwise the first wiki pages are discovered. +- `E2E_PERF_ITERATIONS=10` changes measured iterations. + ## Test Scripts | Script | Description | Mode | @@ -64,3 +78,9 @@ make test-e2e E2E_BASE_URL="https://github.localhost:8080" | `token-api.sh` | User token API smoke flow | Existing server | | `token-lifecycle.sh` | User token lifecycle and revocation behavior | Existing server | | `vector-search-e2e.sh` | Vector and semantic search behavior with a mock embedding server | Self-contained TiDB | + +## Benchmark Scripts + +| Script | Description | Mode | +|--------|-------------|------| +| `benchmark/client-aggregate-performance.sh` | Wall-clock comparison between legacy client call chains and aggregate APIs | Existing server plus token | diff --git a/e2e/benchmark/client-aggregate-performance.sh b/e2e/benchmark/client-aggregate-performance.sh new file mode 100755 index 0000000..788b319 --- /dev/null +++ b/e2e/benchmark/client-aggregate-performance.sh @@ -0,0 +1,355 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT/e2e/lib.sh" + +require_cmd curl +require_cmd jq +require_cmd python3 + +BASE_URL="$(strip_trailing_slash "${E2E_BASE_URL:-http://github.localhost}")" +TOKEN="${E2E_TOKEN:-${ADMIN_TOKEN:-${TEST_TOKEN:-${GH_TOKEN:-}}}}" +ITERATIONS="${E2E_PERF_ITERATIONS:-5}" +WARMUP_ITERATIONS="${E2E_PERF_WARMUP_ITERATIONS:-1}" +PER_PAGE="${E2E_PERF_PER_PAGE:-100}" +COMMENT_PER_PAGE="${E2E_PERF_COMMENT_PER_PAGE:-100}" +WIKI_BATCH_LIMIT="${E2E_PERF_WIKI_BATCH_LIMIT:-10}" + +if [[ -z "$TOKEN" ]]; then + note "Skipping client aggregate performance e2e: set E2E_TOKEN, ADMIN_TOKEN, TEST_TOKEN, or GH_TOKEN." + exit 0 +fi + +if ! [[ "$ITERATIONS" =~ ^[0-9]+$ ]] || [[ "$ITERATIONS" -lt 1 ]]; then + echo "E2E_PERF_ITERATIONS must be a positive integer" >&2 + exit 1 +fi + +if ! [[ "$WARMUP_ITERATIONS" =~ ^[0-9]+$ ]]; then + echo "E2E_PERF_WARMUP_ITERATIONS must be a non-negative integer" >&2 + exit 1 +fi + +RESULTS="$(mktemp)" +trap 'rm -f "$RESULTS"' EXIT +REQUESTS=0 + +now_ns() { + python3 - <<'PY' +import time +print(time.perf_counter_ns()) +PY +} + +elapsed_ms() { + local start="$1" + local end="$2" + python3 - "$start" "$end" <<'PY' +import sys +start = int(sys.argv[1]) +end = int(sys.argv[2]) +print(f"{(end - start) / 1_000_000:.3f}") +PY +} + +url_path_escape() { + python3 - "$1" <<'PY' +import sys +from urllib.parse import quote +print(quote(sys.argv[1], safe="")) +PY +} + +auth_args() { + printf '%s\0' -H "Authorization: token $TOKEN" +} + +request() { + local method="$1" + local url="$2" + local body="${3:-}" + local tmp + tmp="$(mktemp)" + local code + REQUESTS=$((REQUESTS + 1)) + if [[ -n "$body" ]]; then + code="$(curl -ksS -o "$tmp" -w "%{http_code}" -X "$method" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "$url")" + else + code="$(curl -ksS -o "$tmp" -w "%{http_code}" -X "$method" \ + -H "Authorization: token $TOKEN" \ + "$url")" + fi + if [[ "$code" != 2* ]]; then + echo "unexpected status: method=$method code=$code url=$url" >&2 + echo "response body:" >&2 + cat "$tmp" >&2 + rm -f "$tmp" + return 1 + fi + cat "$tmp" + rm -f "$tmp" +} + +request_optional() { + local method="$1" + local url="$2" + local body="${3:-}" + local tmp + tmp="$(mktemp)" + local code + REQUESTS=$((REQUESTS + 1)) + if [[ -n "$body" ]]; then + code="$(curl -ksS -o "$tmp" -w "%{http_code}" -X "$method" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: application/json" \ + -d "$body" \ + "$url")" + else + code="$(curl -ksS -o "$tmp" -w "%{http_code}" -X "$method" \ + -H "Authorization: token $TOKEN" \ + "$url")" + fi + if [[ "$code" == 2* ]]; then + cat "$tmp" + fi + rm -f "$tmp" +} + +measure_once() { + local case_name="$1" + local mode="$2" + local fn="$3" + local iteration="$4" + REQUESTS=0 + local start end ms + start="$(now_ns)" + "$fn" >/dev/null + end="$(now_ns)" + ms="$(elapsed_ms "$start" "$end")" + printf '%s\t%s\t%s\t%s\t%s\n' "$case_name" "$mode" "$iteration" "$ms" "$REQUESTS" >> "$RESULTS" +} + +benchmark_pair() { + local case_name="$1" + local old_fn="$2" + local new_fn="$3" + local reason="${4:-}" + if [[ -n "$reason" ]]; then + note "Skipping $case_name: $reason" + return 0 + fi + + note "Benchmarking $case_name" + for i in $(seq 1 "$WARMUP_ITERATIONS"); do + measure_once "$case_name" old "$old_fn" "warmup-$i" >/dev/null + measure_once "$case_name" new "$new_fn" "warmup-$i" >/dev/null + done + for i in $(seq 1 "$ITERATIONS"); do + measure_once "$case_name" old "$old_fn" "$i" + measure_once "$case_name" new "$new_fn" "$i" + done +} + +discover_repo() { + if [[ -n "${E2E_PERF_REPO:-}" ]]; then + echo "$E2E_PERF_REPO" + return 0 + fi + request GET "$BASE_URL/api/v3/user/repos?per_page=1" | jq -r '.[0].full_name // empty' +} + +discover_org() { + if [[ -n "${E2E_PERF_ORG:-}" ]]; then + echo "$E2E_PERF_ORG" + return 0 + fi + request GET "$BASE_URL/api/v3/user/orgs" | jq -r '.[0].login // empty' +} + +discover_issue_number() { + local repo_full="$1" + if [[ -n "${E2E_PERF_ISSUE_NUMBER:-}" ]]; then + echo "$E2E_PERF_ISSUE_NUMBER" + return 0 + fi + request_optional GET "$BASE_URL/api/v3/repos/$repo_full/issues?state=all&kind=issue&fields=number&per_page=1" | jq -r '.[0].number // empty' +} + +discover_wiki_slugs() { + local repo_full="$1" + if [[ -n "${E2E_PERF_WIKI_SLUGS:-}" ]]; then + echo "$E2E_PERF_WIKI_SLUGS" + return 0 + fi + request_optional GET "$BASE_URL/api/v3/repos/$repo_full/wiki/catalog?include=pages&recursive=true" | jq -r --argjson limit "$WIKI_BATCH_LIMIT" '[.pages[]?.slug][0:$limit] | join(",")' +} + +viewer_old_chain() { + request GET "$BASE_URL/api/v3/user" + request GET "$BASE_URL/api/v3/user/repos?per_page=$PER_PAGE" + request GET "$BASE_URL/api/v3/user/orgs" + request GET "$BASE_URL/api/v3/user/repository_invitations" + request GET "$BASE_URL/api/v3/user/organization_invitations" + request GET "$BASE_URL/api/v3/user/agents" +} + +viewer_new_chain() { + request GET "$BASE_URL/api/v3/viewer/summary?include=user,orgs,repositories,invitations,agent_bindings&per_page=$PER_PAGE" +} + +notifications_old_chain() { + request GET "$BASE_URL/api/v3/notifications?per_page=$PER_PAGE" +} + +notifications_new_chain() { + request GET "$BASE_URL/api/v3/notifications/summary?include=subject,repository&per_page=$PER_PAGE" +} + +org_old_chain() { + request GET "$BASE_URL/api/v3/orgs/$ORG" + request GET "$BASE_URL/api/v3/orgs/$ORG/repos?per_page=$PER_PAGE" + request GET "$BASE_URL/api/v3/orgs/$ORG/members?per_page=$PER_PAGE" + request GET "$BASE_URL/api/v3/orgs/$ORG/invitations?per_page=$PER_PAGE" + request GET "$BASE_URL/api/v3/orgs/$ORG/teams?per_page=$PER_PAGE" +} + +org_new_chain() { + request GET "$BASE_URL/api/v3/orgs/$ORG/management-summary?include=org,viewer,repos,members,invitations,teams" +} + +repo_old_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL" + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/labels" + request_optional GET "$BASE_URL/api/v3/repos/$REPO_FULL/wiki/pages?recursive=true&per_page=$PER_PAGE" + request GET "$BASE_URL/api/v3/user/agents" +} + +repo_new_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/summary?include=repo,viewer,counts,labels,wiki,agents" +} + +issue_list_old_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/issues?state=all&per_page=$PER_PAGE" | jq --arg prefix "${E2E_PERF_TITLE_PREFIX:-}" ' + if $prefix == "" then . else map(select(.title | startswith($prefix))) end + ' +} + +issue_list_new_chain() { + local url="$BASE_URL/api/v3/repos/$REPO_FULL/issues?state=all&kind=issue&fields=number,title,state,updated_at&per_page=$PER_PAGE" + if [[ -n "${E2E_PERF_TITLE_PREFIX:-}" ]]; then + url="$url&title_prefix=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "${E2E_PERF_TITLE_PREFIX}")" + fi + request GET "$url" +} + +issue_thread_old_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/issues/$ISSUE_NUMBER" + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/issues/$ISSUE_NUMBER/comments?per_page=$COMMENT_PER_PAGE" +} + +issue_thread_new_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/issues/$ISSUE_NUMBER/thread?include=issue,comments&comments_per_page=$COMMENT_PER_PAGE" +} + +wiki_catalog_old_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/wiki/tree" + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/wiki/pages?recursive=true&per_page=$PER_PAGE" +} + +wiki_catalog_new_chain() { + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/wiki/catalog?include=tree,pages,labels&recursive=true" +} + +wiki_batch_old_chain() { + local slug + for slug in "${WIKI_SLUGS[@]}"; do + request GET "$BASE_URL/api/v3/repos/$REPO_FULL/wiki/pages/$(url_path_escape "$slug")" + done +} + +wiki_batch_new_chain() { + local slugs_json + local body + slugs_json="$(printf '%s\n' "${WIKI_SLUGS[@]}" | jq -R . | jq -s .)" + body="$(jq -cn --argjson slugs "$slugs_json" '{slugs: $slugs, include: ["body", "labels"], body_limit: 20000}')" + request POST "$BASE_URL/api/v3/repos/$REPO_FULL/wiki/pages/batch" "$body" +} + +note "Base URL: $BASE_URL" +note "Iterations: $ITERATIONS, warmup: $WARMUP_ITERATIONS" + +VIEWER="$(request GET "$BASE_URL/api/v3/user" | jq -r '.login // empty')" +if [[ -z "$VIEWER" ]]; then + echo "unable to resolve authenticated viewer" >&2 + exit 1 +fi +note "Viewer: $VIEWER" + +REPO_FULL="$(discover_repo)" +ORG="$(discover_org)" +ISSUE_NUMBER="" +WIKI_SLUGS_CSV="" +WIKI_SLUGS=() + +if [[ -n "$REPO_FULL" ]]; then + note "Repo target: $REPO_FULL" + ISSUE_NUMBER="$(discover_issue_number "$REPO_FULL")" + WIKI_SLUGS_CSV="$(discover_wiki_slugs "$REPO_FULL")" + if [[ -n "$WIKI_SLUGS_CSV" ]]; then + IFS=',' read -r -a WIKI_SLUGS <<< "$WIKI_SLUGS_CSV" + note "Wiki page targets: ${#WIKI_SLUGS[@]}" + fi +else + note "No repository target found. Set E2E_PERF_REPO=owner/repo to enable repo benchmarks." +fi + +if [[ -n "$ORG" ]]; then + note "Org target: $ORG" +else + note "No organization target found. Set E2E_PERF_ORG=org to enable org benchmarks." +fi + +benchmark_pair "viewer_summary" viewer_old_chain viewer_new_chain +benchmark_pair "notifications_summary" notifications_old_chain notifications_new_chain +benchmark_pair "org_management_summary" org_old_chain org_new_chain "$([[ -z "$ORG" ]] && echo "no org target")" +benchmark_pair "repo_summary" repo_old_chain repo_new_chain "$([[ -z "$REPO_FULL" ]] && echo "no repo target")" +benchmark_pair "issue_list_filters" issue_list_old_chain issue_list_new_chain "$([[ -z "$REPO_FULL" ]] && echo "no repo target")" +benchmark_pair "issue_thread" issue_thread_old_chain issue_thread_new_chain "$([[ -z "$REPO_FULL" || -z "$ISSUE_NUMBER" ]] && echo "no issue target")" +benchmark_pair "wiki_catalog" wiki_catalog_old_chain wiki_catalog_new_chain "$([[ -z "$REPO_FULL" || "${#WIKI_SLUGS[@]}" -eq 0 ]] && echo "no wiki pages")" +benchmark_pair "wiki_batch_pages" wiki_batch_old_chain wiki_batch_new_chain "$([[ -z "$REPO_FULL" || "${#WIKI_SLUGS[@]}" -eq 0 ]] && echo "no wiki pages")" + +echo +echo "Client aggregate performance results" +echo "case,old_requests,new_requests,old_avg_ms,new_avg_ms,delta_ms,speedup" +awk -F'\t' ' + $3 !~ /^warmup-/ { + key=$1 SUBSEP $2 + sum[key]+=$4 + count[key]++ + req[key]+=$5 + cases[$1]=1 + } + END { + for (c in cases) { + old_key=c SUBSEP "old" + new_key=c SUBSEP "new" + if (count[old_key] == 0 || count[new_key] == 0) { + continue + } + old_avg=sum[old_key]/count[old_key] + new_avg=sum[new_key]/count[new_key] + old_req=req[old_key]/count[old_key] + new_req=req[new_key]/count[new_key] + delta=old_avg-new_avg + speedup=(new_avg > 0 ? old_avg/new_avg : 0) + printf "%s,%.1f,%.1f,%.3f,%.3f,%.3f,%.2fx\n", c, old_req, new_req, old_avg, new_avg, delta, speedup + } + } +' "$RESULTS" | sort + +ok "client aggregate performance benchmark completed" diff --git a/internal/rest/handlers_agent.go b/internal/rest/handlers_agent.go index 9ddb894..4a94b06 100644 --- a/internal/rest/handlers_agent.go +++ b/internal/rest/handlers_agent.go @@ -99,28 +99,32 @@ func (d *Deps) ListBoundAgents(w http.ResponseWriter, r *http.Request) { } out := make([]any, 0, len(agents)) for _, item := range agents { - var tokenStatus any - if item.TokenStatus.CreatedAt != nil { - tokenStatus = map[string]any{ - "state": item.TokenStatus.State, - "created_at": item.TokenStatus.CreatedAt.UTC().Format(time.RFC3339), - } - } else { - tokenStatus = map[string]any{"state": item.TokenStatus.State} - } - out = append(out, map[string]any{ - "agent": transform.User(item.Agent), - "bound_at": item.BoundAt.UTC().Format(time.RFC3339), - "token_status": tokenStatus, - "access_summary": map[string]any{ - "repos": item.AccessSummary.Repos, - "teams": item.AccessSummary.Teams, - }, - }) + out = append(out, boundAgentJSON(item)) } respond.JSON(w, http.StatusOK, out) } +func boundAgentJSON(item service.BoundAgent) map[string]any { + var tokenStatus any + if item.TokenStatus.CreatedAt != nil { + tokenStatus = map[string]any{ + "state": item.TokenStatus.State, + "created_at": item.TokenStatus.CreatedAt.UTC().Format(time.RFC3339), + } + } else { + tokenStatus = map[string]any{"state": item.TokenStatus.State} + } + return map[string]any{ + "agent": transform.User(item.Agent), + "bound_at": item.BoundAt.UTC().Format(time.RFC3339), + "token_status": tokenStatus, + "access_summary": map[string]any{ + "repos": item.AccessSummary.Repos, + "teams": item.AccessSummary.Teams, + }, + } +} + // RenameBoundAgent handles PATCH /api/v3/agent-bindings/{agent_login}. func (d *Deps) RenameBoundAgent(w http.ResponseWriter, r *http.Request) { u, err := d.Svc.GetCurrentUser(r.Context()) diff --git a/internal/rest/handlers_aggregate.go b/internal/rest/handlers_aggregate.go new file mode 100644 index 0000000..dac1fcb --- /dev/null +++ b/internal/rest/handlers_aggregate.go @@ -0,0 +1,830 @@ +package rest + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/ngaut/agent-git-service/internal/db" + "github.com/ngaut/agent-git-service/internal/rest/respond" + "github.com/ngaut/agent-git-service/internal/rest/transform" + "github.com/ngaut/agent-git-service/internal/service" +) + +const ( + defaultWikiBatchBodyLimit = 20_000 + maxWikiBatchBodyLimit = 100_000 + maxWikiBatchSlugs = 50 +) + +// GetViewerSummary handles GET /api/v3/viewer/summary. +func (d *Deps) GetViewerSummary(w http.ResponseWriter, r *http.Request) { + viewer, err := d.Svc.GetCurrentUser(r.Context()) + if err != nil { + respond.Error(w, http.StatusUnauthorized, "Bad credentials") + return + } + include := parseIncludeSet(r, []string{"user", "orgs", "repositories", "invitations", "agent_bindings"}) + out := map[string]any{} + + if include["user"] { + out["user"] = transform.UserPrivate(viewer) + } + if include["orgs"] { + orgs, err := d.Svc.ListOrgs(r.Context()) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(orgs)) + for _, org := range orgs { + row := transform.User(org) + if membership, err := d.Svc.GetOrgMembership(r.Context(), org.ID, viewer.Login); err == nil { + row["role"] = membership.Role + row["state"] = membership.State + row["permissions"] = map[string]any{ + "manage_members": membership.Role == "admin", + "manage_repos": membership.Role == "admin", + } + } + items = append(items, row) + } + out["organizations"] = map[string]any{ + "total_count": len(orgs), + "items": items, + } + } + if include["repositories"] { + page, perPage := parsePagination(r) + repos, err := d.Svc.ListViewerRepos(r.Context()) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + repoAffiliation, err := parseRepoAffiliationSet(r.URL.Query().Get("repo_affiliation")) + if err != nil { + respond.ValidationFailed(w, err.Error()) + return + } + repos = filterViewerReposByAffiliation(repos, viewer, repoAffiliation) + paged := paginate(w, r, d.Svc.BaseURL, repos, page, perPage) + items := make([]any, 0, len(paged)) + for _, row := range paged { + stats := d.repoStats(r, row.Repository) + stats.HasPermissions = true + stats.Permissions = repoPermissionsFor(row.Permission) + item := transform.Repo(row.Repository, stats) + item["permission"] = row.Permission.String() + items = append(items, item) + } + out["repositories"] = map[string]any{ + "total_count": len(repos), + "items": items, + } + } + if include["invitations"] { + repoInvs, err := d.Svc.ListUserInvitations(r.Context(), viewer.ID) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + orgInvs, err := d.Svc.ListUserOrganizationInvitations(r.Context(), viewer.ID) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + repoItems := make([]any, 0, len(repoInvs)) + for _, inv := range repoInvs { + repoItems = append(repoItems, repositoryInvitationJSON(inv)) + } + orgItems := make([]any, 0, len(orgInvs)) + for _, inv := range orgInvs { + orgItems = append(orgItems, organizationInvitationJSON(inv)) + } + out["invitations"] = map[string]any{ + "total_count": len(repoInvs) + len(orgInvs), + "repositories": map[string]any{ + "total_count": len(repoInvs), + "items": repoItems, + }, + "organizations": map[string]any{ + "total_count": len(orgInvs), + "items": orgItems, + }, + } + } + if include["agent_bindings"] { + agents, err := d.Svc.ListBoundAgents(r.Context(), viewer.ID) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(agents)) + for _, agent := range agents { + items = append(items, boundAgentJSON(agent)) + } + out["agent_bindings"] = map[string]any{ + "total_count": len(agents), + "items": items, + } + } + + respond.JSON(w, http.StatusOK, out) +} + +// GetOrgManagementSummary handles GET /api/v3/orgs/{org}/management-summary. +func (d *Deps) GetOrgManagementSummary(w http.ResponseWriter, r *http.Request) { + org := d.mustGetOrg(w, r) + if org == nil { + return + } + if !d.requireOrgAdmin(w, r, org) { + return + } + include := parseIncludeSet(r, []string{"org", "viewer", "repos", "members", "invitations", "teams", "outside_collaborators"}) + out := map[string]any{} + + if include["org"] { + out["organization"] = transform.User(*org) + } + if include["viewer"] { + viewer, err := d.Svc.GetCurrentUser(r.Context()) + if err != nil { + respond.Error(w, http.StatusUnauthorized, "Bad credentials") + return + } + membership, err := d.Svc.GetOrgMembership(r.Context(), org.ID, viewer.Login) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + out["viewer"] = map[string]any{ + "user": transform.User(viewer), + "role": membership.Role, + "state": membership.State, + "permissions": map[string]any{ + "manage_members": true, + "manage_repos": true, + "manage_teams": true, + }, + } + } + counts := map[string]int{} + if include["repos"] { + repos, err := d.Svc.ListUserRepos(r.Context(), org.Login) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(repos)) + for _, repo := range repos { + items = append(items, transform.Repo(repo, d.repoStats(r, repo))) + } + counts["repos"] = len(repos) + out["repositories"] = items + } + if include["members"] { + members, err := d.Svc.ListOrgMembers(r.Context(), org.ID, "") + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(members)) + for _, member := range members { + items = append(items, orgMemberSummaryJSON(member)) + } + counts["members"] = len(members) + out["members"] = items + } + if include["invitations"] { + invs, err := d.Svc.ListOrganizationInvitations(r.Context(), org.ID) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(invs)) + for _, inv := range invs { + items = append(items, organizationInvitationJSON(inv)) + } + counts["invitations"] = len(invs) + out["invitations"] = items + } + if include["teams"] { + teams, err := d.Svc.ListOrgTeams(r.Context(), org.ID) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(teams)) + for _, team := range teams { + team.Organization = *org + items = append(items, transform.Team(team)) + } + counts["teams"] = len(teams) + out["teams"] = items + } + if include["outside_collaborators"] { + rows, err := d.Svc.ListOutsideCollaborators(r.Context(), org.ID) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(rows)) + for _, row := range rows { + user := transform.User(row.User) + user["organization_member"] = false + user["outside_collaborator"] = true + user["created_at"] = row.CreatedAt.UTC().Format(time.RFC3339) + items = append(items, user) + } + counts["outside_collaborators"] = len(rows) + out["outside_collaborators"] = items + } + out["counts"] = counts + respond.JSON(w, http.StatusOK, out) +} + +// GetRepoSummary handles GET /api/v3/repos/{owner}/{repo}/summary. +func (d *Deps) GetRepoSummary(w http.ResponseWriter, r *http.Request) { + full := repoFullName(r) + repo, err := d.Svc.GetRepo(r.Context(), full) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + include := parseIncludeSet(r, []string{"repo", "viewer", "counts", "labels", "wiki", "agents"}) + out := map[string]any{} + + if include["repo"] { + out["repository"] = transform.Repo(repo, d.repoStats(r, repo)) + } + if include["viewer"] { + out["viewer"] = d.repoViewerSummary(r, repo) + } + if include["counts"] { + aggregates := d.Svc.LoadRepoAggregates(r.Context(), repo.ID) + out["counts"] = map[string]any{ + "open_issues": aggregates.OpenIssuesCount, + "forks": aggregates.ForksCount, + "stargazers": aggregates.StargazersCount, + } + } + if include["labels"] { + labels, err := d.Svc.ListLabels(r.Context(), full) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(labels)) + for _, label := range labels { + items = append(items, transform.Label(label)) + } + out["labels"] = map[string]any{ + "total_count": len(labels), + "items": items, + } + } + if include["wiki"] { + pages, err := d.Svc.ListWikiPages(r.Context(), full, service.ListWikiPagesOptions{Recursive: true}) + if err != nil && !errors.Is(err, service.ErrNotFound) { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(pages)) + for _, page := range pages { + items = append(items, transform.WikiPageSummary(full, page)) + } + out["wiki"] = map[string]any{ + "total_count": len(pages), + "pages": items, + } + } + if include["agents"] { + out["agents"] = d.repoVisibleAgentBindings(r) + } + + respond.JSON(w, http.StatusOK, out) +} + +// GetIssueThread handles GET /api/v3/repos/{owner}/{repo}/issues/{number}/thread. +func (d *Deps) GetIssueThread(w http.ResponseWriter, r *http.Request) { + full := repoFullName(r) + num, ok := mustIntParam(w, r, "number") + if !ok { + return + } + include := parseIncludeSet(r, []string{"issue", "comments", "viewer"}) + out := map[string]any{} + issue, err := d.Svc.GetIssue(r.Context(), full, num) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + + if include["issue"] { + cc := d.Svc.CountIssueComments(r.Context(), issue.RepositoryID, issue.Number) + reactionCounts, err := d.Svc.CountReactions(r.Context(), issue.ID, 0) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + assoc := d.authorAssociationChecks(r.Context(), issue.Repository) + out["issue"] = transform.Issue(issue, d.userResolver(r.Context()), assoc, transform.IssueCounts{ + Comments: cc, + Reactions: reactionCounts, + }) + } + if include["comments"] { + page := intQuery(r, "comments_page", 1) + perPage := intQuery(r, "comments_per_page", 30) + if perPage < 1 { + perPage = 30 + } + if perPage > 100 { + perPage = 100 + } + sortParam := normalizedQueryChoiceAny(r, []string{"comment_sort", "comments_sort"}, "created", []string{"created", "updated"}) + direction := normalizedQueryChoiceAny(r, []string{"comment_direction", "comments_direction"}, "asc", []string{"asc", "desc"}) + comments, total, err := d.Svc.ListIssueCommentsPaginated(r.Context(), full, num, "", sortParam, direction, page, perPage) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + commentIDs := make([]uint, len(comments)) + for i, comment := range comments { + commentIDs[i] = comment.ID + } + allReactions, err := d.Svc.CountReactionsBatchForComments(r.Context(), commentIDs) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + assoc := d.authorAssociationChecks(r.Context(), issue.Repository) + items := make([]any, 0, len(comments)) + for _, comment := range comments { + items = append(items, transform.IssueComment(comment, assoc, allReactions[comment.ID])) + } + out["comments"] = map[string]any{ + "total_count": total, + "page": page, + "per_page": perPage, + "items": items, + } + } + if include["viewer"] { + out["viewer"] = d.repoViewerSummary(r, issue.Repository) + } + respond.JSON(w, http.StatusOK, out) +} + +// BatchGetWikiPages handles POST /api/v3/repos/{owner}/{repo}/wiki/pages/batch. +func (d *Deps) BatchGetWikiPages(w http.ResponseWriter, r *http.Request) { + full := repoFullName(r) + var req struct { + Slugs []string `json:"slugs"` + Include []string `json:"include"` + BodyLimit int `json:"body_limit"` + Ref string `json:"ref"` + } + if err := decodeBodyStrict(r, &req); err != nil { + respond.ValidationFailed(w, "invalid JSON") + return + } + include := includeSliceSet(req.Include) + if len(include) == 0 { + include["body"] = true + include["labels"] = true + } + if len(req.Slugs) == 0 { + respond.ValidationFailed(w, "slugs are required") + return + } + if len(req.Slugs) > maxWikiBatchSlugs { + respond.ValidationFailed(w, fmt.Sprintf("slugs must contain at most %d entries", maxWikiBatchSlugs)) + return + } + bodyLimit := req.BodyLimit + if bodyLimit == 0 { + bodyLimit = defaultWikiBatchBodyLimit + } + if bodyLimit < 0 || bodyLimit > maxWikiBatchBodyLimit { + respond.ValidationFailed(w, fmt.Sprintf("body_limit must be between 0 and %d", maxWikiBatchBodyLimit)) + return + } + + items := make([]any, 0, len(req.Slugs)) + missing := make([]string, 0) + seen := map[string]struct{}{} + for _, rawSlug := range req.Slugs { + slug := strings.TrimSpace(rawSlug) + if slug == "" { + continue + } + if _, ok := seen[slug]; ok { + continue + } + seen[slug] = struct{}{} + page, err := d.Svc.GetWikiPageAtRef(r.Context(), full, slug, strings.TrimSpace(req.Ref)) + if err != nil { + if errors.Is(err, service.ErrNotFound) { + missing = append(missing, slug) + continue + } + respond.ServiceErrorRequest(r, w, err) + return + } + item := transform.WikiPage(full, page) + if !include["body"] { + delete(item, "body") + } else if body, ok := item["body"].(string); ok && bodyLimit > 0 { + item["body"] = truncateRunes(body, bodyLimit) + item["body_truncated"] = len([]rune(body)) > bodyLimit + } + if !include["labels"] { + delete(item, "labels") + } + if include["backlinks"] || include["backlink_count"] { + backlinks, err := d.Svc.ListWikiBacklinks(r.Context(), full, page.Slug) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + item["backlink_count"] = len(backlinks) + if include["backlinks"] { + backlinkItems := make([]any, 0, len(backlinks)) + for _, backlink := range backlinks { + backlinkItems = append(backlinkItems, transform.WikiBacklink(full, backlink)) + } + item["backlinks"] = backlinkItems + } + } + items = append(items, item) + } + respond.JSON(w, http.StatusOK, map[string]any{ + "items": items, + "missing": missing, + "limits": map[string]any{ + "max_slugs": maxWikiBatchSlugs, + "body_limit": bodyLimit, + }, + }) +} + +// GetWikiCatalog handles GET /api/v3/repos/{owner}/{repo}/wiki/catalog. +func (d *Deps) GetWikiCatalog(w http.ResponseWriter, r *http.Request) { + full := repoFullName(r) + include := parseIncludeSet(r, []string{"tree", "pages", "labels"}) + path := strings.Trim(strings.TrimSpace(r.URL.Query().Get("path")), "/") + recursive := strings.TrimSpace(r.URL.Query().Get("recursive")) != "false" + pages, err := d.Svc.ListWikiPages(r.Context(), full, service.ListWikiPagesOptions{ + Path: path, + Recursive: recursive, + Labels: parseCSV(r.URL.Query().Get("labels")), + ExcludeLabels: parseCSV(r.URL.Query().Get("exclude_labels")), + }) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + out := map[string]any{} + if include["pages"] { + items := make([]any, 0, len(pages)) + for _, page := range pages { + items = append(items, transform.WikiPageSummary(full, page)) + } + out["pages"] = items + } + if include["tree"] { + tree, err := d.Svc.ListWikiTreeAtRef(r.Context(), full, path, "") + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + items := make([]any, 0, len(tree)) + for _, entry := range tree { + items = append(items, wikiTreeEntryJSON(full, entry)) + } + out["tree"] = items + } + if include["labels"] { + labelNames := collectWikiCatalogLabels(pages) + out["labels"] = labelNames + } + out["total_count"] = len(pages) + respond.JSON(w, http.StatusOK, out) +} + +// GetNotificationsSummary handles GET /api/v3/notifications/summary. +func (d *Deps) GetNotificationsSummary(w http.ResponseWriter, r *http.Request) { + user, err := d.Svc.GetCurrentUser(r.Context()) + if err != nil { + respond.Error(w, http.StatusUnauthorized, "Bad credentials") + return + } + page, perPage := parsePagination(r) + unreadOnly := true + if allParam := strings.TrimSpace(r.URL.Query().Get("all")); allParam != "" { + all, parseErr := strconv.ParseBool(allParam) + if parseErr != nil { + respond.ValidationFailed(w, "all must be a boolean") + return + } + unreadOnly = !all + } + reasonFilter := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("reason"))) + include := parseIncludeSet(r, []string{"subject", "repository"}) + includeComments := include["latest_comments"] + commentLimit := intQuery(r, "latest_comments_limit", 3) + if commentLimit < 1 { + commentLimit = 1 + } + if commentLimit > 20 { + commentLimit = 20 + } + + notifications, err := d.Svc.ListNotifications(r.Context(), user.ID, unreadOnly, 1000) + if err != nil { + respond.ServiceErrorRequest(r, w, err) + return + } + filtered := make([]db.Notification, 0, len(notifications)) + for _, notification := range notifications { + if reasonFilter != "" && notificationReason(notification.Type) != reasonFilter { + continue + } + filtered = append(filtered, notification) + } + paged := paginate(w, r, d.Svc.BaseURL, filtered, page, perPage) + out := make([]any, 0, len(paged)) + for _, notification := range paged { + item, buildErr := d.notificationJSON(r.Context(), notification) + if buildErr != nil { + continue + } + if !include["subject"] { + delete(item, "subject") + } + if !include["repository"] { + delete(item, "repository") + } + if includeComments && notification.SubjectType == service.NotificationSubjectIssue { + issueNumber := notificationSubjectNumber(r.Context(), d, notification) + if issueNumber > 0 { + comments, _, err := d.Svc.ListIssueCommentsPaginated(r.Context(), notification.Repository.FullName, issueNumber, "", "updated", "desc", 1, commentLimit) + if err == nil { + item["latest_comments"] = latestCommentSummaries(comments) + } + } + } + out = append(out, item) + } + respond.JSON(w, http.StatusOK, out) +} + +func (d *Deps) repoViewerSummary(r *http.Request, repo db.Repository) map[string]any { + viewer, ok := service.UserFromContext(r.Context()) + if !ok || viewer.ID == 0 { + if !repo.Private { + return map[string]any{ + "authenticated": false, + "permission": service.RepoPermissionRead.String(), + "permissions": repoPermissionMapFor(service.RepoPermissionRead), + } + } + return map[string]any{ + "authenticated": false, + "permission": service.RepoPermissionNone.String(), + "permissions": repoPermissionMapFor(service.RepoPermissionNone), + } + } + perm, err := d.Svc.HasRepoAccess(r.Context(), repo.ID, viewer.ID) + if err != nil { + perm = service.RepoPermissionNone + } + return map[string]any{ + "authenticated": true, + "user": transform.User(viewer), + "permission": perm.String(), + "permissions": repoPermissionMapFor(perm), + } +} + +func (d *Deps) repoVisibleAgentBindings(r *http.Request) map[string]any { + viewer, ok := service.UserFromContext(r.Context()) + if !ok || viewer.ID == 0 { + return map[string]any{"total_count": 0, "items": []any{}} + } + agents, err := d.Svc.ListBoundAgents(r.Context(), viewer.ID) + if err != nil { + return map[string]any{"total_count": 0, "items": []any{}} + } + items := make([]any, 0, len(agents)) + for _, agent := range agents { + items = append(items, boundAgentJSON(agent)) + } + return map[string]any{"total_count": len(agents), "items": items} +} + +func parseIncludeSet(r *http.Request, defaults []string) map[string]bool { + raw := strings.TrimSpace(r.URL.Query().Get("include")) + out := map[string]bool{} + if raw == "" { + for _, item := range defaults { + out[item] = true + } + return out + } + for _, item := range parseCSV(raw) { + out[item] = true + } + return out +} + +func includeSliceSet(items []string) map[string]bool { + out := map[string]bool{} + for _, item := range items { + key := strings.ToLower(strings.TrimSpace(item)) + if key != "" { + out[key] = true + } + } + return out +} + +func parseCSV(raw string) []string { + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + seen := map[string]struct{}{} + for _, part := range parts { + item := strings.ToLower(strings.TrimSpace(part)) + if item == "" { + continue + } + if _, ok := seen[item]; ok { + continue + } + seen[item] = struct{}{} + out = append(out, item) + } + return out +} + +func intQuery(r *http.Request, key string, fallback int) int { + raw := strings.TrimSpace(r.URL.Query().Get(key)) + if raw == "" { + return fallback + } + value, err := strconv.Atoi(raw) + if err != nil { + return fallback + } + return value +} + +func normalizedQueryChoice(r *http.Request, key, fallback string, allowed []string) string { + value := strings.ToLower(strings.TrimSpace(r.URL.Query().Get(key))) + if value == "" { + return fallback + } + for _, candidate := range allowed { + if value == candidate { + return value + } + } + return fallback +} + +func normalizedQueryChoiceAny(r *http.Request, keys []string, fallback string, allowed []string) string { + for _, key := range keys { + if strings.TrimSpace(r.URL.Query().Get(key)) == "" { + continue + } + return normalizedQueryChoice(r, key, fallback, allowed) + } + return fallback +} + +func parseRepoAffiliationSet(raw string) (map[string]bool, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + out := map[string]bool{} + for _, item := range parseCSV(raw) { + switch item { + case "owner", "collaborator", "organization_member": + out[item] = true + default: + return nil, fmt.Errorf("repo_affiliation must contain only owner, collaborator, or organization_member") + } + } + return out, nil +} + +func filterViewerReposByAffiliation(repos []service.RepoWithPermission, viewer db.User, affiliation map[string]bool) []service.RepoWithPermission { + if len(affiliation) == 0 { + return repos + } + out := make([]service.RepoWithPermission, 0, len(repos)) + for _, row := range repos { + owner := row.Repository.Owner + isOwner := owner.ID == viewer.ID + isOrgRepo := owner.Type == db.TypeOrganization + isCollaborator := !isOwner && !isOrgRepo + if affiliation["owner"] && isOwner { + out = append(out, row) + continue + } + if affiliation["organization_member"] && isOrgRepo { + out = append(out, row) + continue + } + if affiliation["collaborator"] && isCollaborator { + out = append(out, row) + } + } + return out +} + +func truncateRunes(s string, limit int) string { + runes := []rune(s) + if len(runes) <= limit { + return s + } + return string(runes[:limit]) +} + +func orgMemberSummaryJSON(member service.OrganizationMembershipView) map[string]any { + row := transform.User(member.User) + row["role"] = member.Role + row["state"] = member.State + return row +} + +func wikiTreeEntryJSON(repoFullName string, entry service.WikiTreeEntry) map[string]any { + out := map[string]any{ + "path": entry.Path, + "name": entry.Name, + "type": entry.Kind, + "kind": entry.Kind, + "sha": entry.SHA, + "size": entry.Size, + "title": entry.Title, + } + if entry.Slug != "" { + out["slug"] = entry.Slug + encodedSlug := url.PathEscape(entry.Slug) + out["html_url"] = fmt.Sprintf("%s/%s/wiki/%s", transform.HTMLBase(), repoFullName, encodedSlug) + out["url"] = fmt.Sprintf("%s/repos/%s/wiki/pages/%s", transform.APIBase(), repoFullName, encodedSlug) + } + return out +} + +func collectWikiCatalogLabels(pages []service.WikiPageSummary) []string { + seen := map[string]struct{}{} + for _, page := range pages { + for _, label := range page.Labels { + name := strings.TrimSpace(label.Name) + if name != "" { + seen[name] = struct{}{} + } + } + } + labels := make([]string, 0, len(seen)) + for name := range seen { + labels = append(labels, name) + } + sort.Strings(labels) + return labels +} + +func notificationSubjectNumber(ctx context.Context, d *Deps, notification db.Notification) int { + if notification.SubjectType != service.NotificationSubjectIssue { + return 0 + } + issue, err := d.Svc.GetIssueByID(ctx, notification.SubjectID) + if err != nil { + return 0 + } + return issue.Number +} + +func latestCommentSummaries(comments []db.IssueComment) []any { + items := make([]any, 0, len(comments)) + for _, comment := range comments { + items = append(items, map[string]any{ + "id": comment.ID, + "body": comment.Body, + "user": transform.User(comment.Author), + "created_at": comment.CreatedAt.UTC().Format(time.RFC3339), + "updated_at": comment.UpdatedAt.UTC().Format(time.RFC3339), + }) + } + return items +} diff --git a/internal/rest/handlers_aggregate_test.go b/internal/rest/handlers_aggregate_test.go new file mode 100644 index 0000000..57f0370 --- /dev/null +++ b/internal/rest/handlers_aggregate_test.go @@ -0,0 +1,207 @@ +package rest_test + +import ( + "net/http" + "net/url" + "testing" + + "github.com/ngaut/agent-git-service/internal/testharness" +) + +func TestAggregateViewerAndRepoSummary(t *testing.T) { + h := testharness.New(t) + compatSeedRepo(t, h, "aggregate-summary") + w := h.DoRESTJSON(t, "POST", "/api/v3/user/orgs", map[string]any{"login": "aggregate-viewer-org"}) + assertStatusCode(t, w, http.StatusCreated) + w = h.DoRESTJSON(t, "POST", "/api/v3/orgs/aggregate-viewer-org/repos", map[string]any{ + "name": "metadata", + "auto_init": true, + }) + assertStatusCode(t, w, http.StatusCreated) + + w = h.DoREST(t, "GET", "/api/v3/viewer/summary?include=user,repositories&per_page=1", nil) + assertStatusCode(t, w, http.StatusOK) + viewerSummary := testharness.DecodeJSON(t, w) + user := viewerSummary["user"].(map[string]any) + if user["login"] != "testuser" { + t.Fatalf("viewer login = %v, want testuser", user["login"]) + } + repos := viewerSummary["repositories"].(map[string]any) + if repos["total_count"].(float64) < 1 { + t.Fatalf("viewer repositories total_count = %v, want at least 1", repos["total_count"]) + } + items := repos["items"].([]any) + if len(items) != 1 { + t.Fatalf("viewer repositories page size = %d, want 1", len(items)) + } + w = h.DoREST(t, "GET", "/api/v3/viewer/summary?include=repositories&repo_affiliation=organization_member", nil) + assertStatusCode(t, w, http.StatusOK) + orgRepoSummary := testharness.DecodeJSON(t, w) + orgRepos := orgRepoSummary["repositories"].(map[string]any) + if orgRepos["total_count"].(float64) != 1 { + t.Fatalf("organization_member repos total_count = %v, want 1", orgRepos["total_count"]) + } + orgRepo := orgRepos["items"].([]any)[0].(map[string]any) + if orgRepo["full_name"] != "aggregate-viewer-org/metadata" { + t.Fatalf("organization_member repo = %v, want aggregate-viewer-org/metadata", orgRepo["full_name"]) + } + + w = h.DoREST(t, "GET", "/api/v3/repos/testuser/aggregate-summary/summary?include=repo,viewer,counts", nil) + assertStatusCode(t, w, http.StatusOK) + repoSummary := testharness.DecodeJSON(t, w) + repo := repoSummary["repository"].(map[string]any) + if repo["full_name"] != "testuser/aggregate-summary" { + t.Fatalf("repo full_name = %v, want testuser/aggregate-summary", repo["full_name"]) + } + viewer := repoSummary["viewer"].(map[string]any) + if viewer["permission"] != "admin" { + t.Fatalf("repo viewer permission = %v, want admin", viewer["permission"]) + } + if _, ok := repoSummary["counts"].(map[string]any); !ok { + t.Fatalf("repo summary missing counts: %#v", repoSummary) + } +} + +func TestAggregateIssueThreadAndIssueListFilters(t *testing.T) { + h := testharness.New(t) + compatSeedRepo(t, h, "aggregate-issues") + + w := h.DoRESTJSON(t, "POST", "/api/v3/repos/testuser/aggregate-issues/issues", map[string]any{ + "title": "Run: build worker", + "body": "worker body", + }) + assertStatusCode(t, w, http.StatusCreated) + w = h.DoRESTJSON(t, "POST", "/api/v3/repos/testuser/aggregate-issues/issues/1/comments", map[string]any{ + "body": "first comment", + }) + assertStatusCode(t, w, http.StatusCreated) + w = h.DoRESTJSON(t, "POST", "/api/v3/repos/testuser/aggregate-issues/issues", map[string]any{ + "title": "Chat: general", + "body": "chat body", + }) + assertStatusCode(t, w, http.StatusCreated) + + w = h.DoREST(t, "GET", "/api/v3/repos/testuser/aggregate-issues/issues/1/thread?comments_per_page=10", nil) + assertStatusCode(t, w, http.StatusOK) + thread := testharness.DecodeJSON(t, w) + issue := thread["issue"].(map[string]any) + if issue["title"] != "Run: build worker" { + t.Fatalf("thread issue title = %v, want Run: build worker", issue["title"]) + } + comments := thread["comments"].(map[string]any) + if comments["total_count"].(float64) != 1 { + t.Fatalf("thread comments total_count = %v, want 1", comments["total_count"]) + } + + query := url.Values{} + query.Set("kind", "issue") + query.Set("title_prefix", "Run: ") + query.Set("include", "body") + query.Set("fields", "number,title,body") + w = h.DoREST(t, "GET", "/api/v3/repos/testuser/aggregate-issues/issues?"+query.Encode(), nil) + assertStatusCode(t, w, http.StatusOK) + rows := testharness.DecodeJSONArray(t, w) + if len(rows) != 1 { + t.Fatalf("filtered issue rows = %d, want 1; body: %s", len(rows), w.Body.String()) + } + row := rows[0] + if row["title"] != "Run: build worker" || row["body"] != "worker body" { + t.Fatalf("filtered issue row = %#v", row) + } + if _, ok := row["user"]; ok { + t.Fatalf("fields filter should omit user: %#v", row) + } +} + +func TestAggregateWikiCatalogAndBatch(t *testing.T) { + h := testharness.New(t) + compatSeedRepo(t, h, "aggregate-wiki") + full := "testuser/aggregate-wiki" + + w := h.DoRESTJSON(t, "POST", "/api/v3/repos/"+full+"/labels", map[string]any{ + "name": "ops", + "color": "8be9fd", + "description": "Operations", + }) + assertStatusCode(t, w, http.StatusCreated) + w = h.DoRESTJSON(t, "PUT", wikiPagePath(full, "home"), map[string]any{ + "body": "# Home\n\nwelcome", + }) + assertStatusCode(t, w, http.StatusOK) + w = h.DoRESTJSON(t, "PUT", wikiPagePath(full, "guides/setup"), map[string]any{ + "body": "# Setup\n\n0123456789abcdefghijklmnopqrstuvwxyz", + }) + assertStatusCode(t, w, http.StatusOK) + h.Svc.Wg.Wait() + + w = h.DoRESTJSON(t, "POST", wikiPageSubresourcePath(full, "guides/setup", "labels"), map[string]any{ + "labels": []string{"ops"}, + }) + assertStatusCode(t, w, http.StatusOK) + h.Svc.Wg.Wait() + + w = h.DoREST(t, "GET", "/api/v3/repos/"+full+"/wiki/catalog?include=pages,tree,labels&path=guides", nil) + assertStatusCode(t, w, http.StatusOK) + catalog := testharness.DecodeJSON(t, w) + if catalog["total_count"].(float64) != 1 { + t.Fatalf("catalog total_count = %v, want 1", catalog["total_count"]) + } + labels := catalog["labels"].([]any) + if len(labels) != 1 || labels[0] != "ops" { + t.Fatalf("catalog labels = %#v, want [ops]", labels) + } + + w = h.DoRESTJSON(t, "POST", "/api/v3/repos/"+full+"/wiki/pages/batch", map[string]any{ + "slugs": []string{"guides/setup", "missing"}, + "include": []string{"body", "labels"}, + "body_limit": 8, + }) + assertStatusCode(t, w, http.StatusOK) + batch := testharness.DecodeJSON(t, w) + items := batch["items"].([]any) + if len(items) != 1 { + t.Fatalf("batch items = %d, want 1", len(items)) + } + item := items[0].(map[string]any) + if item["slug"] != "guides/setup" { + t.Fatalf("batch item slug = %v, want guides/setup", item["slug"]) + } + if item["body_truncated"] != true { + t.Fatalf("batch item body_truncated = %v, want true", item["body_truncated"]) + } + missing := batch["missing"].([]any) + if len(missing) != 1 || missing[0] != "missing" { + t.Fatalf("batch missing = %#v, want [missing]", missing) + } +} + +func TestAggregateOrgManagementSummary(t *testing.T) { + h := testharness.New(t) + + w := h.DoRESTJSON(t, "POST", "/api/v3/user/orgs", map[string]any{"login": "aggregate-org"}) + assertStatusCode(t, w, http.StatusCreated) + w = h.DoRESTJSON(t, "POST", "/api/v3/orgs/aggregate-org/repos", map[string]any{ + "name": "metadata", + "auto_init": true, + }) + assertStatusCode(t, w, http.StatusCreated) + + w = h.DoREST(t, "GET", "/api/v3/orgs/aggregate-org/management-summary?include=org,viewer,repos,members,teams,invitations,outside_collaborators", nil) + assertStatusCode(t, w, http.StatusOK) + summary := testharness.DecodeJSON(t, w) + org := summary["organization"].(map[string]any) + if org["login"] != "aggregate-org" { + t.Fatalf("organization login = %v, want aggregate-org", org["login"]) + } + counts := summary["counts"].(map[string]any) + if counts["repos"].(float64) != 1 { + t.Fatalf("org repos count = %v, want 1", counts["repos"]) + } + if counts["members"].(float64) != 1 { + t.Fatalf("org members count = %v, want 1", counts["members"]) + } + viewer := summary["viewer"].(map[string]any) + if viewer["role"] != "admin" { + t.Fatalf("viewer org role = %v, want admin", viewer["role"]) + } +} diff --git a/internal/rest/handlers_issue_list.go b/internal/rest/handlers_issue_list.go index ed04ee6..e57186a 100644 --- a/internal/rest/handlers_issue_list.go +++ b/internal/rest/handlers_issue_list.go @@ -29,6 +29,10 @@ type issueListParams struct { assignee string creator string mentioned string + kind string + titlePrefix string + includeBody bool + fields []string sort string direction string milestone string @@ -68,13 +72,15 @@ func (d *Deps) ListIssues(w http.ResponseWriter, r *http.Request) { RepoFullName: params.repoFullName, State: params.state, Labels: params.labels, + Kind: params.kind, + TitlePrefix: params.titlePrefix, Sort: params.sort, Direction: params.direction, Milestone: params.milestone, Since: params.since, Page: page, PerPage: perPage, - OmitIssueBody: true, + OmitIssueBody: !params.includeBody, }) if err != nil { respond.ServiceErrorRequest(r, w, err) @@ -90,6 +96,7 @@ func (d *Deps) ListIssues(w http.ResponseWriter, r *http.Request) { respond.ServiceErrorRequest(r, w, err) return } + out = filterIssueListResponseFields(out, params.fields) respond.JSON(w, 200, out) } @@ -100,6 +107,7 @@ func (d *Deps) listIssuesLegacy(w http.ResponseWriter, r *http.Request, params * return } items := mergeIssuesAndPRs(issues, prs) + items = filterIssueListItems(items, params) if err := d.countCommentsForItems(r.Context(), items); err != nil { respond.ServiceErrorRequest(r, w, err) return @@ -122,6 +130,7 @@ func (d *Deps) listIssuesLegacy(w http.ResponseWriter, r *http.Request, params * respond.ServiceErrorRequest(r, w, err) return } + out = filterIssueListResponseFields(out, params.fields) respond.JSON(w, 200, out) } @@ -252,6 +261,21 @@ func parseIssueListParams(r *http.Request) (*issueListParams, error) { assignee := r.URL.Query().Get("assignee") creator := r.URL.Query().Get("creator") mentioned := r.URL.Query().Get("mentioned") + kind := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("kind"))) + if kind == "" { + kind = "all" + } + if kind == "pr" { + kind = "pull" + } + switch kind { + case "issue", "pull", "all": + default: + return nil, validationError("kind must be one of: issue, pull, all") + } + titlePrefix := strings.TrimSpace(r.URL.Query().Get("title_prefix")) + includeBody := queryListContains(r.URL.Query().Get("include"), "body") + fields := parseIssueListFields(r.URL.Query().Get("fields")) sortParam := strings.TrimSpace(r.URL.Query().Get("sort")) if sortParam != "" { sortParam = strings.ToLower(sortParam) @@ -292,6 +316,10 @@ func parseIssueListParams(r *http.Request) (*issueListParams, error) { assignee: assignee, creator: creator, mentioned: mentioned, + kind: kind, + titlePrefix: titlePrefix, + includeBody: includeBody, + fields: fields, sort: sortParam, direction: direction, milestone: milestone, @@ -304,45 +332,119 @@ func parseIssueListParams(r *http.Request) (*issueListParams, error) { func (d *Deps) fetchIssuesAndPRs(ctx context.Context, params *issueListParams) ([]db.Issue, []db.PullRequest, error) { var issues []db.Issue var err error - if params.assignee != "" || params.creator != "" || params.mentioned != "" { - issues, err = d.Svc.ListIssuesFiltered(ctx, service.IssueListFilter{ + if params.kind != "pull" { + if params.assignee != "" || params.creator != "" || params.mentioned != "" { + issues, err = d.Svc.ListIssuesFiltered(ctx, service.IssueListFilter{ + RepoFullName: params.repoFullName, + State: params.state, + Assignee: params.assignee, + Mentioned: params.mentioned, + CreatedBy: params.creator, + Labels: params.labels, + Sort: params.sort, + Direction: params.direction, + Milestone: params.milestone, + Since: params.since, + }) + } else { + issues, err = d.Svc.ListIssuesForREST(ctx, params.repoFullName, params.state, params.labels, params.sort, params.direction, params.milestone, params.since) + } + if err != nil { + return nil, nil, err + } + } + + var prs []db.PullRequest + if params.kind != "issue" { + prs, err = d.Svc.ListPRsFiltered(ctx, service.PRListFilter{ RepoFullName: params.repoFullName, State: params.state, - Assignee: params.assignee, Mentioned: params.mentioned, - CreatedBy: params.creator, - Labels: params.labels, - Sort: params.sort, - Direction: params.direction, - Milestone: params.milestone, - Since: params.since, }) - } else { - issues, err = d.Svc.ListIssuesForREST(ctx, params.repoFullName, params.state, params.labels, params.sort, params.direction, params.milestone, params.since) + if err != nil { + return nil, nil, err + } + labelFilters := parseFilterList(params.labels) + if len(prs) > 0 { + var invalidMilestone bool + prs, invalidMilestone = filterPRs(prs, labelFilters, params.assignee, params.creator, params.milestone, params.sinceTime, params.hasSince) + if invalidMilestone { + prs = nil + } + } } - if err != nil { - return nil, nil, err + + return issues, prs, nil +} + +func queryListContains(raw, wanted string) bool { + wanted = strings.ToLower(strings.TrimSpace(wanted)) + for _, part := range strings.Split(raw, ",") { + if strings.ToLower(strings.TrimSpace(part)) == wanted { + return true + } } + return false +} - var prs []db.PullRequest - prs, err = d.Svc.ListPRsFiltered(ctx, service.PRListFilter{ - RepoFullName: params.repoFullName, - State: params.state, - Mentioned: params.mentioned, - }) - if err != nil { - return nil, nil, err +func parseIssueListFields(raw string) []string { + parts := strings.Split(raw, ",") + fields := make([]string, 0, len(parts)) + seen := make(map[string]struct{}) + for _, part := range parts { + field := strings.TrimSpace(part) + if field == "" { + continue + } + if _, ok := seen[field]; ok { + continue + } + seen[field] = struct{}{} + fields = append(fields, field) } - labelFilters := parseFilterList(params.labels) - if len(prs) > 0 { - var invalidMilestone bool - prs, invalidMilestone = filterPRs(prs, labelFilters, params.assignee, params.creator, params.milestone, params.sinceTime, params.hasSince) - if invalidMilestone { - prs = nil + return fields +} + +func filterIssueListItems(items []issueListItem, params *issueListParams) []issueListItem { + if params.titlePrefix == "" { + return items + } + out := make([]issueListItem, 0, len(items)) + for _, item := range items { + title := "" + if item.issue != nil { + title = item.issue.Title + } + if item.pr != nil { + title = item.pr.Title + } + if strings.HasPrefix(strings.ToLower(title), strings.ToLower(params.titlePrefix)) { + out = append(out, item) } } + return out +} - return issues, prs, nil +func filterIssueListResponseFields(rows []any, fields []string) []any { + if len(fields) == 0 { + return rows + } + out := make([]any, len(rows)) + for i, row := range rows { + src, ok := row.(map[string]any) + if !ok { + out[i] = row + continue + } + dst := make(map[string]any, len(fields)) + for _, field := range fields { + if value, ok := src[field]; ok { + dst[field] = value + } + } + out[i] = dst + } + return out } func mergeIssuesAndPRs(issues []db.Issue, prs []db.PullRequest) []issueListItem { diff --git a/internal/router/router.go b/internal/router/router.go index a8c2fd8..75075b0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -329,6 +329,7 @@ func registerAgentBindingRoutes(r chi.Router, handlers *rest.Deps) { func registerUserScopedRoutes(r chi.Router, handlers *rest.Deps) { // Current user r.Get("/api/v3/user", handlers.GetAuthenticatedUser) + r.Get("/api/v3/viewer/summary", handlers.GetViewerSummary) r.Post("/api/v3/user/repos", handlers.CreateUserRepo) r.Get("/api/v3/user/repos", handlers.ListUserRepos) r.Get("/api/v3/user/orgs", handlers.ListUserOrgs) @@ -371,6 +372,7 @@ func registerUserScopedRoutes(r chi.Router, handlers *rest.Deps) { // Notifications r.Get("/api/v3/notifications", handlers.ListNotifications) + r.Get("/api/v3/notifications/summary", handlers.GetNotificationsSummary) r.Put("/api/v3/notifications", handlers.MarkNotificationsRead) // Repository Invitations (user-specific) @@ -396,6 +398,7 @@ func registerUserLookupRoutes(r chi.Router, handlers *rest.Deps) { func registerOrgRoutes(r chi.Router, handlers *rest.Deps) { // Orgs r.Get("/api/v3/orgs/{org}", handlers.GetOrg) + r.Get("/api/v3/orgs/{org}/management-summary", handlers.GetOrgManagementSummary) r.Get("/api/v3/orgs/{org}/members", handlers.ListOrgMembers) r.Delete("/api/v3/orgs/{org}/members/{username}", handlers.DeleteOrgMember) r.Put("/api/v3/orgs/{org}/memberships/{username}", handlers.SetOrgMembership) @@ -457,12 +460,14 @@ func registerRepoWikiRoutes(r chi.Router, handlers *rest.Deps) { r.Post("/api/v3/admin/wiki/repos/{owner}/{repo}/repair-locks", handlers.RepairWikiLocks) r.Get("/api/v3/repos/{owner}/{repo}/wiki/state", handlers.GetWikiState) r.Get("/api/v3/repos/{owner}/{repo}/wiki/tree", handlers.ListWikiTree) + r.Get("/api/v3/repos/{owner}/{repo}/wiki/catalog", handlers.GetWikiCatalog) r.Post("/api/v3/repos/{owner}/{repo}/wiki/reconcile/request", handlers.RequestWikiReconcile) r.Post("/api/v3/repos/{owner}/{repo}/wiki/reconcile", handlers.ReconcileWiki) r.Post("/api/v3/repos/{owner}/{repo}/wiki/compact", handlers.CompactWikiHistory) r.Get("/api/v3/repos/{owner}/{repo}/wiki/compact/{jobID}", handlers.GetWikiCompactionJob) r.Post("/api/v3/repos/{owner}/{repo}/wiki/move", handlers.MoveWikiPagePrefix) r.Get("/api/v3/repos/{owner}/{repo}/wiki/pages", handlers.ListWikiPages) + r.Post("/api/v3/repos/{owner}/{repo}/wiki/pages/batch", handlers.BatchGetWikiPages) r.Get("/api/v3/repos/{owner}/{repo}/wiki/search", handlers.SearchWikiPages) r.Get("/api/v3/repos/{owner}/{repo}/wiki/pages/{slug}/labels", handlers.ListWikiPageLabels) r.Post("/api/v3/repos/{owner}/{repo}/wiki/pages/{slug}/labels", handlers.AddWikiPageLabels) @@ -479,6 +484,7 @@ func registerRepoWikiRoutes(r chi.Router, handlers *rest.Deps) { func registerRepoCoreRoutes(r chi.Router, handlers *rest.Deps) { // Repos + r.Get("/api/v3/repos/{owner}/{repo}/summary", handlers.GetRepoSummary) r.Get("/api/v3/repos/{owner}/{repo}", handlers.GetRepo) r.Patch("/api/v3/repos/{owner}/{repo}", handlers.UpdateRepo) r.Delete("/api/v3/repos/{owner}/{repo}", handlers.DeleteRepo) @@ -541,6 +547,7 @@ func registerIssueRoutes(r chi.Router, handlers *rest.Deps) { r.Post("/api/v3/repos/{owner}/{repo}/issues", handlers.CreateIssue) r.Get("/api/v3/repos/{owner}/{repo}/issues/comments", handlers.ListRepoIssueComments) r.Get("/api/v3/repos/{owner}/{repo}/issues/comments/{comment_id}", handlers.GetIssueComment) + r.Get("/api/v3/repos/{owner}/{repo}/issues/{number}/thread", handlers.GetIssueThread) r.Get("/api/v3/repos/{owner}/{repo}/issues/{number}", handlers.GetIssue) r.Patch("/api/v3/repos/{owner}/{repo}/issues/{number}", handlers.UpdateIssue) r.Get("/api/v3/repos/{owner}/{repo}/issues/{number}/comments", handlers.ListIssueComments) diff --git a/internal/service/issue_list_page.go b/internal/service/issue_list_page.go index 43b2ca3..51af438 100644 --- a/internal/service/issue_list_page.go +++ b/internal/service/issue_list_page.go @@ -16,6 +16,8 @@ type IssueListPageFilter struct { RepoFullName string State string Labels string + Kind string + TitlePrefix string Sort string Direction string Milestone string @@ -140,13 +142,15 @@ func (s *Service) countIssueListPageRows(ctx context.Context, repoID uint, filte } type normalizedIssueListPageFilter struct { - state string - sort string - direction string - milestone string - since *time.Time - page int - perPage int + state string + kind string + titlePrefix string + sort string + direction string + milestone string + since *time.Time + page int + perPage int } func normalizeIssueListPageFilter(filter IssueListPageFilter) (normalizedIssueListPageFilter, error) { @@ -154,6 +158,14 @@ func normalizeIssueListPageFilter(filter IssueListPageFilter) (normalizedIssueLi if state == "" { state = db.StateOpen } + kind := strings.ToLower(strings.TrimSpace(filter.Kind)) + switch kind { + case "", "all": + kind = "all" + case "issue", "pull": + default: + return normalizedIssueListPageFilter{}, fmt.Errorf("%w: kind must be one of: issue, pull, all", ErrValidation) + } sortKey := strings.ToLower(strings.TrimSpace(filter.Sort)) switch sortKey { case "", "created": @@ -186,13 +198,15 @@ func normalizeIssueListPageFilter(filter IssueListPageFilter) (normalizedIssueLi since = &parsed } return normalizedIssueListPageFilter{ - state: state, - sort: sortKey, - direction: direction, - milestone: strings.TrimSpace(filter.Milestone), - since: since, - page: page, - perPage: perPage, + state: state, + kind: kind, + titlePrefix: strings.TrimSpace(filter.TitlePrefix), + sort: sortKey, + direction: direction, + milestone: strings.TrimSpace(filter.Milestone), + since: since, + page: page, + perPage: perPage, }, nil } @@ -243,10 +257,19 @@ func splitIssueListPageLabels(raw string) []string { } func buildIssueListPageQuery(repoID uint, filter normalizedIssueListPageFilter, labelNames []string, labelIDsByName map[string][]uint, paginate bool, includeComments bool, limit int) (string, []any) { - issueSQL, issueArgs := buildIssueListPageEntitySQL("issue", "issues", repoID, filter, labelNames, labelIDsByName, includeComments) - prSQL, prArgs := buildIssueListPageEntitySQL("pr", "pull_requests", repoID, filter, labelNames, labelIDsByName, includeComments) - args := append(issueArgs, prArgs...) - unionSQL := issueSQL + " UNION ALL " + prSQL + parts := make([]string, 0, 2) + args := make([]any, 0) + if filter.kind == "all" || filter.kind == "issue" { + issueSQL, issueArgs := buildIssueListPageEntitySQL("issue", "issues", repoID, filter, labelNames, labelIDsByName, includeComments) + parts = append(parts, issueSQL) + args = append(args, issueArgs...) + } + if filter.kind == "all" || filter.kind == "pull" { + prSQL, prArgs := buildIssueListPageEntitySQL("pr", "pull_requests", repoID, filter, labelNames, labelIDsByName, includeComments) + parts = append(parts, prSQL) + args = append(args, prArgs...) + } + unionSQL := strings.Join(parts, " UNION ALL ") if !paginate { return "SELECT COUNT(*) FROM (" + unionSQL + ") AS combined", args } @@ -299,6 +322,10 @@ func buildIssueListPageEntitySQL(kind, table string, repoID uint, filter normali where = append(where, table+".updated_at >= ?") args = append(args, *filter.since) } + if filter.titlePrefix != "" { + where = append(where, "LOWER("+table+".title) LIKE ? ESCAPE '\\'") + args = append(args, strings.ToLower(escapeSQLLike(filter.titlePrefix))+"%") + } where, args = appendIssueListPageMilestoneWhere(where, args, table, repoID, filter.milestone) where, args = appendIssueListPageLabelWhere(where, args, table, labelNames, labelIDsByName) @@ -315,6 +342,13 @@ func buildIssueListPageEntitySQL(kind, table string, repoID uint, filter normali ), args } +func escapeSQLLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + func appendIssueListPageMilestoneWhere(where []string, args []any, table string, repoID uint, rawMilestone string) ([]string, []any) { milestone := strings.ToLower(strings.TrimSpace(rawMilestone)) switch milestone {