diff --git a/.github/workflows/test-and-build-artifacts.yml b/.github/workflows/test-and-build-artifacts.yml index 954e917..4113f74 100644 --- a/.github/workflows/test-and-build-artifacts.yml +++ b/.github/workflows/test-and-build-artifacts.yml @@ -11,7 +11,7 @@ on: - "go.sum" - "CHANGELOG.md" - "broker/**" - - "www/**" + - "internal/web/www/**" - "testsuite/**" - ".github/workflows/test-and-build-artifacts.yml" - ".github/workflows/test.yml" @@ -26,7 +26,7 @@ on: - "go.sum" - "CHANGELOG.md" - "broker/**" - - "www/**" + - "internal/web/www/**" - "testsuite/**" - ".github/workflows/test-and-build-artifacts.yml" - ".github/workflows/test.yml" @@ -75,6 +75,19 @@ jobs: - name: Unit Tests run: go test ./... + - name: External SDK Consumer + working-directory: testsuite/sdk-consumer + run: go test ./... + + - name: Vet Public Packages And CLI + run: go vet ./... + + - name: Verify Exported API Inventory + run: go run ./tools/api_inventory -check API.md + + - name: Broker Protocol Fixtures + run: node broker/test_support/signing_fixture_test.js + build: name: Build Multiarchitecture Artifacts needs: @@ -214,7 +227,7 @@ jobs: $env:GOARCH = $target.GOARCH $env:CGO_ENABLED = "0" - go build -trimpath -ldflags="-s -w -X main.version=$env:APP_VERSION" -o "dist/$artifact" . + go build -trimpath -ldflags="-s -w -X github.com/bucketgit/bgit.version=$env:APP_VERSION" -o "dist/$artifact" ./cmd/bgit if (!(Test-Path "dist/$artifact")) { throw "Missing expected artifact: dist/$artifact" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d39e5a8..820a599 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,6 +42,19 @@ jobs: - name: Unit Tests run: go test ./... + - name: External SDK Consumer + working-directory: testsuite/sdk-consumer + run: go test ./... + + - name: Vet Public Packages And CLI + run: go vet ./... + + - name: Verify Exported API Inventory + run: go run ./tools/api_inventory -check API.md + + - name: Broker Protocol Fixtures + run: node broker/test_support/signing_fixture_test.js + local-broker-integration: name: Local Broker Integration (${{ matrix.runtime }} / ${{ matrix.os }}) runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 02fc5ac..41e6b09 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ go.work.sum # .vscode/ bgit +!cmd/ +!cmd/bgit/ +!cmd/bgit/main.go *.test *.out coverage.txt diff --git a/API.md b/API.md new file mode 100644 index 0000000..72c8949 --- /dev/null +++ b/API.md @@ -0,0 +1,448 @@ +# BucketGit Exported Go API + +Generated by `go run ./tools/api_inventory`. This is the public BucketGit Go SDK v1 compatibility surface. + +## `github.com/bucketgit/bgit/protocol` + +- `const CIBuilding` +- `const CIFinished` +- `const CIQueued` +- `const HeaderKey` +- `const HeaderKeyFingerprint` +- `const HeaderNonce` +- `const HeaderSignature` +- `const HeaderSignatureMessage` +- `const HeaderSignatureVersion` +- `const HeaderSignedHost` +- `const HeaderTimestamp` +- `const LaneBacklog` +- `const LaneDoing` +- `const LaneDone` +- `const LaneReady` +- `const LaneReview` +- `const OperationDelete` +- `const OperationMerge` +- `const OperationRead` +- `const OperationWrite` +- `const ProviderFile` +- `const ProviderGCS` +- `const ProviderS3` +- `const PullRequestClosed` +- `const PullRequestMerged` +- `const PullRequestOpen` +- `const RoleAdmin` +- `const RoleDeveloper` +- `const RoleMaintainer` +- `const RoleOwner` +- `const RoleRead` +- `const RoleTriage` +- `const SignaturePrefix` +- `const SignatureVersion` +- `func SignatureMessage` +- `func ValidateRequestMetadata` +- `method BrokerError.Error` +- `method BrokerError.Unwrap` +- `method Repository.Validate` +- `method Role.Valid` +- `type AWSCredentials` +- `type AdminRepositoryInfoResponse` +- `type AuthRequest` +- `type AuthResponse` +- `type AuthStatus` +- `type AuthStatusRequest` +- `type BrokerError` +- `type CILogResponse` +- `type CIRequest` +- `type CIRun` +- `type CIStatus` +- `type Identity` +- `type Issue` +- `type IssueEvent` +- `type IssueReply` +- `type IssueRequest` +- `type Key` +- `type KeyRequest` +- `type KeysResponse` +- `type Lane` +- `type ObjectCapabilityRequest` +- `type ObjectCapabilityResponse` +- `type ObjectRequest` +- `type ObjectResponse` +- `type Operation` +- `type OwnerRequest` +- `type OwnerTransferRequest` +- `type OwnerTransferResponse` +- `type Protection` +- `type Provider` +- `type PullRequest` +- `type PullRequestComment` +- `type PullRequestNote` +- `type PullRequestRequest` +- `type PullRequestResponse` +- `type PullRequestStatus` +- `type PullRequestsResponse` +- `type RefUpdateRequest` +- `type RefsRequest` +- `type RefsResponse` +- `type RepositoriesMineResponse` +- `type Repository` +- `type RepositoryAdminRequest` +- `type RepositoryInfo` +- `type RepositoryInfoRequest` +- `type RepositoryInfoResponse` +- `type RepositoryInvite` +- `type RepositoryInvitesResponse` +- `type RepositoryListResponse` +- `type RepositoryMembership` +- `type RepositoryRequest` +- `type RepositoryTeamGrant` +- `type RepositoryTeamsResponse` +- `type RepositoryUserGrant` +- `type RepositoryUsersResponse` +- `type RequestMetadata` +- `type Role` +- `type Team` +- `type TeamMember` +- `type TeamsResponse` +- `type UserInfo` +- `type UserProfile` +- `type UserProfileKey` +- `type UserProfileRequest` +- `type UserProfileResponse` +- `type UsersResponse` +- `var ErrConflict` +- `var ErrReplay` +- `var ErrSignatureExpired` +- `var ErrUnauthorized` +- `var ErrUnsupported` + +## `github.com/bucketgit/bgit/store` + +- `func IsZeroOID` +- `func ReadRefs` +- `func ValidSHA1` +- `func ValidatePath` +- `func ValidateRefUpdate` +- `method FallbackReader.List` +- `method FallbackReader.Read` +- `type CompareAndSwapper` +- `type FallbackReader` +- `type ObjectState` +- `type Reader` +- `type RefStore` +- `type Writer` +- `var ErrConflict` +- `var ErrInvalidPath` +- `var ErrNotFound` +- `var ErrNotSupported` +- `var ErrReadOnly` + +## `github.com/bucketgit/bgit/store/fs` + +- `func New` +- `method Store.CompareAndSwap` +- `method Store.CompareAndSwapRef` +- `method Store.Delete` +- `method Store.List` +- `method Store.ListRefs` +- `method Store.Read` +- `method Store.Write` +- `type Store` + +## `github.com/bucketgit/bgit/store/s3` + +- `func Load` +- `func New` +- `method Store.CompareAndSwap` +- `method Store.CompareAndSwapRef` +- `method Store.Delete` +- `method Store.List` +- `method Store.ListRefs` +- `method Store.Read` +- `method Store.Write` +- `type API` +- `type Options` +- `type Store` + +## `github.com/bucketgit/bgit/store/gcs` + +- `func Load` +- `func New` +- `func NewWithAPI` +- `method Store.Close` +- `method Store.CompareAndSwap` +- `method Store.CompareAndSwapRef` +- `method Store.Delete` +- `method Store.List` +- `method Store.ListRefs` +- `method Store.Read` +- `method Store.Write` +- `type API` +- `type ObjectAttrs` +- `type Options` +- `type Store` + +## `github.com/bucketgit/bgit/repository` + +- `const Added` +- `const BlobObject` +- `const CommitObject` +- `const DefaultMaxObjectSize` +- `const Deleted` +- `const Modified` +- `const TagObject` +- `const TreeObject` +- `func Open` +- `func ParseCommitData` +- `func ParseOID` +- `func WithMaxObjectSize` +- `func WithObjectVerification` +- `method OID.String` +- `method Repository.ArchiveTar` +- `method Repository.Commit` +- `method Repository.Diff` +- `method Repository.IsAncestor` +- `method Repository.ListRefs` +- `method Repository.MergeBase` +- `method Repository.Object` +- `method Repository.ReachableObjects` +- `method Repository.Resolve` +- `method Repository.Snapshot` +- `method Repository.Tree` +- `method Repository.TreeEntries` +- `type Change` +- `type ChangeStatus` +- `type Commit` +- `type File` +- `type OID` +- `type Object` +- `type ObjectType` +- `type Option` +- `type Repository` +- `type Signature` +- `type TreeEntry` + +## `github.com/bucketgit/bgit/broker/client` + +- `func New` +- `func NewEndpoints` +- `func NewV2Signatures` +- `func RandomNonce` +- `method Client.AuthStatus` +- `method Client.Authorize` +- `method Client.ListCIRuns` +- `method Client.ListIssues` +- `method Client.ListPullRequests` +- `method Client.ListRefs` +- `method Client.ObjectCapability` +- `method Client.PostJSON` +- `method Client.PostJSONBytes` +- `method Client.UpdateRef` +- `method Endpoints.AcceptBrokerUserInvite` +- `method Endpoints.AcceptOwnerTransfer` +- `method Endpoints.AcceptRepositoryInvite` +- `method Endpoints.AddKey` +- `method Endpoints.ArchiveIssue` +- `method Endpoints.AssignIssue` +- `method Endpoints.AuthStatus` +- `method Endpoints.Authorize` +- `method Endpoints.CILogs` +- `method Endpoints.CancelBrokerUserInvite` +- `method Endpoints.CancelOwnerTransfer` +- `method Endpoints.CancelRepositoryInvite` +- `method Endpoints.CloseIssue` +- `method Endpoints.ClosePullRequest` +- `method Endpoints.CommentIssue` +- `method Endpoints.CommentPullRequest` +- `method Endpoints.ConfirmOwnerTransfer` +- `method Endpoints.CreateBrokerUserInvite` +- `method Endpoints.CreateIssue` +- `method Endpoints.CreatePullRequest` +- `method Endpoints.CreateRepository` +- `method Endpoints.CreateRepositoryInvite` +- `method Endpoints.CreateTeam` +- `method Endpoints.DeleteRepository` +- `method Endpoints.DeleteTeam` +- `method Endpoints.DeleteUser` +- `method Endpoints.GetCIRun` +- `method Endpoints.GetIssue` +- `method Endpoints.GetPullRequest` +- `method Endpoints.GetRepository` +- `method Endpoints.GetUserProfile` +- `method Endpoints.IssueAssignees` +- `method Endpoints.ListCIRuns` +- `method Endpoints.ListIssues` +- `method Endpoints.ListKeys` +- `method Endpoints.ListProtections` +- `method Endpoints.ListPullRequests` +- `method Endpoints.ListRefs` +- `method Endpoints.ListRepositories` +- `method Endpoints.ListRepositoryInvites` +- `method Endpoints.ListRepositoryTeams` +- `method Endpoints.ListRepositoryUsers` +- `method Endpoints.ListTeams` +- `method Endpoints.ListUsers` +- `method Endpoints.MergePullRequest` +- `method Endpoints.MoveIssue` +- `method Endpoints.ObjectCapability` +- `method Endpoints.ReindexRepositoryMembers` +- `method Endpoints.RemoveKey` +- `method Endpoints.RemoveProtection` +- `method Endpoints.RemoveRepositoryTeam` +- `method Endpoints.RemoveRepositoryUser` +- `method Endpoints.RemoveTeamMember` +- `method Endpoints.RenameRepository` +- `method Endpoints.ReopenIssue` +- `method Endpoints.ReopenPullRequest` +- `method Endpoints.ReorderIssue` +- `method Endpoints.ReplyPullRequest` +- `method Endpoints.RepositoryInfo` +- `method Endpoints.ResolveTeam` +- `method Endpoints.ReviewPullRequest` +- `method Endpoints.RotateCISecret` +- `method Endpoints.RunCI` +- `method Endpoints.SuspendKey` +- `method Endpoints.SyncPullRequests` +- `method Endpoints.TakeIssue` +- `method Endpoints.UpdateIssue` +- `method Endpoints.UpdateRef` +- `method Endpoints.UpdateRepository` +- `method Endpoints.UpdateUserProfile` +- `method Endpoints.UpsertProtection` +- `method Endpoints.UpsertRepositoryTeam` +- `method Endpoints.UpsertRepositoryUser` +- `method Endpoints.UpsertTeamMember` +- `method Endpoints.UpsertUser` +- `method SignatureProviderFunc.HeaderSets` +- `method V2Signatures.HeaderSets` +- `type Caller` +- `type Client` +- `type Endpoints` +- `type ErrorDecoder` +- `type NonceSource` +- `type Options` +- `type RetryPredicate` +- `type SignatureProvider` +- `type SignatureProviderFunc` +- `type SuccessObserver` +- `type V2SignatureOptions` +- `type V2Signatures` + +## `github.com/bucketgit/bgit/broker/capability` + +- `func New` +- `method Store.CompareAndSwapRef` +- `method Store.Delete` +- `method Store.List` +- `method Store.ListRefs` +- `method Store.Read` +- `method Store.Write` +- `type Broker` +- `type ErrorClassifier` +- `type LocalHandler` +- `type Options` +- `type Store` + +## `github.com/bucketgit/bgit/broker/local` + +- `func IsZeroOID` +- `func New` +- `method Broker.ArchiveIssue` +- `method Broker.AssignIssue` +- `method Broker.Authorize` +- `method Broker.BucketDir` +- `method Broker.CommentIssue` +- `method Broker.CompareAndSwapJSON` +- `method Broker.CreateIssue` +- `method Broker.CreateRepository` +- `method Broker.DeleteObject` +- `method Broker.FindRepository` +- `method Broker.GetIssue` +- `method Broker.GetRepository` +- `method Broker.Handler` +- `method Broker.ListIssues` +- `method Broker.ListObjects` +- `method Broker.ListRepositories` +- `method Broker.LoadJSON` +- `method Broker.LoadJSONState` +- `method Broker.LoadOwners` +- `method Broker.LoadRepositoryIndex` +- `method Broker.LoadRepositoryState` +- `method Broker.LockRepository` +- `method Broker.MoveStory` +- `method Broker.MoveStoryToLaneEnd` +- `method Broker.ObjectPath` +- `method Broker.ObjectRoot` +- `method Broker.ReadObject` +- `method Broker.ReconcileRepository` +- `method Broker.Repository` +- `method Broker.Root` +- `method Broker.SaveJSON` +- `method Broker.SaveRepositoryState` +- `method Broker.SetIssueClosed` +- `method Broker.TakeStory` +- `method Broker.UpdateIssue` +- `method Broker.UpdateRef` +- `method Broker.UpsertOwners` +- `method Broker.UpsertRepository` +- `method Broker.WriteObject` +- `method IdentityVerifierFunc.Verify` +- `method RepositoryStore.CompareAndSwap` +- `method RepositoryStore.CompareAndSwapRef` +- `method RepositoryStore.Delete` +- `method RepositoryStore.List` +- `method RepositoryStore.ListRefs` +- `method RepositoryStore.Read` +- `method RepositoryStore.Write` +- `method ResolverFunc.Resolve` +- `type Broker` +- `type IdentityVerifier` +- `type IdentityVerifierFunc` +- `type IssueFilter` +- `type Options` +- `type Owners` +- `type RepositoryIndex` +- `type RepositoryState` +- `type RepositoryStore` +- `type Resolver` +- `type ResolverFunc` +- `type StoryMove` + +## `github.com/bucketgit/bgit/transport` + +- `const DataPacket` +- `const DefaultMaxReceivePackSize` +- `const DelimiterPacket` +- `const FlushPacket` +- `const MaxPayload` +- `const ReceivePackService` +- `const ResponseEndPacket` +- `const UploadPackService` +- `func EncodePack` +- `func ReadPacket` +- `func ReadReceivePackRequest` +- `func ReadUploadPackRequest` +- `func ReceivePackCapabilities` +- `func RemoteHelperAddress` +- `func ServeReceivePack` +- `func ServeRemoteHelper` +- `func ServeUploadPack` +- `func UploadPackCapabilities` +- `func WriteAdvertisedRefs` +- `func WriteDelimiter` +- `func WriteFlush` +- `func WritePacket` +- `func WriteResponseEnd` +- `func WriteSideband` +- `func WriteString` +- `method Capabilities.String` +- `method ResolverFunc.Serve` +- `type Capabilities` +- `type Packet` +- `type PacketKind` +- `type ReceiveCommand` +- `type ReceiveRequest` +- `type ReceiveStore` +- `type Resolver` +- `type ResolverFunc` +- `type UploadPackRequest` diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ce532d..78c9a45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,22 @@ All notable changes to `bgit` are documented in this file. This project follows semantic versioning. -## 1.3.9 +## 1.4.0 + +Added + +- Added an importable Go SDK for the BucketGit protocol, filesystem/S3/GCS + stores, Git repository engine, remote and in-process brokers, capability + storage, and upload-pack/receive-pack transport. +- Added shared provider contracts, cross-language broker signature fixtures, + external-consumer examples, generated API inventory, SDK migration and + security documentation, and architecture dependency checks. + +Changed + +- Segmented the CLI into a minimal executable and internal application, + configuration, identity, setup, and web packages without changing existing + repository layouts, local-broker state, or broker wire compatibility. Fixed diff --git a/MIGRATING-SDK.md b/MIGRATING-SDK.md new file mode 100644 index 0000000..9ff649f --- /dev/null +++ b/MIGRATING-SDK.md @@ -0,0 +1,23 @@ +# Migrating To The BucketGit Go SDK + +Code that previously copied BucketGit internals should import the owning public +package from `github.com/bucketgit/bgit` instead. + +| Previous responsibility | Public package | +|---|---| +| raw filesystem/S3/GCS object access | `store/fs`, `store/s3`, `store/gcs` | +| Git objects, refs, revisions, packs, diffs | `repository` | +| broker request/response JSON | `protocol` | +| signed remote broker calls | `broker/client` | +| broker-issued object capabilities | `broker/capability` | +| in-process local control plane | `broker/local` | +| upload-pack, receive-pack, remote helper | `transport` | + +Do not import `internal/*`; those packages implement the `bgit` application and +may change without SDK compatibility guarantees. Do not use a raw store to +replace broker authorization. For a deployed broker, object access uses +`broker/capability` and final ref mutation remains a signed broker CAS call. + +Pin a tagged release in `go.mod`. `API.md` records the v1 public surface; +breaking changes require a future major module path. Existing CLI configuration and local-broker state +are application concerns and are intentionally not exposed as public SDK types. diff --git a/README.md b/README.md index 2768704..aa0bd0a 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,11 @@ coordinates access through a lightweight broker. Use it when you want Git repositories in cloud object storage without running a Git server. +The Git, storage, broker protocol, and transport implementations are also +importable Go packages. See [SDK.md](SDK.md) for supported compositions and +responsibility boundaries. +The reviewed exported symbol list is maintained in [API.md](API.md). + ## Project - Homepage: https://bucketgit.com/ @@ -28,7 +33,7 @@ Or build from source: ```bash git clone https://github.com/bucketgit/bgit.git cd bgit -go build -o bgit . +go build -o bgit ./cmd/bgit ``` Check the installed version: @@ -580,7 +585,7 @@ Use them for repair, test cleanup, or broker decommissioning. Build from source: ```bash -go build -o bgit . +go build -o bgit ./cmd/bgit ``` Run unit tests: @@ -608,7 +613,7 @@ Runtime requirements depend on the command: - `gcloud` for GCP setup and profile creation. - AWS config/credentials files and optionally the AWS CLI for AWS setup/profile creation. -- Go 1.24 or newer to build from source. +- Go 1.25 or newer to build from source or import the current SDK packages. ## Unsupported Commands diff --git a/SDK.md b/SDK.md new file mode 100644 index 0000000..1dce670 --- /dev/null +++ b/SDK.md @@ -0,0 +1,150 @@ +# BucketGit Go SDK + +BucketGit's module path is `github.com/bucketgit/bgit`. The CLI composes the +same packages exposed to other Go programs; cloud credentials and terminal UI +are not dependencies of the repository engine. + +The SDK shares BucketGit's existing v1 release line. Consumers should pin a +tagged BucketGit release; incompatible public API changes require a future +major module path. +See [API.md](API.md) for the generated public surface, +[MIGRATING-SDK.md](MIGRATING-SDK.md) for migration guidance, and +[SECURITY-SDK.md](SECURITY-SDK.md) for authorization boundaries. + +## Compatibility + +| Component | Compatibility contract | +|---|---| +| Go SDK | v1 from the first segmented release; pin a normal `bgit` release | +| CLI configuration | existing global and repository configuration remains readable | +| local-broker state | `.bucketgit/broker-state/` remains readable without conversion | +| broker wire protocol | v2 signatures and additive JSON fields; protocol version is independent of module version | +| deployed brokers | supported older responses and missing optional endpoints retain compatibility handling | + +## Packages + +- `protocol`: broker JSON models, constants, typed errors, and v2 request + signature canonicalization and replay-field validation. +- `store`: provider-neutral object and ref interfaces, path validation, fallback + readers, ref parsing, and shared errors. +- `store/fs`: atomic filesystem object storage and cross-process ref CAS. +- `store/s3`: injected-client or standard AWS credential-chain S3 storage with + conditional ref updates. +- `store/gcs`: injected-client or ADC GCS storage with generation-matched ref + updates. +- `repository`: SHA-1 Git objects, refs, commits, trees, tags, pack indexes, + deltas, reachability, ancestry, and merge-base operations. +- `broker/client`: signed broker HTTP requests and typed endpoint methods. +- `broker/capability`: broker-authorized object access through S3 STS, GCS + signed URLs, or local capabilities, including supported legacy reads. +- `broker/local`: in-process local broker persistence, scoped repository stores, + ref CAS, and issue/task-board services. +- `transport`: pkt-line framing, advertised capabilities, upload-pack, + receive-pack, and Git remote-helper protocol handling. + +## Filesystem Repository + +```go +objects, err := fsstore.New("/srv/git/demo.git") +if err != nil { + return err +} +repo := repository.Open(objects, objects) +head, err := repo.Resolve(ctx, "main") +``` + +## S3 Repository + +Use `s3store.New` when the application already owns an AWS SDK client. Use +`s3store.Load` for the standard AWS credential chain: + +```go +objects, err := s3store.Load(ctx, s3store.Options{ + Bucket: "123456789012-demo", + Profile: "work", + Region: "eu-west-1", +}) +repo := repository.Open(objects, objects) +``` + +Bucket creation and IAM policy management are provisioning operations and are +not performed by store constructors. + +## GCS Repository + +```go +objects, err := gcsstore.Load(ctx, gcsstore.Options{Bucket: "project-demo"}) +if err != nil { + return err +} +defer objects.Close() +repo := repository.Open(objects, objects) +``` + +`gcsstore.New` accepts an existing `*storage.Client` and never closes it. + +## Remote Broker Repository + +Create a `broker/client.Client` with an injected signature provider, then pass +it to `capability.New`. The capability store performs object I/O while ref +updates remain broker-authorized CAS operations. + +```go +client, err := brokerclient.New(brokerURL, brokerclient.Options{ + HTTPClient: httpClient, + Signatures: signer, +}) +objects, err := capability.New(client, identity, capability.Options{ + HTTPClient: httpClient, +}) +repo := repository.Open(objects, objects) +``` + +Applications are responsible for selecting trusted SSH identities, protecting +private keys, applying HTTP timeouts, and deciding whether unsigned fallback is +allowed for a legacy broker. + +## Local Broker + +```go +broker, err := local.New(local.Options{Root: stateRoot, Resolver: resolver}) +scoped := broker.Repository(protocol.Repository{ + Provider: "file", Bucket: "demo.git", Logical: "demo.git", +}) +repo := repository.Open(scoped, scoped) +``` + +A resolver may return S3 or GCS stores for cloud-backed local repositories. The +repository's `.bucketgit/broker-state/` namespace is authoritative; callers +must not rewrite it outside broker APIs. Resolver stores used for mutable +broker metadata must implement `store.CompareAndSwapper`; the built-in +filesystem, S3, and GCS stores use expiring file leases, ETags, and generations +respectively to reject stale metadata writes. + +## Transport + +`transport.ServeUploadPack` is read-only. `transport.ServeReceivePack` requires +a `ReceiveStore`, validates and ingests the pack, checks fast-forward updates, +and performs ref CAS. Authorization must happen before invoking either service. + +`transport.ServeRemoteHelper` handles the helper protocol but delegates address +resolution and service authorization through its `Resolver` interface. + +## Concurrency And Errors + +- Store paths are canonical, relative slash-separated paths. +- Missing objects use `fs.ErrNotExist`. +- Conditional ref failures use `store.ErrConflict`. +- Conditional metadata failures also use `store.ErrConflict`; reload broker + state before retrying a mutation. +- Broker failures expose `protocol.BrokerError` and support `errors.Is` for + conflict, authorization, and unsupported-operation classes. +- Repository instances own mutable caches and are safe for concurrent reads. +- Repository decoding defaults to a 128 MiB decompressed object ceiling; + applications that intentionally store larger Git objects can opt in with + `repository.WithMaxObjectSize` after assessing their memory budget. +- Receive-pack rejects encoded packs larger than 512 MiB and delta-expanded + objects larger than the repository default before allocating their declared + size. +- GCS clients returned by `Load` must be closed; injected clients remain owned + by the caller. diff --git a/SECURITY-SDK.md b/SECURITY-SDK.md new file mode 100644 index 0000000..dbd44e9 --- /dev/null +++ b/SECURITY-SDK.md @@ -0,0 +1,26 @@ +# BucketGit SDK Security Boundaries + +The storage packages provide scoped object access. They do not authenticate a +user, evaluate roles, enforce branch protection, or authorize a ref update. + +- Use `broker/client` plus `broker/capability` for deployed-broker repositories. + Keep final ref updates on the broker endpoint. +- Use `broker/local` when an application needs the in-process local control + plane, persisted conflict safeguards, and repository-scoped metadata. +- Treat S3/GCS credentials, SSH signers, signed URLs, and capability responses + as secrets. Do not persist them in Git configuration or ordinary logs. +- Inject bounded HTTP clients and honor context cancellation. +- Keep v2 timestamp and nonce validation enabled. An unsigned compatibility + fallback is an explicit caller decision for a trusted legacy broker only. +- Do not bypass `store.ValidatePath`; filesystem stores additionally reject + symlink escapes. +- Handle `store.ErrConflict` as a failed compare-and-swap and reconcile before + retrying. This applies to refs and local-broker metadata; blind replacement + loses concurrent updates. +- Repository decoding defaults to a 128 MiB expanded-object limit and transport + receive-pack limits encoded input to 512 MiB. Raising either limit increases + memory-exhaustion exposure. + +Applications embedding upload-pack or receive-pack must authenticate and +authorize the request before invoking the transport service. `transport` owns +Git framing and update correctness, not user identity or repository policy. diff --git a/SEGMENT.md b/SEGMENT.md new file mode 100644 index 0000000..42202a9 --- /dev/null +++ b/SEGMENT.md @@ -0,0 +1,928 @@ +# BucketGit Go Package Segmentation Plan + +## Objective + +Refactor `bgit` from a single `package main` implementation into a small public +Go SDK plus CLI-specific packages. The resulting packages must: + +- Make the `bgit` executable easier to navigate and maintain. +- Let other Go applications read, write, serve, and coordinate BucketGit + repositories without invoking the CLI as a subprocess. +- Preserve the existing on-disk/object-storage layout and broker wire protocol. +- Preserve all current repository modes throughout the migration. +- Keep authorization, final ref updates, branch protection, and compare-and-swap + behavior at least as strong as they are today. + +This is a code-organization and API-extraction effort. It is not a new storage +format, broker protocol version, or CLI redesign. + +## Repository Modes To Preserve + +The implementation should treat repository modes as compositions of reusable +broker and storage packages, not as five independent implementations. + +| User-facing mode | Control plane | Object storage | Credentials used by client | +|---|---|---|---| +| `file://repo.git` | In-process local broker | Local filesystem | Local filesystem access | +| `s3://repo.git` | In-process local broker | AWS S3 | Configured AWS credentials | +| `gs://repo.git` | In-process local broker | Google Cloud Storage | Configured GCP credentials | +| AWS deployed broker | Remote AWS broker | S3 through broker-issued capabilities | SSH key plus short-lived STS credentials | +| GCP deployed broker | Remote GCP broker | GCS through broker-issued capabilities | SSH key plus signed URLs/upload sessions | + +The package graph must make these combinations explicit. Provider-specific +packages should implement storage. Broker packages should implement control +plane behavior. The Git repository package should depend on interfaces, not on +AWS, GCP, HTTP, CLI config, or the local broker. + +## Non-Goals + +- Do not rewrite the cloud broker runtimes in Go as part of this effort. +- Do not change Git object paths, ref paths, pack formats, metadata namespaces, + or the `.bucketgit/broker-state/` layout. +- Do not weaken broker-owned final ref updates for deployed brokers. +- Do not make raw object-store access equivalent to broker authorization. +- Do not expose the current CLI `config` struct as a public SDK API. +- Do not move the SDK to a separate repository until the API has stabilized. +- Do not combine package extraction with unrelated CLI or web UI redesigns. + +## Current Coupling To Remove + +The current codebase contains approximately 42,000 lines of Go in one package. +The relevant coupling points are: + +- `main.go` owns CLI config, URI parsing, cloud clients, remote-store + construction, and command dispatch. +- `native_git.go` defines the generic store interfaces, GCS and filesystem + stores, Git object parsing, pack access, refs, fetch, pull, and push. +- `s3_store.go` implements S3 but consumes the CLI `config` type. +- `web.go` owns `brokerGitStore`, even though that store is needed outside the + web UI. +- `ssh.go` mixes Git transport, broker signing, broker HTTP, cloud deployment, + SSH identity discovery, and administration. +- `broker_data_path.go` mixes broker protocol DTOs with capability-backed S3 + and GCS object transfer. +- `local_broker_native.go` mixes the local broker runtime, protocol routing, + metadata persistence, object capabilities, refs, issues, and board behavior. +- Broker models such as `brokerRepo`, issues, PRs, CI runs, users, capabilities, + and ref updates are spread across CLI and web files. +- AWS and GCP broker runtimes are JavaScript implementations, so a Go-only + protocol package cannot by itself prevent cross-language drift. + +The migration must untangle these dependencies without a big-bang rewrite. + +## Module And Versioning Strategy + +### Initial location + +Keep the packages in the `bucketgit/bgit` repository. Change the module path: + +```go +module github.com/bucketgit/bgit +``` + +The executable can remain at the repository root during early phases. Move it +to `cmd/bgit` only after the extracted packages no longer import package-main +types. + +### Public API maturity + +- Tag SDK releases with normal `bgit` releases. Because the product already + publishes `v1.x` tags and this plan intentionally keeps one module, the first + segmented release establishes a Go module v1 API; a separate v0 line is not + representable without introducing a second module. +- Review and minimize the exported API before that release. Future breaking + changes require a `/v2` module path or a new major-version package. +- Keep the broker wire protocol version independent from the Go module version. +- Continue using replay-resistant broker request signature protocol v2. +- Do not create a second Go module or repository until package boundaries and + public types have survived at least two releases. + +### Compatibility policy + +- Existing CLI configuration remains readable. +- Existing local-broker state remains readable and writable. +- Existing AWS/GCP brokers remain compatible with new clients. +- New Go clients must be able to communicate with supported older brokers. +- Wire fields may be added compatibly; fields may not be renamed or removed + without a broker protocol version change. + +## Target Package Layout + +```text +bgit/ + cmd/bgit/ executable entry point + protocol/ public broker wire model and signing contract + store/ provider-neutral object and ref interfaces + store/fs/ filesystem object storage + store/s3/ AWS S3 object storage + store/gcs/ Google Cloud Storage + repository/ Git objects, refs, revisions, packs and trees + broker/client/ remote broker HTTP client + broker/capability/ capability-backed object store + broker/local/ in-process broker engine and persisted state + transport/ upload-pack, receive-pack and remote helper + internal/config/ global and repository CLI configuration + internal/identity/ SSH-agent/key discovery and selection + internal/setup/ cloud discovery and deployment primitives + internal/cli/ shared parsing and terminal presentation + internal/web/ web routes, assets, middleware and events + internal/app/ feature flows and concrete CLI/web adapters + spec/ protocol documentation, schemas and test vectors +``` + +Package names can be adjusted during implementation, but the dependency +direction below is mandatory. + +## Required Dependency Direction + +```text +cmd/bgit + -> root compatibility facade -> internal/app + -> internal/cli + -> internal/config, internal/setup, internal/identity, internal/web + -> transport, repository, broker/client, broker/local + -> store/fs, store/s3, store/gcs + +transport -> repository +repository -> store +broker/capability -> protocol, store +broker/client -> protocol +broker/local -> protocol, store +store/fs -> store +store/s3 -> store +store/gcs -> store +protocol -> standard library and narrowly scoped crypto dependencies +``` + +Forbidden dependencies: + +- Public SDK packages must not import `internal/cli`, `internal/config`, or + `internal/app`; the root CLI compatibility facade is the sole exception. +- `repository` must not import AWS, GCP, HTTP broker, local broker, or CLI + configuration packages. +- Provider stores must not import broker or Git repository packages. +- `protocol` must not import storage providers, repository logic, or CLI code. +- Broker packages must not depend on command-line parsing or terminal output. + +Use a dependency test or architecture linter to enforce these rules. + +## Public Package APIs + +The exact names may evolve before the first segmented release, but +implementation should target the following responsibilities and shapes. + +### `store` + +`store` is a provider-neutral key/value object namespace. All paths are relative +to an already scoped repository prefix. + +```go +package store + +type Reader interface { + Read(ctx context.Context, path string) ([]byte, error) + List(ctx context.Context, prefix string) ([]string, error) +} + +type Writer interface { + Reader + Write(ctx context.Context, path string, data []byte) error + Delete(ctx context.Context, path string) error +} + +type RefStore interface { + ListRefs(ctx context.Context) (map[string]string, error) + CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error +} +``` + +Requirements: + +- Standardize not-found behavior on `fs.ErrNotExist`. +- Validate that paths are relative, canonical, and cannot escape the repository + namespace. +- Return deterministic, relative paths from `List`. +- Accept injected provider clients for tests and advanced consumers. +- Keep credential discovery in provider convenience constructors, not in the + base interfaces. +- Distinguish raw object writes from authorized ref mutation. + +### `store/fs` + +```go +fsstore.New(root string, options ...Option) (*Store, error) +``` + +Responsibilities: + +- Filesystem reads, lists, atomic writes, deletes, and sync where required. +- Symlink/path-escape prevention. +- Durable temporary-file-plus-rename writes. +- Optional file locking primitives used by the local broker. + +### `store/s3` + +```go +s3store.New(client S3API, bucket, prefix string, options ...Option) (*Store, error) +s3store.Load(ctx context.Context, Options) (*Store, error) +``` + +Responsibilities: + +- S3 get/list/put/delete behavior. +- Region/profile/default credential convenience loading. +- Error translation into shared store errors. +- Repository prefix scoping. +- No broker authorization decisions. + +### `store/gcs` + +```go +gcsstore.New(client *storage.Client, bucket, prefix string, options ...Option) (*Store, error) +gcsstore.Load(ctx context.Context, Options) (*Store, error) +``` + +Responsibilities mirror S3. The constructor receiving an existing client is +the primary testable API; credential-loading helpers are optional convenience. + +### `repository` + +`repository` implements Git repository semantics over `store.Reader` or +`store.Writer`. + +```go +type Repository struct { /* unexported fields */ } + +func Open(objects store.Reader, refs store.RefStore, options ...Option) *Repository + +func (r *Repository) Resolve(ctx context.Context, revision string) (OID, error) +func (r *Repository) Object(ctx context.Context, oid OID) (Object, error) +func (r *Repository) Commit(ctx context.Context, oid OID) (Commit, error) +func (r *Repository) Tree(ctx context.Context, revision string) ([]TreeEntry, error) +func (r *Repository) ListRefs(ctx context.Context) (map[string]OID, error) +func (r *Repository) MergeBase(ctx context.Context, a, b OID) (OID, error) +``` + +Additional internal or public sub-APIs should cover pack indexes, deltas, +reachable-object traversal, archive generation, diff data, upload preparation, +and receive-pack ingestion. + +Requirements: + +- Replace methods that parse CLI args or write formatted output with typed + inputs and typed results. +- Keep SHA-1 as the supported object format initially, but model OIDs so future + object formats are not impossible. +- Preserve loose object and pack behavior, including thin-pack ingestion. +- Keep caches private to a repository instance. +- Do not expose provider clients or CLI config. + +### `protocol` + +`protocol` defines the supported broker contract: + +- Repository identity and logical-to-physical mapping. +- Authentication and authorization requests/responses. +- Object capability requests/responses. +- Ref listing and compare-and-swap updates. +- Users, keys, roles, teams, invites, and ownership transfer. +- Branch protection. +- Pull requests, reviews, comments, issues, task-board stories, and CI runs. +- Broker health, version, and capabilities. +- Canonical v2 request signing and verification test vectors. + +Example API: + +```go +type Repository struct { + Provider string `json:"provider,omitempty"` + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Logical string `json:"logical"` + TeamID string `json:"team_id,omitempty"` + Profile string `json:"profile,omitempty"` + Region string `json:"region,omitempty"` +} + +type Signer interface { + Sign(ctx context.Context, message []byte) (Signature, error) +} + +func SignatureMessage(RequestMetadata, []byte) ([]byte, error) +``` + +Requirements: + +- No request type should depend on CLI-only configuration. +- Unix timestamps, nonces, host, path, method, and body digest remain part of + canonical v2 signing. +- JSON tags and omission behavior are compatibility-sensitive. +- Add golden JSON and signature vectors under `spec/testdata`. +- Run the same vectors against Go, GCP JavaScript, and embedded AWS JavaScript. + +### `broker/client` + +```go +type Client struct { /* base URL, HTTP client, signer, clock, nonce source */ } + +func New(baseURL string, options ...Option) (*Client, error) +func (c *Client) Authorize(ctx context.Context, req protocol.AuthRequest) (...) +func (c *Client) ListRefs(ctx context.Context, repo protocol.Repository) (...) +func (c *Client) UpdateRef(ctx context.Context, req protocol.RefUpdateRequest) error +func (c *Client) ObjectCapability(ctx context.Context, req protocol.ObjectCapabilityRequest) (...) +``` + +Requirements: + +- Accept an injected `http.Client`, signer, clock, and nonce source. +- Own HTTP status/error decoding and broker-version compatibility messages. +- Never print; return typed errors with endpoint, status, broker code, and safe + message. +- Support explicit retry policy only for safe operations. +- Do not retry ref updates or other non-idempotent mutations implicitly. +- Keep private material and signed headers out of error strings and logs. + +### `broker/capability` + +Turn broker-issued capabilities into a `store.Writer` implementation: + +```go +func New(client *brokerclient.Client, repo protocol.Repository, options ...Option) *Store +``` + +Responsibilities: + +- Request per-object read/write/delete capabilities. +- Use S3 STS credentials, GCS signed URLs, resumable sessions, or local + capabilities as returned by the broker. +- Use `/refs/list` and broker ref CAS rather than listing or writing raw ref + objects when a broker controls the repository. +- Treat capability mode/provider as server-returned data, not client guesses. + +### `broker/local` + +The local broker should become a Go service object with direct methods. It must +not require an HTTP loopback server or simulate HTTP internally. + +```go +type Broker struct { /* stores, identity verifier, clock, locks */ } + +func New(options Options) (*Broker, error) +func (b *Broker) Authorize(ctx context.Context, req protocol.AuthRequest) (...) +func (b *Broker) ObjectCapability(ctx context.Context, req protocol.ObjectCapabilityRequest) (...) +func (b *Broker) UpdateRef(ctx context.Context, req protocol.RefUpdateRequest) error +``` + +Responsibilities: + +- Persist authoritative metadata in `.bucketgit/broker-state/`. +- Support filesystem, S3, and GCS object backends. +- Preserve storage-backed ref state, short leases, and conflict recovery. +- Preserve users, teams, issues, board ordering/archive state, PRs, protection, + and all current local endpoints. +- Keep the current local URL/config adapter in the CLI, not in the broker core. +- Offer an optional `http.Handler` adapter for conformance tests and embedders; + the CLI should call methods directly. + +### `transport` + +```go +func ServeUploadPack(ctx context.Context, repo *repository.Repository, in io.Reader, out io.Writer) error +func ServeReceivePack(ctx context.Context, repo *repository.Repository, refs RefUpdater, in io.Reader, out io.Writer) error +func ServeRemoteHelper(ctx context.Context, resolver Resolver, args []string, in io.Reader, out, errOut io.Writer) error +``` + +Responsibilities: + +- Git pkt-line protocol and advertised capabilities. +- Upload-pack reachable-object collection and pack generation. +- Receive-pack parsing, thin-pack handling, validation, report-status, atomic + behavior, and ref updates. +- `git-remote-bgit` helper protocol. +- No cloud credential discovery, CLI config parsing, or terminal UI. + +## Protocol Specification And Cross-Language Contract + +The cloud brokers remain JavaScript. A public Go package therefore needs an +implementation-independent protocol source of truth. + +Create: + +```text +spec/ + README.md + broker-v2.md + schemas/ + repository.json + auth.json + object-capability.json + ref-update.json + issue.json + pull-request.json + ci-run.json + testdata/ + signing-v2.json + canonical-json/ + error-responses/ +``` + +The specification must define: + +- Endpoint paths, methods, status codes, and error body shape. +- Required and optional fields. +- Canonical request-signature construction. +- Timestamp tolerance and replay/nonce rules. +- Repository logical identity and physical mapping semantics. +- Allowed object capability paths and operations. +- Ref CAS and branch-protection semantics. +- Compatibility expectations for unknown fields and missing endpoints. + +Do not generate all runtime code initially. First establish golden fixtures and +conformance tests. Generation can be considered after schemas prove stable. + +## Error Model + +Introduce typed errors early, before moving implementations: + +```go +var ErrNotFound = fs.ErrNotExist +var ErrConflict = errors.New("bucketgit conflict") +var ErrUnauthorized = errors.New("bucketgit unauthorized") +var ErrUnsupported = errors.New("bucketgit unsupported") + +type BrokerError struct { + Endpoint string + Status int + Code string + Message string +} +``` + +Requirements: + +- Preserve `errors.Is` behavior for not-found, conflict, unauthorized, and + unsupported cases. +- Separate safe user-facing messages from provider/debug details. +- Avoid string matching for normal control flow once typed errors exist. +- Preserve compatibility hints for genuinely old brokers without mislabeling + missing identities or permissions as version skew. + +## Configuration Boundary + +The current `config` struct is a CLI aggregation object and must remain +internal. Replace it at package boundaries with explicit options: + +```go +type S3Options struct { + Bucket string + Prefix string + Region string + Profile string +} + +type RemoteRepository struct { + BrokerURL string + Repository protocol.Repository +} +``` + +`internal/config` remains responsible for: + +- `~/.bgit/config.yaml`. +- `.git/config` BucketGit keys. +- Profile and region selection. +- Environment-variable overrides. +- Mapping configuration into public package constructors. + +Public packages should not run `git config`, `gcloud`, `aws`, or interactive +prompts unless the package is explicitly a CLI/setup convenience package. + +## Security Invariants + +Every phase must preserve these invariants: + +- All broker mutations use v2 request signatures. +- Replay protection remains based on timestamp and nonce validation. +- Object capability paths are canonical, repo-relative, and restricted to + approved Git/object namespaces. +- Broker-issued credentials remain short-lived and repository-scoped. +- Deployed-broker final ref updates always go through broker CAS. +- Branch protection and PR requirements cannot be bypassed by the SDK's normal + broker-backed API. +- Local-broker ref updates retain storage-backed leases and conflict checks. +- Filesystem stores reject symlink and traversal escapes. +- Provider credentials and signed URLs are never persisted in repository + metadata or returned in ordinary errors. +- Direct/raw stores are documented as storage primitives, not authorization or + policy enforcement. + +## Migration Strategy + +Use adapters and type aliases to migrate incrementally. At the end of every +phase, the CLI must build and the existing integration suite must pass. + +### Phase 0: Baseline And Architecture Guardrails + +- [x] Record current `go test ./...` results and integration-suite results. +- [x] Record golden CLI output for clone/fetch/push, local broker, PR, board, + issues, CI, and web operations. +- [x] Add an architecture decision record describing package boundaries and + dependency direction. +- [x] Change the module path to `github.com/bucketgit/bgit`. +- [x] Add a package dependency test that rejects forbidden imports. +- [x] Add `go vet ./...` and public example compilation to CI. +- [x] Define supported Go version and SDK compatibility policy. + +Exit criteria: + +- Existing behavior is captured well enough to distinguish refactoring from + regression. +- Module-path change builds on all supported OS/architecture targets. + +### Phase 1: Extract Protocol Types And Signing + +- [x] Inventory every request/response model in `ssh.go`, + `broker_commands.go`, `broker_data_path.go`, `auth_capabilities.go`, and + `web.go`. +- [x] Create `protocol` types with stable JSON tags. +- [x] Move role, provider, operation, lane, PR status, and CI status constants + into typed string definitions. +- [x] Extract v2 canonical signature message construction. +- [x] Extract signature header names and timestamp/nonce validation helpers. +- [x] Create golden signing fixtures covering RSA and Ed25519 keys. +- [x] Run those fixtures against Go and both JavaScript broker runtimes. +- [x] Use type aliases in package main during migration where practical. +- [x] Replace generic broker errors with typed protocol/client errors. + +Exit criteria: + +- All broker DTOs used by more than one feature come from `protocol`. +- Existing deployed brokers accept requests produced by the extracted signer. +- No JSON payload changes appear in captured fixtures. + +### Phase 2: Extract Store Interfaces And Filesystem Backend + +- [x] Create `store.Reader`, `store.Writer`, and `store.RefStore`. +- [x] Move fallback-store behavior into `store`. +- [x] Extract common path validation and object-prefix helpers. +- [x] Move `localGitStore` into `store/fs`. +- [x] Implement atomic filesystem writes and traversal/symlink tests. +- [x] Create a reusable store contract suite. +- [x] Adapt `nativeGitRepo` to consume the public interfaces through aliases or + adapters. + +Store contract tests must cover: + +- Read/write/delete round trips. +- Missing objects and `fs.ErrNotExist`. +- Empty and nested prefix listing. +- Stable relative path results. +- Zero-byte and large objects. +- Concurrent writes where supported. +- `../`, absolute path, backslash, encoded traversal, and symlink escapes. + +Exit criteria: + +- Filesystem repositories pass the existing local-broker and Git tests through + the extracted package. +- No public store API depends on CLI config. + +### Phase 3: Extract S3 And GCS Backends + +- [x] Move `s3GitStore` and provider error translation into `store/s3`. +- [x] Move `gcsGitStore` into `store/gcs`. +- [x] Introduce minimal provider client interfaces for unit tests. +- [x] Keep constructors accepting existing SDK clients. +- [x] Add convenience constructors for profile/default credentials. +- [x] Move bucket creation out of the core store into explicit provisioning + helpers used by the CLI/local-broker bootstrap flow. +- [x] Run the shared store contract suite against fake/in-memory provider + clients. +- [x] Keep real S3/GCS smoke tests outside normal CI unless credentials are + explicitly supplied. + +Exit criteria: + +- `file://`, `s3://`, and `gs://` local-broker repositories use the same + provider-neutral store interfaces. +- Real-provider smoke tests cover create, clone, fetch, push, ref conflict, and + cleanup. + +### Phase 4: Extract Git Repository Engine + +- [x] Move object parsing, hashing, trees, commits, tags, refs, packs, deltas, + revision resolution, and reachable-object traversal into `repository`. +- [x] Convert CLI-oriented methods into typed operations and results. +- [x] Move pack index and thin-pack behavior with focused tests. +- [x] Move archive/diff primitives needed by other Go consumers. +- [x] Keep local worktree porcelain separate; do not force it into the remote + repository API in this phase. +- [x] Add examples for opening filesystem, S3, and GCS repositories. +- [x] Add fuzz tests for object, pkt-line, pack index, delta, and malformed ref + parsing where practical. + +Exit criteria: + +- The same repository engine serves all five repository modes. +- `repository` imports only `store`, standard library, and narrowly justified + Git-specific dependencies. +- Native Git interoperability tests pass for clone/fetch/push/branch/tag and + thin packs. + +### Phase 5: Extract Remote Broker Client And Capability Store + +- [x] Move signed HTTP request handling into `broker/client`. +- [x] Move endpoint methods from CLI/web call sites into typed client methods. +- [x] Move `brokerGitStore` out of `web.go` into `broker/capability`. +- [x] Move STS and signed-URL capability execution into that package. +- [x] Inject HTTP clients, clocks, nonce sources, and signers. +- [x] Add compatibility tests against old supported broker responses. +- [x] Add cancellation, timeout, and safe retry tests. +- [x] Ensure ref listing and updates use broker methods, not raw storage refs. + +Exit criteria: + +- CLI, SSH transport, remote helper, and web UI share one broker client. +- No broker HTTP or capability implementation remains in `web.go`. +- No normal control flow depends on parsing broker error strings. + +### Phase 6: Extract Local Broker + +- [x] Replace endpoint-switch internals with direct service methods. +- [x] Move local metadata models to `protocol` or private local-broker state + types as appropriate. +- [x] Move local state persistence, locks, leases, and reconciliation into + `broker/local`. +- [x] Define a storage resolver mapping repository metadata to filesystem, S3, + or GCS stores. +- [x] Preserve `.bucketgit/broker-state/` compatibility with migration tests. +- [x] Preserve issue/board/PR/protection behavior. +- [x] Add optional `http.Handler` adapter implementing the broker protocol for + conformance and embedding. +- [x] Keep `local://` parsing and global-profile lookup in `internal/config`. +- [x] Remove any remaining assumption that the local broker is a long-lived + HTTP server. + +Exit criteria: + +- Existing local repositories open without conversion. +- File/S3/GCS-backed local brokers pass the same broker behavior suite. +- Concurrent local-broker instances detect stale state and ref conflicts. +- Local broker can be embedded by another Go process without CLI globals. + +### Phase 7: Extract Git Transport + +- [x] Move pkt-line helpers and advertised capabilities into `transport`. +- [x] Move upload-pack and receive-pack services. +- [x] Move remote-helper protocol support. +- [x] Inject authorization/ref-update interfaces into receive-pack. +- [x] Remove transport dependencies on global configuration and `os.Stdin` / + `os.Stdout`. +- [x] Keep SSH command parsing in the CLI adapter; pass typed service requests + into `transport`. + +Exit criteria: + +- Another Go process can serve a BucketGit repository over upload-pack and + receive-pack using only public packages. +- `git-remote-bgit`, `core.sshCommand`, and integration tests use the same + transport implementation. + +### Phase 8: Segment CLI, Setup, Identity, And Web + +- [x] Move global/repository config into `internal/config`. +- [x] Move SSH key discovery/signing selection into `internal/identity`, while + retaining public signer interfaces in `protocol` or `broker/client`. +- [x] Move AWS/GCP profile discovery, deployment command construction, source + asset materialization, and retry primitives into `internal/setup`; keep the + end-user deployment flow in `internal/app`. +- [x] Move global command parsing, help output, and shared terminal + presentation into `internal/cli`; keep feature-specific dialogs in + `internal/app`. +- [x] Move web routing, assets, CSRF validation, mutation policy, and event + infrastructure into `internal/web`; keep repository-specific handler + adapters in `internal/app`. +- [x] Move executable entry point to `cmd/bgit`. +- [x] Reduce root-level Go files to public packages, docs, and compatibility + adapters scheduled for removal. + +Exit criteria: + +- `cmd/bgit` is a minimal entry point; `internal/app` owns composition and + feature-specific presentation. +- Provider, repository, broker, and transport logic can be tested without + invoking CLI commands. +- `setup.go`, `web.go`, `broker_commands.go`, and `main.go` no longer contain + reusable protocol/storage/Git implementations. + +### Phase 9: Public SDK Hardening + +- [x] Add package documentation and runnable examples. +- [x] Add a top-level SDK overview documenting supported compositions. +- [x] Add examples for: + - filesystem repository; + - S3 repository; + - GCS repository; + - remote AWS/GCP broker client; + - in-process broker; + - upload-pack/receive-pack server; + - custom object store implementation. +- [x] Document concurrency, credential, and authorization responsibilities. +- [x] Run `go vet`, `staticcheck`, race tests, fuzz smoke tests, and API review. +- [x] Generate an exported API inventory and review every symbol. +- [x] Remove or deprecate migration aliases. +- [ ] Publish the first explicitly supported SDK release. + +Exit criteria: + +- A separate Go project can import the packages from a tagged release. +- Examples compile against that tag. +- No example needs internal packages or CLI subprocesses. +- Public API ownership and compatibility policy are documented. + +## Test Architecture + +### Store contract suite + +Provide a reusable test package that accepts a store factory. Run it against: + +- Filesystem store on every platform. +- S3 fake client in unit tests. +- GCS fake client/emulator in unit tests. +- Broker capability store using a fake broker. +- Real S3/GCS only in opt-in smoke tests. + +### Repository conformance suite + +For every backend composition: + +- Empty repository behavior. +- Initial push and clone. +- Fetch and pull. +- Branch creation/deletion. +- Tags and annotated tags. +- Fast-forward and rejected non-fast-forward push. +- Forced updates where policy allows. +- Packed and loose objects. +- Thin packs with existing bases. +- Large files and multiple packs. +- Concurrent ref updates and stale CAS. + +### Broker conformance suite + +Run the same behavioral cases against: + +- In-process Go local broker. +- GCP JavaScript broker test harness. +- AWS embedded JavaScript broker test harness. +- Optional deployed AWS/GCP brokers in smoke tests. + +Cover: + +- Signature v2 and replay rejection. +- Repository visibility and roles. +- Object path capability allow/deny cases. +- Ref listing and CAS. +- Branch protection and PR merge paths. +- Users, teams, invites, ownership transfer, and key types. +- Issues, board ordering/archive state, PRs, CI records, and profiles. +- Materializer token rotation and old-token rejection. + +### Compatibility fixtures + +Keep serialized fixtures for at least the oldest supported broker version and +current version. Test unknown fields, missing optional fields, unknown +endpoints, and scoped compatibility warnings. + +### Platform matrix + +Run unit and local integration suites on: + +- Linux amd64 and arm64 build targets. +- macOS amd64 and arm64 build targets. +- Windows amd64 and arm64 build targets. + +Real cloud smoke tests can remain operator-run, but the package contract and +fake-provider suites must run in normal CI. + +## CI Changes + +- Add package-level jobs so failures identify `protocol`, `store`, + `repository`, `broker`, `transport`, or CLI ownership. +- Run `go test -race` for provider-neutral packages on Linux. +- Compile all examples. +- Run JavaScript syntax and protocol fixture tests for both broker runtimes. +- Retain the existing full `go test ./...` and local broker integration suites. +- Add module/API checks before release. +- Keep real cloud smoke tests manual and credential-gated. + +## Documentation Deliverables + +- `spec/broker-v2.md`: wire protocol and security contract. +- Package docs for every exported package. +- SDK overview with the five supported compositions. +- Migration guide for code currently copying bgit internals. +- Custom store example. +- Custom broker-client example. +- Security guide explaining the difference between raw storage access and + broker-authorized access. +- Version compatibility table for SDK, CLI, and broker protocol. + +## Release And Rollout Plan + +1. Merge each phase independently with no user-visible behavior change unless + explicitly documented. +2. Publish normal `bgit` patch/minor releases during extraction. +3. Treat the generated `API.md` surface as the v1 compatibility contract from + the first release containing the segmented module path. +4. Keep old internal adapters for at least one release after their replacement. +5. Deploy current broker assets before relying on newly added optional fields. +6. Do not require a broker upgrade for package-only refactors. +7. Announce the first supported SDK release only after external-example tests + compile from a clean module. +8. Re-evaluate a separate `bucketgit-go` repository/module after two stable SDK + releases. Split only if independent release cadence provides clear value. + +## Risks And Mitigations + +### Accidental wire incompatibility + +Mitigation: protocol fixtures, cross-language signing vectors, captured JSON, +and old-broker compatibility tests before moving call sites. + +### Over-exported API + +Mitigation: keep implementation types unexported, use narrow interfaces, +publish as `v0`, and require an API review for every exported symbol. + +### Circular dependencies + +Mitigation: enforce the dependency graph in CI. Keep provider-neutral +interfaces in lower-level packages and composition in the CLI. + +### Provider SDK leakage + +Mitigation: provider-specific clients remain in provider packages. Generic +repository and broker APIs use BucketGit-owned interfaces/types. + +### Security regression during refactor + +Mitigation: preserve signed-request and object-path tests, test CAS at package +boundaries, and prohibit direct ref writes in broker-backed composition. + +### Local state corruption or incompatible migration + +Mitigation: fixture repositories from older releases, read/write round-trip +tests, backup-before-migration if a future format change becomes unavoidable, +and no format change in this segmentation effort. + +### Refactor duration and merge conflicts + +Mitigation: extract one boundary at a time, use adapters, avoid broad renames, +and require the full suite to pass after every phase. + +## Suggested Pull Request Sequence + +Keep pull requests reviewable and independently releasable: + +1. Module path, ADR, baseline tests, and architecture checks. +2. Protocol constants/models without call-site migration. +3. V2 signer plus cross-language golden vectors. +4. Store interfaces and filesystem implementation. +5. S3 implementation. +6. GCS implementation. +7. Repository object/ref core. +8. Pack and revision engine. +9. Broker HTTP client. +10. Capability-backed store. +11. Local broker state/core service. +12. Local broker issues/board/PR/protection services. +13. Upload-pack and receive-pack transport. +14. Remote helper and SSH adapters. +15. Internal config/identity/setup packages. +16. Web and CLI segmentation. +17. Public examples, API review, and SDK release documentation. + +Avoid PRs that simultaneously move files, rename all symbols, alter behavior, +and change protocol payloads. + +## Definition Of Done + +The segmentation is complete when: + +- [x] `bgit` is composed from public Git/storage/broker/transport packages and + internal CLI/setup/web packages. +- [x] The five repository modes use the same provider-neutral abstractions. +- [ ] Another Go module can import a tagged BucketGit release and open or serve + repositories without invoking `bgit` as a subprocess. +- [x] Filesystem, S3, GCS, remote broker, and local broker implementations pass + shared contract suites. +- [x] AWS and GCP JavaScript brokers pass the same protocol fixtures. +- [x] Existing repositories and local-broker state require no migration. +- [x] Existing supported brokers remain compatible. +- [x] Object capabilities, signature v2, branch protection, CAS, and local + conflict safeguards retain direct test coverage. +- [x] The CLI, native Git SSH bridge, Git remote helper, web UI, and full + integration suite pass on supported platforms. +- [ ] Public API documentation, examples, version policy, and security + responsibilities are published. diff --git a/architecture/0001-go-package-segmentation.md b/architecture/0001-go-package-segmentation.md new file mode 100644 index 0000000..12ff125 --- /dev/null +++ b/architecture/0001-go-package-segmentation.md @@ -0,0 +1,47 @@ +# ADR 0001: Segment BucketGit Into Importable Go Packages + +Status: accepted + +## Context + +BucketGit's CLI, Git repository engine, cloud stores, broker protocol, local +broker, and transport previously shared one root package. Other Go programs +cannot reuse the protocol or repository behavior without copying code or +invoking the CLI. + +## Decision + +BucketGit will expose provider-neutral packages for protocol, storage, Git +repository behavior, broker clients/local brokers, and Git transport. The +executable delegates through a small root compatibility facade to +`internal/app`; configuration, setup, identity discovery, command parsing, and +web presentation live in focused `internal` packages. + +Repository modes are compositions: + +- `file://`: local broker plus filesystem store. +- `s3://`: local broker plus S3 store. +- `gs://`: local broker plus GCS store. +- AWS broker: remote broker client plus capability-backed S3 store. +- GCP broker: remote broker client plus capability-backed GCS store. + +The SDK remains in the `bucketgit/bgit` repository and uses module path +`github.com/bucketgit/bgit`. Because BucketGit already publishes v1 tags, the +segmented SDK follows the module's v1 compatibility contract. + +## Constraints + +- Existing storage/state formats and broker protocol remain compatible. +- Final broker-backed ref updates remain broker-owned and compare-and-swap. +- Public packages cannot import CLI/internal packages. +- The Git repository package cannot import provider or broker implementations. +- AWS/GCP JavaScript brokers share golden protocol fixtures with Go. + +## Consequences + +- Migration uses adapters and type aliases to keep every intermediate commit + releasable. +- Provider SDK dependencies remain isolated in provider packages. +- Package contracts and cross-language protocol fixtures become release gates. +- A separate SDK repository may be considered only after the public API has + survived two stable releases. diff --git a/architecture/segmentation-baseline.md b/architecture/segmentation-baseline.md new file mode 100644 index 0000000..62f9063 --- /dev/null +++ b/architecture/segmentation-baseline.md @@ -0,0 +1,37 @@ +# Segmentation Verification Baseline + +This record captures the behavior used to verify the package-segmentation +refactor. The migration does not intentionally alter CLI output, repository +layout, broker JSON, Git transport, or local-broker state. + +## Verification + +The segmented implementation passes: + +- `go test ./...` +- `go test -race ./...` +- `go vet ./...` +- `staticcheck ./...` +- `node broker/test_support/signing_fixture_test.js` +- `go test ./...` from the independent `testsuite/sdk-consumer` module +- `./testsuite/run-local-broker.sh gcp` (25 integration files) +- `./testsuite/run-local-broker.sh aws` (25 integration files) +- cross-compilation for Darwin amd64/arm64, Linux amd64/arm64, and Windows + amd64/arm64 +- fuzz smoke tests for objects, refs, pack indexes, deltas, pkt-lines, and + receive packs + +## Behavioral Golden Coverage + +CLI and protocol behavior is captured by the existing tests rather than copied +into a second snapshot format: + +- internal application tests cover usage, clone, fetch, pull, push, PR, board, issue, + CI, local-broker, SSH bridge, and web behavior; +- `testsuite/local` captures local porcelain behavior; +- `testsuite/aws` and `testsuite/gcp` execute identical broker scenarios; +- `spec/testdata/signing-v2.json` is the cross-language signature fixture; +- `testsuite/sdk-consumer` proves use from outside the BucketGit module. + +The command list above is the release baseline. Any expected output change must +update its owning focused test and release notes. diff --git a/architecture_test.go b/architecture_test.go new file mode 100644 index 0000000..3ed4a89 --- /dev/null +++ b/architecture_test.go @@ -0,0 +1,55 @@ +package bgit + +import ( + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestPublicPackageDependencies(t *testing.T) { + publicRoots := []string{"protocol", "store", "repository", "broker", "transport"} + for _, root := range publicRoots { + if _, err := os.Stat(root); err != nil { + continue + } + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".go" || strings.HasSuffix(path, "_test.go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + if err != nil { + return err + } + for _, spec := range file.Imports { + importPath, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return err + } + if strings.Contains(importPath, "/internal/") || importPath == "github.com/bucketgit/bgit/internal" { + t.Errorf("public package file %s imports internal package %s", path, importPath) + } + if root == "protocol" && strings.HasPrefix(importPath, "github.com/bucketgit/bgit/") { + t.Errorf("protocol file %s imports higher-level BucketGit package %s", path, importPath) + } + if root == "repository" && (strings.Contains(importPath, "cloud.google.com/") || strings.Contains(importPath, "aws-sdk-go") || strings.Contains(importPath, "/broker/")) { + t.Errorf("repository file %s imports provider or broker package %s", path, importPath) + } + if root == "store" && strings.Contains(importPath, "github.com/bucketgit/bgit/broker/") { + t.Errorf("store file %s imports broker package %s", path, importPath) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } +} diff --git a/bgit.go b/bgit.go new file mode 100644 index 0000000..9b5f9e9 --- /dev/null +++ b/bgit.go @@ -0,0 +1,47 @@ +// Package bgit provides the compatibility entry points used by the bgit +// executable. Reusable repository functionality lives in the protocol, store, +// repository, broker, and transport packages. +package bgit + +import ( + "embed" + "fmt" + "io" + "os" + "sync" + + "github.com/bucketgit/bgit/internal/app" +) + +//go:embed CHANGELOG.md +var embeddedChangelog string + +//go:embed broker/gcp/package.json broker/gcp/index.js broker/gcp/materializer.js broker/aws/template.yaml broker/test_support/sqlite_broker.js +var brokerAssets embed.FS + +var version = "" +var configureOnce sync.Once + +func configureApplication() { + configureOnce.Do(func() { + app.ConfigureRuntime(app.RuntimeOptions{Version: version, Changelog: embeddedChangelog}) + app.ConfigureBrokerAssets(brokerAssets) + }) +} + +func Main() { + if err := RunExecutable(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "fatal:", err) + os.Exit(1) + } +} + +func RunExecutable(executable string, args []string, stdin io.Reader, stdout, stderr io.Writer) error { + configureApplication() + return app.RunExecutable(executable, args, stdin, stdout, stderr) +} + +func Run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { + configureApplication() + return app.Run(args, stdin, stdout, stderr) +} diff --git a/broker/capability/store.go b/broker/capability/store.go new file mode 100644 index 0000000..bc94734 --- /dev/null +++ b/broker/capability/store.go @@ -0,0 +1,344 @@ +// Package capability exposes a BucketGit object store backed by broker-issued +// short-lived object capabilities and broker-owned ref operations. +package capability + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + brokerclient "github.com/bucketgit/bgit/broker/client" + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" +) + +type Broker interface { + PostJSON(ctx context.Context, endpoint string, request, response any, headers http.Header) error +} + +type LocalHandler interface { + Read(ctx context.Context, capability protocol.ObjectCapabilityResponse) ([]byte, error) + Write(ctx context.Context, capability protocol.ObjectCapabilityResponse, data []byte) error + Delete(ctx context.Context, capability protocol.ObjectCapabilityResponse) error +} + +type ErrorClassifier func(error) bool + +type Options struct { + HTTPClient *http.Client + Local LocalHandler + IsNotFound ErrorClassifier + IsUnsupported ErrorClassifier + ResumableAt int64 +} + +type Store struct { + broker Broker + repo protocol.Repository + opts Options +} + +func New(broker Broker, repo protocol.Repository, opts Options) (*Store, error) { + if broker == nil { + return nil, fmt.Errorf("broker client is required") + } + if err := repo.Validate(); err != nil { + return nil, err + } + if opts.HTTPClient == nil { + opts.HTTPClient = http.DefaultClient + } + if opts.ResumableAt == 0 { + opts.ResumableAt = 32 * 1024 * 1024 + } + return &Store{broker: broker, repo: repo, opts: opts}, nil +} + +func (s *Store) Read(ctx context.Context, objectPath string) ([]byte, error) { + objectPath, err := store.ValidatePath(objectPath, false) + if err != nil { + return nil, err + } + capability, err := s.objectCapability(ctx, objectPath, string(protocol.OperationRead), 0) + if err == nil { + return s.readCapability(ctx, capability) + } + if s.isNotFound(err) { + return nil, fs.ErrNotExist + } + if !s.isUnsupported(err) { + return nil, err + } + var response protocol.ObjectResponse + if err := s.broker.PostJSON(ctx, "/objects/read", protocol.ObjectRequest{Repo: s.repo, Path: objectPath}, &response, nil); err != nil { + if s.isNotFound(err) { + return nil, fs.ErrNotExist + } + return nil, err + } + return base64.StdEncoding.DecodeString(response.Data) +} + +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + prefix, err := store.ValidatePath(prefix, true) + if err != nil { + return nil, err + } + var response protocol.ObjectResponse + if err := s.broker.PostJSON(ctx, "/objects/list", protocol.ObjectRequest{Repo: s.repo, Prefix: prefix}, &response, nil); err != nil { + return nil, err + } + return response.Paths, nil +} + +func (s *Store) Write(ctx context.Context, objectPath string, data []byte) error { + objectPath, err := store.ValidatePath(objectPath, false) + if err != nil { + return err + } + capability, err := s.objectCapability(ctx, objectPath, string(protocol.OperationWrite), int64(len(data))) + if err != nil { + return err + } + return s.writeCapability(ctx, capability, data) +} + +func (s *Store) Delete(ctx context.Context, objectPath string) error { + objectPath, err := store.ValidatePath(objectPath, false) + if err != nil { + return err + } + capability, err := s.objectCapability(ctx, objectPath, string(protocol.OperationDelete), 0) + if err != nil { + return err + } + return s.deleteCapability(ctx, capability) +} + +func (s *Store) ListRefs(ctx context.Context) (map[string]string, error) { + var response protocol.RefsResponse + if err := s.broker.PostJSON(ctx, "/refs/list", protocol.RefsRequest{Repo: s.repo}, &response, nil); err != nil { + return nil, err + } + refs := make(map[string]string, len(response.Refs)) + for ref, oid := range response.Refs { + oid = strings.TrimSpace(oid) + if strings.HasPrefix(ref, "refs/") && isSHA1(oid) { + refs[ref] = oid + } + } + return refs, nil +} + +func isSHA1(value string) bool { + if len(value) != 40 { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} + +func (s *Store) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + return s.broker.PostJSON(ctx, "/refs/update", protocol.RefUpdateRequest{ + Repo: s.repo, Ref: ref, Old: oldOID, New: newOID, + }, nil, nil) +} + +func (s *Store) objectCapability(ctx context.Context, objectPath, operation string, size int64) (protocol.ObjectCapabilityResponse, error) { + var response protocol.ObjectCapabilityResponse + err := s.broker.PostJSON(ctx, "/objects/capability", protocol.ObjectCapabilityRequest{ + Repo: s.repo, Path: objectPath, Operation: operation, Size: size, + Resumable: s.repo.Provider == string(protocol.ProviderGCS) && operation == string(protocol.OperationWrite) && size > s.opts.ResumableAt, + }, &response, nil) + return response, err +} + +func (s *Store) readCapability(ctx context.Context, capability protocol.ObjectCapabilityResponse) ([]byte, error) { + if capability.Mode == "local" { + if s.opts.Local == nil { + return nil, fmt.Errorf("local capability handler is required") + } + return s.opts.Local.Read(ctx, capability) + } + if capability.Mode == "sts" || capability.Provider == string(protocol.ProviderS3) { + client := s3Client(capability) + out, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(capability.Bucket), Key: aws.String(capability.Object)}) + if err != nil { + if isS3NotFound(err) { + return nil, fs.ErrNotExist + } + return nil, err + } + defer out.Body.Close() + return io.ReadAll(out.Body) + } + if err := validateCapabilityURL(capability.URL); err != nil { + return nil, err + } + request, err := http.NewRequestWithContext(ctx, firstNonEmpty(capability.Method, http.MethodGet), capability.URL, nil) + if err != nil { + return nil, err + } + setHeaders(request.Header, capability.Headers) + response, err := s.opts.HTTPClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return nil, fs.ErrNotExist + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, responseError("GET", response) + } + return io.ReadAll(response.Body) +} + +func (s *Store) writeCapability(ctx context.Context, capability protocol.ObjectCapabilityResponse, data []byte) error { + if capability.Mode == "local" { + if s.opts.Local == nil { + return fmt.Errorf("local capability handler is required") + } + return s.opts.Local.Write(ctx, capability, data) + } + if capability.Mode == "sts" || capability.Provider == string(protocol.ProviderS3) { + _, err := s3Client(capability).PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(capability.Bucket), Key: aws.String(capability.Object), Body: bytes.NewReader(data), + }) + return err + } + method := firstNonEmpty(capability.Method, http.MethodPut) + if err := validateCapabilityURL(capability.URL); err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, method, capability.URL, bytes.NewReader(data)) + if err != nil { + return err + } + setHeaders(request.Header, capability.Headers) + response, err := s.opts.HTTPClient.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return responseError(method, response) + } + return nil +} + +func (s *Store) deleteCapability(ctx context.Context, capability protocol.ObjectCapabilityResponse) error { + if capability.Mode == "local" { + if s.opts.Local == nil { + return fmt.Errorf("local capability handler is required") + } + return s.opts.Local.Delete(ctx, capability) + } + if capability.Mode == "sts" || capability.Provider == string(protocol.ProviderS3) { + _, err := s3Client(capability).DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(capability.Bucket), Key: aws.String(capability.Object), + }) + return err + } + method := firstNonEmpty(capability.Method, http.MethodDelete) + if err := validateCapabilityURL(capability.URL); err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, method, capability.URL, nil) + if err != nil { + return err + } + setHeaders(request.Header, capability.Headers) + response, err := s.opts.HTTPClient.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return responseError(method, response) + } + return nil +} + +func (s *Store) isNotFound(err error) bool { + return errors.Is(err, fs.ErrNotExist) || (s.opts.IsNotFound != nil && s.opts.IsNotFound(err)) +} + +func (s *Store) isUnsupported(err error) bool { + return errors.Is(err, protocol.ErrUnsupported) || (s.opts.IsUnsupported != nil && s.opts.IsUnsupported(err)) +} + +func s3Client(capability protocol.ObjectCapabilityResponse) *s3.Client { + provider := credentials.NewStaticCredentialsProvider( + capability.Credentials.AccessKeyID, + capability.Credentials.SecretAccessKey, + capability.Credentials.SessionToken, + ) + return s3.New(s3.Options{Region: firstNonEmpty(capability.Region, "us-east-1"), Credentials: aws.NewCredentialsCache(provider)}) +} + +func setHeaders(target http.Header, values map[string]string) { + for key, value := range values { + target.Set(key, value) + } +} + +func responseError(method string, response *http.Response) error { + body, _ := io.ReadAll(response.Body) + return fmt.Errorf("broker object %s: %s %s", method, response.Status, strings.TrimSpace(string(body))) +} + +func validateCapabilityURL(value string) error { + parsed, err := url.Parse(strings.TrimSpace(value)) + if err != nil || parsed.Host == "" || parsed.User != nil || (parsed.Scheme != "https" && parsed.Scheme != "http") { + return errors.New("broker returned an invalid object capability URL") + } + if parsed.Scheme == "http" { + host := parsed.Hostname() + if host != "localhost" && host != "127.0.0.1" && host != "::1" { + return errors.New("broker returned an insecure object capability URL") + } + } + return nil +} + +func isS3NotFound(err error) bool { + var apiErr smithy.APIError + if !errors.As(err, &apiErr) { + return false + } + switch apiErr.ErrorCode() { + case "NoSuchBucket", "NoSuchKey", "NotFound", "404": + return true + default: + return false + } +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +var _ Broker = (*brokerclient.Client)(nil) +var _ store.Writer = (*Store)(nil) +var _ store.RefStore = (*Store)(nil) diff --git a/broker/capability/store_test.go b/broker/capability/store_test.go new file mode 100644 index 0000000..ac64c5a --- /dev/null +++ b/broker/capability/store_test.go @@ -0,0 +1,176 @@ +package capability + +import ( + "context" + "encoding/base64" + "io" + "io/fs" + "net/http" + "sort" + "strings" + "sync" + "testing" + + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" + "github.com/bucketgit/bgit/store/storetest" +) + +type fakeBroker struct { + post func(context.Context, string, any, any) error +} + +func TestCapabilityRejectsInsecureRemoteURL(t *testing.T) { + broker := fakeBroker{post: func(_ context.Context, _ string, _ any, response any) error { + *response.(*protocol.ObjectCapabilityResponse) = protocol.ObjectCapabilityResponse{URL: "http://metadata.google.internal/token", Method: http.MethodGet} + return nil + }} + s, err := New(broker, protocol.Repository{Logical: "demo.git"}, Options{}) + if err != nil { + t.Fatal(err) + } + if _, err := s.Read(context.Background(), "HEAD"); err == nil || !strings.Contains(err.Error(), "insecure") { + t.Fatalf("error=%v", err) + } +} + +func (f fakeBroker) PostJSON(ctx context.Context, endpoint string, request, response any, _ http.Header) error { + return f.post(ctx, endpoint, request, response) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestReadSignedURLCapability(t *testing.T) { + broker := fakeBroker{post: func(_ context.Context, endpoint string, _ any, response any) error { + if endpoint != "/objects/capability" { + t.Fatalf("endpoint = %s", endpoint) + } + *response.(*protocol.ObjectCapabilityResponse) = protocol.ObjectCapabilityResponse{URL: "https://objects.example.com/blob", Method: http.MethodGet} + return nil + }} + httpClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Status: "200 OK", Body: io.NopCloser(strings.NewReader("payload")), Header: http.Header{}}, nil + })} + s, err := New(broker, protocol.Repository{Logical: "demo.git"}, Options{HTTPClient: httpClient}) + if err != nil { + t.Fatal(err) + } + data, err := s.Read(context.Background(), "objects/ab/cdef") + if err != nil || string(data) != "payload" { + t.Fatalf("Read = %q, %v", data, err) + } +} + +func TestReadLegacyFallback(t *testing.T) { + broker := fakeBroker{post: func(_ context.Context, endpoint string, _ any, response any) error { + switch endpoint { + case "/objects/capability": + return protocol.ErrUnsupported + case "/objects/read": + *response.(*protocol.ObjectResponse) = protocol.ObjectResponse{Data: base64.StdEncoding.EncodeToString([]byte("legacy"))} + return nil + default: + t.Fatalf("endpoint = %s", endpoint) + return nil + } + }} + s, err := New(broker, protocol.Repository{Logical: "demo.git"}, Options{}) + if err != nil { + t.Fatal(err) + } + data, err := s.Read(context.Background(), "HEAD") + if err != nil || string(data) != "legacy" { + t.Fatalf("Read = %q, %v", data, err) + } +} + +func TestCapabilityStoreContract(t *testing.T) { + storetest.Run(t, func(t *testing.T) store.Writer { + memory := &capabilityMemory{values: map[string][]byte{}} + broker := capabilityContractBroker{memory: memory} + backend, err := New(broker, protocol.Repository{Logical: "contract.git"}, Options{Local: memory}) + if err != nil { + t.Fatal(err) + } + return backend + }) +} + +type capabilityContractBroker struct{ memory *capabilityMemory } + +func (b capabilityContractBroker) PostJSON(ctx context.Context, endpoint string, request, response any, _ http.Header) error { + if err := ctx.Err(); err != nil { + return err + } + switch endpoint { + case "/objects/capability": + value := request.(protocol.ObjectCapabilityRequest) + *response.(*protocol.ObjectCapabilityResponse) = protocol.ObjectCapabilityResponse{Mode: "local", Object: value.Path} + case "/objects/list": + value := request.(protocol.ObjectRequest) + paths, err := b.memory.list(ctx, value.Prefix) + if err != nil { + return err + } + *response.(*protocol.ObjectResponse) = protocol.ObjectResponse{Paths: paths} + default: + return protocol.ErrUnsupported + } + return nil +} + +type capabilityMemory struct { + mu sync.RWMutex + values map[string][]byte +} + +func (m *capabilityMemory) Read(ctx context.Context, capability protocol.ObjectCapabilityResponse) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + m.mu.RLock() + defer m.mu.RUnlock() + value, ok := m.values[capability.Object] + if !ok { + return nil, fs.ErrNotExist + } + return append([]byte(nil), value...), nil +} +func (m *capabilityMemory) Write(ctx context.Context, capability protocol.ObjectCapabilityResponse, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + m.values[capability.Object] = append([]byte(nil), data...) + return nil +} +func (m *capabilityMemory) Delete(ctx context.Context, capability protocol.ObjectCapabilityResponse) error { + if err := ctx.Err(); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.values[capability.Object]; !ok { + return fs.ErrNotExist + } + delete(m.values, capability.Object) + return nil +} +func (m *capabilityMemory) list(ctx context.Context, prefix string) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + m.mu.RLock() + defer m.mu.RUnlock() + var paths []string + for path := range m.values { + if strings.HasPrefix(path, strings.TrimSuffix(prefix, "/")) { + paths = append(paths, path) + } + } + sort.Strings(paths) + return paths, nil +} diff --git a/broker/client/client.go b/broker/client/client.go new file mode 100644 index 0000000..2401d98 --- /dev/null +++ b/broker/client/client.go @@ -0,0 +1,163 @@ +// Package client implements the signed BucketGit broker HTTP protocol. +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/bucketgit/bgit/protocol" +) + +type SignatureProvider interface { + HeaderSets(ctx context.Context, baseURL, path string, payload []byte) ([]http.Header, error) +} + +type SignatureProviderFunc func(context.Context, string, string, []byte) ([]http.Header, error) + +func (f SignatureProviderFunc) HeaderSets(ctx context.Context, baseURL, path string, payload []byte) ([]http.Header, error) { + return f(ctx, baseURL, path, payload) +} + +type ErrorDecoder func(endpoint string, status int, message string) error +type RetryPredicate func(status int, message string) bool +type SuccessObserver func(baseURL string, payload []byte, headers http.Header) + +type Options struct { + HTTPClient *http.Client + Signatures SignatureProvider + AllowUnsignedFallback bool + DecodeError ErrorDecoder + Retry RetryPredicate + ObserveSuccess SuccessObserver + MaxResponseBytes int64 +} + +type Client struct { + baseURL string + http *http.Client + opts Options +} + +func New(baseURL string, opts Options) (*Client, error) { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + parsed, err := url.Parse(baseURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return nil, fmt.Errorf("invalid broker URL %q", baseURL) + } + if opts.HTTPClient == nil { + opts.HTTPClient = http.DefaultClient + } + if opts.MaxResponseBytes <= 0 { + opts.MaxResponseBytes = 64 << 20 + } + return &Client{baseURL: baseURL, http: opts.HTTPClient, opts: opts}, nil +} + +func (c *Client) PostJSON(ctx context.Context, endpoint string, request, response any, extraHeaders http.Header) error { + payload, err := json.Marshal(request) + if err != nil { + return err + } + return c.PostJSONBytes(ctx, endpoint, payload, response, extraHeaders) +} + +func (c *Client) PostJSONBytes(ctx context.Context, endpoint string, payload []byte, response any, extraHeaders http.Header) error { + if !strings.HasPrefix(endpoint, "/") { + return fmt.Errorf("broker endpoint must start with /: %q", endpoint) + } + parsedEndpoint, err := url.Parse(endpoint) + if err != nil || parsedEndpoint.IsAbs() || parsedEndpoint.Host != "" || parsedEndpoint.RawQuery != "" || parsedEndpoint.Fragment != "" || strings.HasPrefix(endpoint, "//") { + return fmt.Errorf("invalid broker endpoint %q", endpoint) + } + headerSets, err := c.signatureHeaderSets(ctx, endpoint, payload) + if err != nil { + return err + } + var lastErr error + for index, headers := range headerSets { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+endpoint, bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("content-type", "application/json") + copyHeaders(req.Header, headers) + copyHeaders(req.Header, extraHeaders) + resp, err := c.http.Do(req) + if err != nil { + return err + } + body, readErr := io.ReadAll(io.LimitReader(resp.Body, c.opts.MaxResponseBytes+1)) + _ = resp.Body.Close() + if readErr != nil { + return readErr + } + if int64(len(body)) > c.opts.MaxResponseBytes { + return fmt.Errorf("broker %s response exceeds %d bytes", endpoint, c.opts.MaxResponseBytes) + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + if c.opts.ObserveSuccess != nil { + c.opts.ObserveSuccess(c.baseURL, payload, headers) + } + if response != nil && len(body) > 0 { + if err := json.Unmarshal(body, response); err != nil { + return err + } + } + return nil + } + message := strings.TrimSpace(string(body)) + if message == "" { + message = resp.Status + } + lastErr = c.decodeError(endpoint, resp.StatusCode, message) + if index == len(headerSets)-1 || c.opts.Retry == nil || !c.opts.Retry(resp.StatusCode, message) { + return lastErr + } + } + return lastErr +} + +func (c *Client) signatureHeaderSets(ctx context.Context, endpoint string, payload []byte) ([]http.Header, error) { + var sets []http.Header + if c.opts.Signatures != nil { + var err error + sets, err = c.opts.Signatures.HeaderSets(ctx, c.baseURL, endpoint, payload) + if err != nil { + return nil, err + } + } + if c.opts.AllowUnsignedFallback || len(sets) == 0 { + sets = append(sets, http.Header{}) + } + return sets, nil +} + +func (c *Client) decodeError(endpoint string, status int, message string) error { + if c.opts.DecodeError != nil { + return c.opts.DecodeError(endpoint, status, message) + } + kind := error(nil) + switch status { + case http.StatusUnauthorized, http.StatusForbidden: + kind = protocol.ErrUnauthorized + case http.StatusConflict: + kind = protocol.ErrConflict + case http.StatusNotImplemented: + kind = protocol.ErrUnsupported + } + return &protocol.BrokerError{Endpoint: endpoint, Status: status, Message: message, Kind: kind} +} + +func copyHeaders(target, source http.Header) { + for key, values := range source { + for _, value := range values { + target.Add(key, value) + } + } +} diff --git a/broker/client/client_test.go b/broker/client/client_test.go new file mode 100644 index 0000000..cdf1997 --- /dev/null +++ b/broker/client/client_test.go @@ -0,0 +1,105 @@ +package client + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/bucketgit/bgit/protocol" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestPostJSONRetriesSignatureAndDecodesResponse(t *testing.T) { + var calls int + httpClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return response(http.StatusForbidden, `{"error":"SSH signature required"}`), nil + } + if req.Header.Get(protocol.HeaderKeyFingerprint) != "SHA256:test" { + t.Fatalf("fingerprint = %q", req.Header.Get(protocol.HeaderKeyFingerprint)) + } + return response(http.StatusOK, `{"allowed":true,"user":"ada"}`), nil + })} + c, err := New("https://broker.example.com", Options{ + HTTPClient: httpClient, + Signatures: SignatureProviderFunc(func(context.Context, string, string, []byte) ([]http.Header, error) { + return []http.Header{{protocol.HeaderKeyFingerprint: []string{"bad"}}, {protocol.HeaderKeyFingerprint: []string{"SHA256:test"}}}, nil + }), + Retry: func(status int, message string) bool { + return status == http.StatusForbidden && strings.Contains(message, "SSH signature required") + }, + }) + if err != nil { + t.Fatal(err) + } + var got protocol.AuthResponse + if err := c.PostJSON(context.Background(), "/auth/check", protocol.AuthRequest{}, &got, nil); err != nil { + t.Fatal(err) + } + if !got.Allowed || got.User != "ada" || calls != 2 { + t.Fatalf("response = %#v, calls = %d", got, calls) + } +} + +func TestPostJSONReturnsTypedError(t *testing.T) { + c, err := New("https://broker.example.com", Options{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return response(http.StatusConflict, `{"error":"stale ref"}`), nil + })}}) + if err != nil { + t.Fatal(err) + } + err = c.PostJSON(context.Background(), "/refs/update", protocol.RefUpdateRequest{}, nil, nil) + if !errors.Is(err, protocol.ErrConflict) { + t.Fatalf("error = %v", err) + } + var brokerErr *protocol.BrokerError + if !errors.As(err, &brokerErr) || brokerErr.Endpoint != "/refs/update" || brokerErr.Status != http.StatusConflict { + t.Fatalf("broker error = %#v", brokerErr) + } +} + +func TestPostJSONHonorsCancellation(t *testing.T) { + client, err := New("https://broker.example.com", Options{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + <-req.Context().Done() + return nil, req.Context().Err() + })}}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := client.PostJSON(ctx, "/auth/check", protocol.AuthRequest{}, nil, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled error=%v", err) + } +} + +func TestPostJSONLimitsResponseBody(t *testing.T) { + client, err := New("https://broker.example.com", Options{MaxResponseBytes: 8, HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return response(http.StatusOK, `{"allowed":true}`), nil })}}) + if err != nil { + t.Fatal(err) + } + if err := client.PostJSON(context.Background(), "/auth/check", protocol.AuthRequest{}, nil, nil); err == nil || !strings.Contains(err.Error(), "response exceeds") { + t.Fatalf("limit error=%v", err) + } +} + +func TestPostJSONRejectsEndpointAuthority(t *testing.T) { + client, err := New("https://broker.example.com", Options{}) + if err != nil { + t.Fatal(err) + } + if err := client.PostJSON(context.Background(), "//attacker.example/path", nil, nil, nil); err == nil { + t.Fatal("authority endpoint accepted") + } +} + +func response(status int, body string) *http.Response { + return &http.Response{StatusCode: status, Status: http.StatusText(status), Body: io.NopCloser(strings.NewReader(body)), Header: http.Header{}} +} diff --git a/broker/client/endpoints.go b/broker/client/endpoints.go new file mode 100644 index 0000000..5630ea2 --- /dev/null +++ b/broker/client/endpoints.go @@ -0,0 +1,385 @@ +package client + +import ( + "context" + "net/http" + + "github.com/bucketgit/bgit/protocol" +) + +type Caller interface { + PostJSON(ctx context.Context, endpoint string, request, response any, headers http.Header) error +} + +type Endpoints struct{ caller Caller } + +func NewEndpoints(caller Caller) *Endpoints { return &Endpoints{caller: caller} } + +func (e *Endpoints) call(ctx context.Context, path string, request, response any) error { + return e.caller.PostJSON(ctx, path, request, response, nil) +} + +func (e *Endpoints) Authorize(ctx context.Context, request protocol.AuthRequest) (protocol.AuthResponse, error) { + var response protocol.AuthResponse + err := e.call(ctx, "/auth/check", request, &response) + return response, err +} +func (e *Endpoints) AuthStatus(ctx context.Context, request protocol.AuthStatusRequest) (protocol.AuthStatus, error) { + var response protocol.AuthStatus + err := e.call(ctx, "/auth/status", request, &response) + return response, err +} +func (e *Endpoints) ListRefs(ctx context.Context, repo protocol.Repository) (map[string]string, error) { + var response protocol.RefsResponse + err := e.call(ctx, "/refs/list", protocol.RefsRequest{Repo: repo}, &response) + return response.Refs, err +} +func (e *Endpoints) UpdateRef(ctx context.Context, request protocol.RefUpdateRequest) error { + return e.call(ctx, "/refs/update", request, nil) +} +func (e *Endpoints) ObjectCapability(ctx context.Context, request protocol.ObjectCapabilityRequest) (protocol.ObjectCapabilityResponse, error) { + var response protocol.ObjectCapabilityResponse + err := e.call(ctx, "/objects/capability", request, &response) + return response, err +} + +func (e *Endpoints) ListIssues(ctx context.Context, request protocol.IssueRequest) ([]protocol.Issue, error) { + var response struct { + Issues []protocol.Issue `json:"issues"` + } + err := e.call(ctx, "/issues/list", request, &response) + return response.Issues, err +} +func (e *Endpoints) GetIssue(ctx context.Context, request protocol.IssueRequest) (protocol.Issue, error) { + var response struct { + Issue protocol.Issue `json:"issue"` + } + err := e.call(ctx, "/issues/view", request, &response) + return response.Issue, err +} +func (e *Endpoints) CreateIssue(ctx context.Context, request protocol.IssueRequest) (protocol.Issue, error) { + var response struct { + Issue protocol.Issue `json:"issue"` + } + err := e.call(ctx, "/issues/create", request, &response) + return response.Issue, err +} +func (e *Endpoints) UpdateIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/update", request, nil) +} +func (e *Endpoints) CommentIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/comment", request, nil) +} +func (e *Endpoints) CloseIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/close", request, nil) +} +func (e *Endpoints) ReopenIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/reopen", request, nil) +} +func (e *Endpoints) MoveIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/move", request, nil) +} +func (e *Endpoints) ReorderIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/reorder", request, nil) +} +func (e *Endpoints) TakeIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/take", request, nil) +} +func (e *Endpoints) AssignIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/assign", request, nil) +} +func (e *Endpoints) ArchiveIssue(ctx context.Context, request protocol.IssueRequest) error { + return e.call(ctx, "/issues/archive", request, nil) +} +func (e *Endpoints) IssueAssignees(ctx context.Context, request protocol.IssueRequest) ([]string, error) { + var response struct { + Users []string `json:"users"` + } + err := e.call(ctx, "/issues/assignees", request, &response) + return response.Users, err +} + +func (e *Endpoints) ListPullRequests(ctx context.Context, request protocol.PullRequestRequest) ([]protocol.PullRequest, error) { + var response protocol.PullRequestsResponse + err := e.call(ctx, "/prs/list", request, &response) + return response.PRs, err +} +func (e *Endpoints) GetPullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + var response protocol.PullRequestResponse + err := e.call(ctx, "/prs/view", request, &response) + return response.PR, err +} +func (e *Endpoints) CreatePullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + var response protocol.PullRequestResponse + err := e.call(ctx, "/prs/create", request, &response) + return response.PR, err +} +func (e *Endpoints) SyncPullRequests(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequestsResponse, error) { + var response protocol.PullRequestsResponse + err := e.call(ctx, "/prs/sync", request, &response) + return response, err +} +func (e *Endpoints) CommentPullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + return e.pullRequestMutation(ctx, "/prs/comment", request) +} +func (e *Endpoints) ReviewPullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + return e.pullRequestMutation(ctx, "/prs/review", request) +} +func (e *Endpoints) ReplyPullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + return e.pullRequestMutation(ctx, "/prs/reply", request) +} +func (e *Endpoints) MergePullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + return e.pullRequestMutation(ctx, "/prs/merge", request) +} +func (e *Endpoints) ClosePullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + return e.pullRequestMutation(ctx, "/prs/close", request) +} +func (e *Endpoints) ReopenPullRequest(ctx context.Context, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + return e.pullRequestMutation(ctx, "/prs/reopen", request) +} +func (e *Endpoints) pullRequestMutation(ctx context.Context, path string, request protocol.PullRequestRequest) (protocol.PullRequest, error) { + var response protocol.PullRequestResponse + err := e.call(ctx, path, request, &response) + return response.PR, err +} + +func (e *Endpoints) ListCIRuns(ctx context.Context, request protocol.CIRequest) ([]protocol.CIRun, error) { + var response struct { + Runs []protocol.CIRun `json:"runs"` + } + err := e.call(ctx, "/ci/list", request, &response) + return response.Runs, err +} +func (e *Endpoints) GetCIRun(ctx context.Context, request protocol.CIRequest) (protocol.CIRun, error) { + var response struct { + Run protocol.CIRun `json:"run"` + } + err := e.call(ctx, "/ci/view", request, &response) + return response.Run, err +} +func (e *Endpoints) RunCI(ctx context.Context, request protocol.CIRequest) (protocol.CIRun, error) { + var response struct { + Run protocol.CIRun `json:"run"` + } + err := e.call(ctx, "/ci/run", request, &response) + return response.Run, err +} +func (e *Endpoints) CILogs(ctx context.Context, request protocol.CIRequest) (protocol.CILogResponse, error) { + var response protocol.CILogResponse + err := e.call(ctx, "/ci/logs", request, &response) + return response, err +} +func (e *Endpoints) RotateCISecret(ctx context.Context, request protocol.CIRequest) error { + return e.call(ctx, "/ci/secret/rotate", request, nil) +} + +func (e *Endpoints) ListKeys(ctx context.Context, request protocol.KeyRequest) ([]protocol.Key, error) { + var response protocol.KeysResponse + err := e.call(ctx, "/keys/list", request, &response) + return response.Keys, err +} +func (e *Endpoints) AddKey(ctx context.Context, request protocol.KeyRequest) error { + return e.call(ctx, "/keys/add", request, nil) +} +func (e *Endpoints) RemoveKey(ctx context.Context, request protocol.KeyRequest) error { + return e.call(ctx, "/keys/remove", request, nil) +} +func (e *Endpoints) SuspendKey(ctx context.Context, request protocol.KeyRequest, suspended bool) error { + path := "/keys/unsuspend" + if suspended { + path = "/keys/suspend" + } + return e.call(ctx, path, request, nil) +} +func (e *Endpoints) CreateRepositoryInvite(ctx context.Context, request protocol.OwnerTransferRequest) (protocol.OwnerTransferResponse, error) { + return e.ownerTransfer(ctx, "/keys/invite/create", request) +} +func (e *Endpoints) AcceptRepositoryInvite(ctx context.Context, request protocol.OwnerTransferRequest) (protocol.OwnerTransferResponse, error) { + return e.ownerTransfer(ctx, "/keys/invite/accept", request) +} +func (e *Endpoints) CancelRepositoryInvite(ctx context.Context, request protocol.OwnerTransferRequest) error { + return e.call(ctx, "/keys/invite/cancel", request, nil) +} +func (e *Endpoints) ListRepositoryInvites(ctx context.Context, request protocol.OwnerTransferRequest) ([]protocol.RepositoryInvite, error) { + var response protocol.RepositoryInvitesResponse + err := e.call(ctx, "/keys/invite/list", request, &response) + return response.Invites, err +} + +func (e *Endpoints) ListUsers(ctx context.Context, request protocol.RepositoryAdminRequest) ([]protocol.UserInfo, error) { + var response protocol.UsersResponse + err := e.call(ctx, "/broker/users/list", request, &response) + return response.Users, err +} +func (e *Endpoints) UpsertUser(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.UserInfo, error) { + var response struct { + User protocol.UserInfo `json:"user"` + } + err := e.call(ctx, "/broker/users/upsert", request, &response) + return response.User, err +} +func (e *Endpoints) DeleteUser(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/broker/users/delete", request, nil) +} +func (e *Endpoints) CreateBrokerUserInvite(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.OwnerTransferResponse, error) { + var response protocol.OwnerTransferResponse + err := e.call(ctx, "/broker/users/invite/create", request, &response) + return response, err +} +func (e *Endpoints) AcceptBrokerUserInvite(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.OwnerTransferResponse, error) { + var response protocol.OwnerTransferResponse + err := e.call(ctx, "/broker/users/invite/accept", request, &response) + return response, err +} +func (e *Endpoints) CancelBrokerUserInvite(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/broker/users/invite/cancel", request, nil) +} + +func (e *Endpoints) ListTeams(ctx context.Context, request protocol.RepositoryAdminRequest) ([]protocol.Team, error) { + var response protocol.TeamsResponse + err := e.call(ctx, "/teams/list", request, &response) + return response.Teams, err +} +func (e *Endpoints) ResolveTeam(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.Team, error) { + var response struct { + Team protocol.Team `json:"team"` + } + err := e.call(ctx, "/teams/resolve", request, &response) + return response.Team, err +} +func (e *Endpoints) CreateTeam(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.Team, error) { + var response struct { + Team protocol.Team `json:"team"` + } + err := e.call(ctx, "/teams/create", request, &response) + return response.Team, err +} +func (e *Endpoints) DeleteTeam(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/teams/delete", request, nil) +} +func (e *Endpoints) UpsertTeamMember(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/teams/member/upsert", request, nil) +} +func (e *Endpoints) RemoveTeamMember(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/teams/member/remove", request, nil) +} + +func (e *Endpoints) ListRepositories(ctx context.Context, request protocol.RepositoryAdminRequest) ([]protocol.RepositoryInfo, error) { + var response protocol.RepositoryListResponse + err := e.call(ctx, "/repos/list", request, &response) + return response.Repos, err +} +func (e *Endpoints) GetRepository(ctx context.Context, request protocol.RepositoryRequest) (protocol.RepositoryInfo, error) { + var response protocol.RepositoryInfo + err := e.call(ctx, "/repos/get", request, &response) + return response, err +} +func (e *Endpoints) CreateRepository(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.RepositoryInfo, error) { + var response protocol.RepositoryInfo + err := e.call(ctx, "/repos/create", request, &response) + return response, err +} +func (e *Endpoints) RepositoryInfo(ctx context.Context, request protocol.RepositoryAdminRequest) (protocol.AdminRepositoryInfoResponse, error) { + var response protocol.AdminRepositoryInfoResponse + err := e.call(ctx, "/repo/info", request, &response) + return response, err +} +func (e *Endpoints) UpdateRepository(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/update", request, nil) +} +func (e *Endpoints) RenameRepository(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/rename", request, nil) +} +func (e *Endpoints) DeleteRepository(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/delete", request, nil) +} +func (e *Endpoints) ReindexRepositoryMembers(ctx context.Context, request protocol.KeyRequest) error { + return e.call(ctx, "/members/reindex", request, nil) +} +func (e *Endpoints) ListRepositoryTeams(ctx context.Context, request protocol.RepositoryAdminRequest) ([]protocol.RepositoryTeamGrant, error) { + var response protocol.RepositoryTeamsResponse + err := e.call(ctx, "/repo/teams/list", request, &response) + return response.Teams, err +} +func (e *Endpoints) UpsertRepositoryTeam(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/teams/upsert", request, nil) +} +func (e *Endpoints) RemoveRepositoryTeam(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/teams/remove", request, nil) +} +func (e *Endpoints) ListRepositoryUsers(ctx context.Context, request protocol.RepositoryAdminRequest) ([]protocol.RepositoryUserGrant, error) { + var response protocol.RepositoryUsersResponse + err := e.call(ctx, "/repo/users/list", request, &response) + return response.Users, err +} +func (e *Endpoints) UpsertRepositoryUser(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/users/upsert", request, nil) +} +func (e *Endpoints) RemoveRepositoryUser(ctx context.Context, request protocol.RepositoryAdminRequest) error { + return e.call(ctx, "/repo/users/remove", request, nil) +} + +func (e *Endpoints) ConfirmOwnerTransfer(ctx context.Context, request protocol.OwnerTransferRequest) (protocol.OwnerTransferResponse, error) { + return e.ownerTransfer(ctx, "/owners/transfer/confirm", request) +} +func (e *Endpoints) AcceptOwnerTransfer(ctx context.Context, request protocol.OwnerTransferRequest) (protocol.OwnerTransferResponse, error) { + return e.ownerTransfer(ctx, "/owners/transfer/accept", request) +} +func (e *Endpoints) CancelOwnerTransfer(ctx context.Context, request protocol.OwnerTransferRequest) error { + return e.call(ctx, "/owners/transfer/cancel", request, nil) +} +func (e *Endpoints) ownerTransfer(ctx context.Context, path string, request protocol.OwnerTransferRequest) (protocol.OwnerTransferResponse, error) { + var response protocol.OwnerTransferResponse + err := e.call(ctx, path, request, &response) + return response, err +} + +func (e *Endpoints) ListProtections(ctx context.Context, request protocol.Protection) ([]protocol.Protection, error) { + var response struct { + Protections []protocol.Protection `json:"protections"` + } + err := e.call(ctx, "/protection/list", request, &response) + return response.Protections, err +} +func (e *Endpoints) UpsertProtection(ctx context.Context, request protocol.Protection) error { + return e.call(ctx, "/protection/upsert", request, nil) +} +func (e *Endpoints) RemoveProtection(ctx context.Context, request protocol.Protection) error { + return e.call(ctx, "/protection/remove", request, nil) +} +func (e *Endpoints) GetUserProfile(ctx context.Context, request protocol.UserProfileRequest) (protocol.UserProfileResponse, error) { + var response protocol.UserProfileResponse + err := e.call(ctx, "/profile/get", request, &response) + return response, err +} +func (e *Endpoints) UpdateUserProfile(ctx context.Context, request protocol.UserProfileRequest) (protocol.UserProfileResponse, error) { + var response protocol.UserProfileResponse + err := e.call(ctx, "/profile/update", request, &response) + return response, err +} + +func (c *Client) endpoints() *Endpoints { return NewEndpoints(c) } +func (c *Client) Authorize(ctx context.Context, request protocol.AuthRequest) (protocol.AuthResponse, error) { + return c.endpoints().Authorize(ctx, request) +} +func (c *Client) AuthStatus(ctx context.Context, request protocol.AuthStatusRequest) (protocol.AuthStatus, error) { + return c.endpoints().AuthStatus(ctx, request) +} +func (c *Client) ListRefs(ctx context.Context, repo protocol.Repository) (map[string]string, error) { + return c.endpoints().ListRefs(ctx, repo) +} +func (c *Client) UpdateRef(ctx context.Context, request protocol.RefUpdateRequest) error { + return c.endpoints().UpdateRef(ctx, request) +} +func (c *Client) ObjectCapability(ctx context.Context, request protocol.ObjectCapabilityRequest) (protocol.ObjectCapabilityResponse, error) { + return c.endpoints().ObjectCapability(ctx, request) +} +func (c *Client) ListIssues(ctx context.Context, request protocol.IssueRequest) ([]protocol.Issue, error) { + return c.endpoints().ListIssues(ctx, request) +} +func (c *Client) ListPullRequests(ctx context.Context, repo protocol.Repository) ([]protocol.PullRequest, error) { + return c.endpoints().ListPullRequests(ctx, protocol.PullRequestRequest{Repo: repo}) +} +func (c *Client) ListCIRuns(ctx context.Context, repo protocol.Repository) ([]protocol.CIRun, error) { + return c.endpoints().ListCIRuns(ctx, protocol.CIRequest{Repo: repo}) +} diff --git a/broker/client/endpoints_test.go b/broker/client/endpoints_test.go new file mode 100644 index 0000000..c3eef92 --- /dev/null +++ b/broker/client/endpoints_test.go @@ -0,0 +1,80 @@ +package client + +import ( + "context" + "net/http" + "testing" + + "github.com/bucketgit/bgit/protocol" +) + +type recordingCaller struct { + path string + request any + response func(any) +} + +func (c *recordingCaller) PostJSON(_ context.Context, path string, request, response any, _ http.Header) error { + c.path, c.request = path, request + if c.response != nil { + c.response(response) + } + return nil +} + +func TestAdministrativeEndpointsUseStableProtocolPaths(t *testing.T) { + repo := protocol.Repository{Logical: "demo.git"} + tests := []struct { + name string + path string + call func(*Endpoints) error + }{ + {"list users", "/broker/users/list", func(e *Endpoints) error { + _, err := e.ListUsers(t.Context(), protocol.RepositoryAdminRequest{}) + return err + }}, + {"list teams", "/teams/list", func(e *Endpoints) error { + _, err := e.ListTeams(t.Context(), protocol.RepositoryAdminRequest{}) + return err + }}, + {"list repositories", "/repos/list", func(e *Endpoints) error { + _, err := e.ListRepositories(t.Context(), protocol.RepositoryAdminRequest{}) + return err + }}, + {"repository info", "/repo/info", func(e *Endpoints) error { + _, err := e.RepositoryInfo(t.Context(), protocol.RepositoryAdminRequest{Repo: repo}) + return err + }}, + {"repository users", "/repo/users/list", func(e *Endpoints) error { + _, err := e.ListRepositoryUsers(t.Context(), protocol.RepositoryAdminRequest{Repo: repo}) + return err + }}, + {"repository teams", "/repo/teams/list", func(e *Endpoints) error { + _, err := e.ListRepositoryTeams(t.Context(), protocol.RepositoryAdminRequest{Repo: repo}) + return err + }}, + {"repository invites", "/keys/invite/list", func(e *Endpoints) error { + _, err := e.ListRepositoryInvites(t.Context(), protocol.OwnerTransferRequest{Repo: repo}) + return err + }}, + {"protections", "/protection/list", func(e *Endpoints) error { + _, err := e.ListProtections(t.Context(), protocol.Protection{Repo: repo}) + return err + }}, + {"profile", "/profile/get", func(e *Endpoints) error { + _, err := e.GetUserProfile(t.Context(), protocol.UserProfileRequest{Repo: repo}) + return err + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + caller := &recordingCaller{} + if err := test.call(NewEndpoints(caller)); err != nil { + t.Fatal(err) + } + if caller.path != test.path { + t.Fatalf("path=%q want %q", caller.path, test.path) + } + }) + } +} diff --git a/broker/client/signing.go b/broker/client/signing.go new file mode 100644 index 0000000..dbf8a6a --- /dev/null +++ b/broker/client/signing.go @@ -0,0 +1,127 @@ +package client + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/bucketgit/bgit/protocol" + "golang.org/x/crypto/ssh" +) + +type NonceSource func() (string, error) + +type V2SignatureOptions struct { + Signers []ssh.Signer + Now func() time.Time + Nonce NonceSource + Random io.Reader + Rank func(fingerprint string) int +} + +// V2Signatures creates replay-resistant broker signature headers. Callers own +// signer discovery and can inject deterministic time, nonce, and entropy in +// tests. +type V2Signatures struct { + options V2SignatureOptions +} + +func NewV2Signatures(options V2SignatureOptions) *V2Signatures { + if options.Now == nil { + options.Now = time.Now + } + if options.Nonce == nil { + options.Nonce = RandomNonce + } + if options.Random == nil { + options.Random = rand.Reader + } + return &V2Signatures{options: options} +} + +func (s *V2Signatures) HeaderSets(_ context.Context, baseURL, path string, payload []byte) ([]http.Header, error) { + host, err := signatureHost(baseURL) + if err != nil { + return nil, err + } + type signedHeader struct { + fingerprint string + header http.Header + } + result := make([]signedHeader, 0, len(s.options.Signers)) + for _, signer := range s.options.Signers { + if signer == nil { + continue + } + nonce, err := s.options.Nonce() + if err != nil { + return nil, err + } + timestamp := fmt.Sprintf("%d", s.options.Now().Unix()) + message := protocol.SignatureMessage(protocol.RequestMetadata{Method: http.MethodPost, Path: path, Host: host, Timestamp: timestamp, Nonce: nonce}, payload) + signature, err := signSSH(signer, s.options.Random, message) + if err != nil { + return nil, err + } + fingerprint := ssh.FingerprintSHA256(signer.PublicKey()) + header := http.Header{} + header.Set(protocol.HeaderSignatureVersion, protocol.SignatureVersion) + header.Set(protocol.HeaderKey, strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey())))) + header.Set(protocol.HeaderKeyFingerprint, fingerprint) + header.Set(protocol.HeaderTimestamp, timestamp) + header.Set(protocol.HeaderNonce, nonce) + header.Set(protocol.HeaderSignedHost, host) + header.Set(protocol.HeaderSignature, base64.StdEncoding.EncodeToString(ssh.Marshal(signature))) + header.Set(protocol.HeaderSignatureMessage, base64.StdEncoding.EncodeToString(message)) + result = append(result, signedHeader{fingerprint: fingerprint, header: header}) + } + if s.options.Rank != nil { + sort.SliceStable(result, func(i, j int) bool { + return s.options.Rank(result[i].fingerprint) < s.options.Rank(result[j].fingerprint) + }) + } + headers := make([]http.Header, len(result)) + for i := range result { + headers[i] = result[i].header + } + return headers, nil +} + +func RandomNonce() (string, error) { + var value [16]byte + if _, err := io.ReadFull(rand.Reader, value[:]); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(value[:]), nil +} + +func signatureHost(baseURL string) (string, error) { + if strings.TrimSpace(baseURL) == "" { + return "", nil + } + parsed, err := url.Parse(baseURL) + if err != nil || parsed.Host == "" { + return "", fmt.Errorf("invalid broker URL %q", baseURL) + } + return strings.ToLower(parsed.Host), nil +} + +func signSSH(signer ssh.Signer, random io.Reader, message []byte) (*ssh.Signature, error) { + if signer.PublicKey().Type() == ssh.KeyAlgoRSA { + if algorithmSigner, ok := signer.(ssh.AlgorithmSigner); ok { + if signature, err := algorithmSigner.SignWithAlgorithm(random, message, ssh.KeyAlgoRSASHA256); err == nil { + return signature, nil + } + } + } + return signer.Sign(random, message) +} + +var _ SignatureProvider = (*V2Signatures)(nil) diff --git a/broker/client/signing_test.go b/broker/client/signing_test.go new file mode 100644 index 0000000..52868bf --- /dev/null +++ b/broker/client/signing_test.go @@ -0,0 +1,56 @@ +package client + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "testing" + "time" + + "github.com/bucketgit/bgit/protocol" + "golang.org/x/crypto/ssh" +) + +func TestV2SignaturesUsesInjectedClockNonceAndSigner(t *testing.T) { + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(privateKey) + if err != nil { + t.Fatal(err) + } + provider := NewV2Signatures(V2SignatureOptions{ + Signers: []ssh.Signer{signer}, + Now: func() time.Time { return time.Unix(1770000000, 0) }, + Nonce: func() (string, error) { return "0123456789abcdef", nil }, + }) + headers, err := provider.HeaderSets(context.Background(), "https://Broker.Example.com", "/repos/mine", []byte(`{"repo":"demo"}`)) + if err != nil || len(headers) != 1 { + t.Fatalf("headers=%#v err=%v", headers, err) + } + header := headers[0] + if header.Get(protocol.HeaderTimestamp) != "1770000000" || header.Get(protocol.HeaderNonce) != "0123456789abcdef" || header.Get(protocol.HeaderSignedHost) != "broker.example.com" { + t.Fatalf("headers=%#v", header) + } + message, err := base64.StdEncoding.DecodeString(header.Get(protocol.HeaderSignatureMessage)) + if err != nil { + t.Fatal(err) + } + want := protocol.SignatureMessage(protocol.RequestMetadata{Method: "POST", Path: "/repos/mine", Host: "broker.example.com", Timestamp: "1770000000", Nonce: "0123456789abcdef"}, []byte(`{"repo":"demo"}`)) + if string(message) != string(want) { + t.Fatalf("message=%q want=%q", message, want) + } + signatureBytes, err := base64.StdEncoding.DecodeString(header.Get(protocol.HeaderSignature)) + if err != nil { + t.Fatal(err) + } + var signature ssh.Signature + if err := ssh.Unmarshal(signatureBytes, &signature); err != nil { + t.Fatal(err) + } + if err := signer.PublicKey().Verify(message, &signature); err != nil { + t.Fatal(err) + } +} diff --git a/broker/local/admin.go b/broker/local/admin.go new file mode 100644 index 0000000..58571de --- /dev/null +++ b/broker/local/admin.go @@ -0,0 +1,149 @@ +package local + +import ( + "context" + "errors" + "io/fs" + "sort" + "strings" + + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" + fsstore "github.com/bucketgit/bgit/store/fs" +) + +const ownersPath = "owners.json" + +func (b *Broker) LoadOwners(ctx context.Context) (Owners, error) { + backend, err := fsstore.New(b.root) + if err != nil { + return Owners{}, err + } + data, err := backend.Read(ctx, ownersPath) + if errors.Is(err, fs.ErrNotExist) { + return Owners{}, nil + } + if err != nil { + return Owners{}, err + } + var owners Owners + if err := decodeJSON(data, &owners); err != nil { + return Owners{}, err + } + return owners, nil +} + +func (b *Broker) UpsertOwners(ctx context.Context, request protocol.OwnerRequest) (Owners, error) { + unlock := b.LockRepository(protocol.Repository{Logical: "__owners__"}) + defer unlock() + backend, err := fsstore.New(b.root) + if err != nil { + return Owners{}, err + } + for attempt := 0; attempt < 16; attempt++ { + var owners Owners + expected, err := loadJSONState(ctx, backend, ownersPath, &owners) + if err != nil { + return Owners{}, err + } + for _, publicKey := range request.PublicKeys { + publicKey = normalizePublicKey(publicKey) + if publicKey == "" { + continue + } + found := false + for index := range owners.Keys { + if normalizePublicKey(owners.Keys[index].PublicKey) == publicKey { + owners.Keys[index].User = first(request.User, "owner") + owners.Keys[index].Role = first(request.Role, "owner") + found = true + } + } + if !found { + owners.Keys = append(owners.Keys, protocol.Key{User: first(request.User, "owner"), Role: first(request.Role, "owner"), PublicKey: publicKey}) + } + } + if err := compareAndSwapJSON(ctx, backend, ownersPath, expected, owners); errors.Is(err, store.ErrConflict) { + continue + } else if err != nil { + return Owners{}, err + } + return owners, nil + } + return Owners{}, store.ErrConflict +} + +// CreateRepository persists repository metadata after the caller has +// provisioned any provider bucket required by the selected storage backend. +func (b *Broker) CreateRepository(ctx context.Context, request protocol.RepositoryAdminRequest, ownerKeys []protocol.Key) (RepositoryState, error) { + repo := request.Repo + if err := repo.Validate(); err != nil { + return RepositoryState{}, err + } + unlock := b.LockRepository(repo) + defer unlock() + if state, err := b.LoadRepositoryState(ctx, repo); err == nil && state.Repo.Logical != "" { + return state, protocol.ErrConflict + } else if err != nil && !errors.Is(err, fs.ErrNotExist) { + return RepositoryState{}, err + } + if existing, found, err := b.FindRepository(ctx, repo.Logical); err != nil { + return RepositoryState{}, err + } else if found { + return RepositoryState{Repo: existing}, protocol.ErrConflict + } + now := b.now().UTC().Format("2006-01-02T15:04:05Z07:00") + state := RepositoryState{Repo: repo, Refs: map[string]string{}, Teams: []protocol.RepositoryTeamGrant{{ID: "t_core", Role: first(request.Role, "developer")}}, CreatedAt: now, UpdatedAt: now} + for _, key := range ownerKeys { + key.PublicKey = normalizePublicKey(key.PublicKey) + if key.PublicKey == "" { + continue + } + key.User = first(request.User, key.User, "owner") + key.Role = first(request.Role, "developer") + state.Keys = append(state.Keys, key) + } + if err := b.CompareAndSwapJSON(ctx, state.Repo, repositoryStatePath, store.ObjectState{}, state); err != nil { + return RepositoryState{}, err + } + if err := b.UpsertRepository(ctx, state.Repo); err != nil { + return RepositoryState{}, err + } + return state, nil +} + +func (b *Broker) GetRepository(ctx context.Context, repo protocol.Repository) (RepositoryState, error) { + if strings.TrimSpace(repo.Logical) != "" { + if indexed, ok, err := b.FindRepository(ctx, repo.Logical); err != nil { + return RepositoryState{}, err + } else if ok { + repo = indexed + } + } + return b.LoadRepositoryState(ctx, repo) +} + +func (b *Broker) ListRepositories(ctx context.Context) ([]protocol.RepositoryInfo, error) { + index, err := b.LoadRepositoryIndex(ctx) + if err != nil { + return nil, err + } + result := make([]protocol.RepositoryInfo, 0, len(index.Repos)) + for _, repo := range index.Repos { + state, err := b.LoadRepositoryState(ctx, repo) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + result = append(result, protocol.RepositoryInfo{Repo: repo, Logical: repo.Logical, Teams: state.Teams}) + } + sort.Slice(result, func(i, j int) bool { return result[i].Repo.Logical < result[j].Repo.Logical }) + return result, nil +} + +func normalizePublicKey(value string) string { + parts := strings.Fields(strings.TrimSpace(value)) + if len(parts) >= 2 { + return parts[0] + " " + parts[1] + } + return strings.TrimSpace(value) +} diff --git a/broker/local/auth.go b/broker/local/auth.go new file mode 100644 index 0000000..e9bcbe9 --- /dev/null +++ b/broker/local/auth.go @@ -0,0 +1,61 @@ +package local + +import ( + "context" + "errors" + "strings" + + "github.com/bucketgit/bgit/protocol" +) + +type IdentityVerifier interface { + Verify(context.Context, protocol.AuthRequest) (protocol.Identity, error) +} +type IdentityVerifierFunc func(context.Context, protocol.AuthRequest) (protocol.Identity, error) + +func (f IdentityVerifierFunc) Verify(ctx context.Context, request protocol.AuthRequest) (protocol.Identity, error) { + return f(ctx, request) +} + +func (b *Broker) Authorize(ctx context.Context, request protocol.AuthRequest) (protocol.AuthResponse, error) { + if b.verifier == nil { + return protocol.AuthResponse{}, errors.New("local broker identity verifier is required") + } + identity, err := b.verifier.Verify(ctx, request) + if err != nil { + return protocol.AuthResponse{}, err + } + state, err := b.LoadRepositoryState(ctx, request.Repo) + if err != nil { + return protocol.AuthResponse{}, err + } + for _, key := range state.Keys { + if key.Suspended { + continue + } + if sameIdentity(identity, key) && roleAllows(key.Role, request.Operation) { + return protocol.AuthResponse{Allowed: true, User: key.User, Role: key.Role}, nil + } + } + return protocol.AuthResponse{Allowed: false}, nil +} + +func sameIdentity(identity protocol.Identity, key protocol.Key) bool { + if strings.TrimSpace(identity.PublicKey) != "" && strings.TrimSpace(identity.PublicKey) == strings.TrimSpace(key.PublicKey) { + return true + } + return strings.TrimSpace(identity.User) != "" && strings.EqualFold(strings.TrimSpace(identity.User), strings.TrimSpace(key.User)) +} +func roleAllows(role, operation string) bool { + rank := map[string]int{"read": 1, "triage": 2, "developer": 3, "maintainer": 4, "admin": 5, "owner": 6}[strings.ToLower(strings.TrimSpace(role))] + switch strings.ToLower(strings.TrimSpace(operation)) { + case "read": + return rank >= 1 + case "write", "delete": + return rank >= 3 + case "merge": + return rank >= 4 + default: + return false + } +} diff --git a/broker/local/broker.go b/broker/local/broker.go new file mode 100644 index 0000000..303ff45 --- /dev/null +++ b/broker/local/broker.go @@ -0,0 +1,272 @@ +// Package local implements the in-process BucketGit broker's durable object +// and metadata substrate. Endpoint parsing and CLI profile lookup intentionally +// live outside this package. +package local + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/fs" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" + fsstore "github.com/bucketgit/bgit/store/fs" +) + +type Resolver interface { + Resolve(ctx context.Context, repo protocol.Repository) (backend store.Writer, close func() error, ok bool, err error) +} + +type ResolverFunc func(context.Context, protocol.Repository) (store.Writer, func() error, bool, error) + +func (f ResolverFunc) Resolve(ctx context.Context, repo protocol.Repository) (store.Writer, func() error, bool, error) { + return f(ctx, repo) +} + +type Options struct { + Root string + Resolver Resolver + Now func() time.Time + Verifier IdentityVerifier +} + +type Broker struct { + root string + objectRoot string + resolver Resolver + locksMu sync.Mutex + locks map[string]*sync.Mutex + now func() time.Time + verifier IdentityVerifier +} + +type RepositoryStore struct { + broker *Broker + repo protocol.Repository +} + +func New(options Options) (*Broker, error) { + root, err := filepath.Abs(strings.TrimSpace(options.Root)) + if err != nil || strings.TrimSpace(options.Root) == "" { + return nil, fmt.Errorf("local broker root is required") + } + objectRoot := filepath.Join(root, "objects") + backend, err := fsstore.New(objectRoot) + if err != nil { + return nil, err + } + _ = backend + now := options.Now + if now == nil { + now = time.Now + } + return &Broker{root: root, objectRoot: objectRoot, resolver: options.Resolver, locks: map[string]*sync.Mutex{}, now: now, verifier: options.Verifier}, nil +} + +func (b *Broker) Root() string { return b.root } +func (b *Broker) ObjectRoot() string { return b.objectRoot } +func (b *Broker) Repository(repo protocol.Repository) *RepositoryStore { + return &RepositoryStore{broker: b, repo: repo} +} + +func (r *RepositoryStore) Read(ctx context.Context, path string) ([]byte, error) { + return r.broker.ReadObject(ctx, r.repo, path) +} +func (r *RepositoryStore) List(ctx context.Context, prefix string) ([]string, error) { + return r.broker.ListObjects(ctx, r.repo, prefix) +} +func (r *RepositoryStore) Write(ctx context.Context, path string, data []byte) error { + return r.broker.WriteObject(ctx, r.repo, path, data) +} +func (r *RepositoryStore) Delete(ctx context.Context, path string) error { + return r.broker.DeleteObject(ctx, r.repo, path) +} +func (r *RepositoryStore) ListRefs(ctx context.Context) (map[string]string, error) { + return store.ReadRefs(ctx, r) +} + +func (r *RepositoryStore) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + backend, closeBackend, err := r.broker.backend(ctx, r.repo) + if err != nil { + return err + } + defer closeIgnoringError(closeBackend) + refs, ok := backend.(store.RefStore) + if !ok { + return store.ErrNotSupported + } + return refs.CompareAndSwapRef(ctx, ref, oldOID, newOID) +} + +func (r *RepositoryStore) CompareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error { + backend, closeBackend, err := r.broker.backend(ctx, r.repo) + if err != nil { + return err + } + defer closeIgnoringError(closeBackend) + cas, ok := backend.(store.CompareAndSwapper) + if !ok { + return store.ErrNotSupported + } + return cas.CompareAndSwap(ctx, path, expected, replacement) +} + +var _ store.Writer = (*RepositoryStore)(nil) +var _ store.RefStore = (*RepositoryStore)(nil) +var _ store.CompareAndSwapper = (*RepositoryStore)(nil) + +func (b *Broker) BucketDir(bucket string) string { + if strings.HasPrefix(bucket, "file://") { + value := strings.TrimPrefix(bucket, "file://") + if filepath.IsAbs(value) { + return filepath.Clean(value) + } + value = filepath.Clean(strings.TrimPrefix(value, "/")) + if value == "." { + value = "repo.git" + } + return filepath.Join(b.objectRoot, value) + } + return filepath.Join(b.objectRoot, base64.RawURLEncoding.EncodeToString([]byte(bucket))) +} + +func (b *Broker) ObjectPath(repo protocol.Repository, objectPath string) (string, error) { + root := b.BucketDir(repo.Bucket) + if prefix := strings.Trim(repo.Prefix, "/"); prefix != "" { + objectPath = prefix + "/" + strings.TrimPrefix(objectPath, "/") + } + validated, err := store.ValidatePath(strings.TrimPrefix(objectPath, "/"), false) + if err != nil { + return "", err + } + return filepath.Join(root, filepath.FromSlash(validated)), nil +} + +func (b *Broker) ReadObject(ctx context.Context, repo protocol.Repository, objectPath string) ([]byte, error) { + backend, closeBackend, err := b.backend(ctx, repo) + if err != nil { + return nil, err + } + defer closeIgnoringError(closeBackend) + return backend.Read(ctx, objectPath) +} + +func (b *Broker) WriteObject(ctx context.Context, repo protocol.Repository, objectPath string, data []byte) error { + backend, closeBackend, err := b.backend(ctx, repo) + if err != nil { + return err + } + defer closeIgnoringError(closeBackend) + return backend.Write(ctx, objectPath, data) +} + +func (b *Broker) DeleteObject(ctx context.Context, repo protocol.Repository, objectPath string) error { + backend, closeBackend, err := b.backend(ctx, repo) + if err != nil { + return err + } + defer closeIgnoringError(closeBackend) + err = backend.Delete(ctx, objectPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +func (b *Broker) ListObjects(ctx context.Context, repo protocol.Repository, prefix string) ([]string, error) { + backend, closeBackend, err := b.backend(ctx, repo) + if err != nil { + return nil, err + } + defer closeIgnoringError(closeBackend) + paths, err := backend.List(ctx, prefix) + sort.Strings(paths) + return paths, err +} + +func (b *Broker) LoadJSON(ctx context.Context, repo protocol.Repository, path string, target any) error { + data, err := b.ReadObject(ctx, repo, path) + if err != nil { + return err + } + return json.Unmarshal(data, target) +} + +func (b *Broker) SaveJSON(ctx context.Context, repo protocol.Repository, path string, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + return b.WriteObject(ctx, repo, path, data) +} + +func (b *Broker) LoadJSONState(ctx context.Context, repo protocol.Repository, path string, target any) (store.ObjectState, error) { + data, err := b.ReadObject(ctx, repo, path) + if errors.Is(err, fs.ErrNotExist) { + return store.ObjectState{}, nil + } + if err != nil { + return store.ObjectState{}, err + } + if err := json.Unmarshal(data, target); err != nil { + return store.ObjectState{}, err + } + return store.ObjectState{Exists: true, Data: data}, nil +} + +func (b *Broker) CompareAndSwapJSON(ctx context.Context, repo protocol.Repository, path string, expected store.ObjectState, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + backend := b.Repository(repo) + return backend.CompareAndSwap(ctx, path, expected, data) +} + +func (b *Broker) LockRepository(repo protocol.Repository) func() { + key := strings.Join([]string{repo.Provider, repo.Bucket, repo.Prefix, repo.Logical}, "\x00") + b.locksMu.Lock() + lock := b.locks[key] + if lock == nil { + lock = &sync.Mutex{} + b.locks[key] = lock + } + b.locksMu.Unlock() + lock.Lock() + return lock.Unlock +} + +func (b *Broker) backend(ctx context.Context, repo protocol.Repository) (store.Writer, func() error, error) { + if b.resolver != nil { + backend, closeBackend, ok, err := b.resolver.Resolve(ctx, repo) + if err != nil { + return nil, nil, err + } + if ok { + if backend == nil { + return nil, nil, errors.New("local broker resolver returned a nil backend") + } + return backend, closeBackend, nil + } + } + root := b.BucketDir(repo.Bucket) + if prefix := strings.Trim(repo.Prefix, "/"); prefix != "" { + root = filepath.Join(root, filepath.FromSlash(prefix)) + } + backend, err := fsstore.New(root) + return backend, nil, err +} + +func closeIgnoringError(closeBackend func() error) { + if closeBackend != nil { + _ = closeBackend() + } +} diff --git a/broker/local/broker_test.go b/broker/local/broker_test.go new file mode 100644 index 0000000..ef21f50 --- /dev/null +++ b/broker/local/broker_test.go @@ -0,0 +1,325 @@ +package local + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" + fsstore "github.com/bucketgit/bgit/store/fs" + "github.com/bucketgit/bgit/store/storetest" +) + +func TestRepositoryStoreContract(t *testing.T) { + factory := func(t *testing.T) store.Writer { + broker, err := New(Options{Root: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + return broker.Repository(protocol.Repository{Provider: "file", Bucket: "contract.git", Logical: "contract.git"}) + } + storetest.Run(t, factory) + storetest.RunCompareAndSwap(t, factory) +} + +func TestCloudResolvedRepositoryStoreContracts(t *testing.T) { + for _, provider := range []string{"s3", "gcs"} { + t.Run(provider, func(t *testing.T) { + factory := func(t *testing.T) store.Writer { + resolved, err := fsstore.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + broker, err := New(Options{Root: t.TempDir(), Resolver: ResolverFunc(func(_ context.Context, repo protocol.Repository) (store.Writer, func() error, bool, error) { + if repo.Provider != provider { + t.Fatalf("provider=%q want %q", repo.Provider, provider) + } + return resolved, nil, true, nil + })}) + if err != nil { + t.Fatal(err) + } + return broker.Repository(protocol.Repository{Provider: provider, Bucket: "contract.git", Logical: "contract.git"}) + } + storetest.Run(t, factory) + storetest.RunCompareAndSwap(t, factory) + }) + } +} + +func TestConcurrentBrokerInstancesDoNotLoseIssueWrites(t *testing.T) { + root := t.TempDir() + repo := protocol.Repository{Provider: "file", Bucket: "shared.git", Logical: "shared.git"} + const writers = 12 + var wait sync.WaitGroup + results := make(chan error, writers) + for index := 0; index < writers; index++ { + wait.Add(1) + go func(index int) { + defer wait.Done() + broker, err := New(Options{Root: root}) + if err == nil { + _, err = broker.CreateIssue(t.Context(), repo, protocol.IssueRequest{Title: fmt.Sprintf("issue-%d", index)}, "owner") + } + results <- err + }(index) + } + wait.Wait() + close(results) + succeeded := 0 + for err := range results { + if err == nil { + succeeded++ + continue + } + if !errors.Is(err, store.ErrConflict) { + t.Fatalf("unexpected concurrent mutation error: %v", err) + } + } + broker, err := New(Options{Root: root}) + if err != nil { + t.Fatal(err) + } + issues, err := broker.ListIssues(t.Context(), repo, IssueFilter{}) + if err != nil { + t.Fatal(err) + } + if len(issues) != succeeded || succeeded == 0 { + t.Fatalf("persisted=%d succeeded=%d", len(issues), succeeded) + } +} + +func TestFilesystemBrokerObjectRoundTripAndIsolation(t *testing.T) { + broker, err := New(Options{Root: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + one := protocol.Repository{Provider: "file", Bucket: "one.git", Logical: "one.git"} + two := protocol.Repository{Provider: "file", Bucket: "two.git", Logical: "two.git"} + if err := broker.WriteObject(context.Background(), one, "objects/aa/object", []byte("one")); err != nil { + t.Fatal(err) + } + data, err := broker.ReadObject(context.Background(), one, "objects/aa/object") + if err != nil || string(data) != "one" { + t.Fatalf("read = %q, %v", data, err) + } + if _, err := broker.ReadObject(context.Background(), two, "objects/aa/object"); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("isolated read error = %v", err) + } +} + +func TestHTTPHandlerUsesDirectBrokerServices(t *testing.T) { + repo := protocol.Repository{Provider: "file", Bucket: "http.git", Logical: "http.git"} + broker, err := New(Options{Root: t.TempDir(), Verifier: IdentityVerifierFunc(func(context.Context, protocol.AuthRequest) (protocol.Identity, error) { + return protocol.Identity{User: "dev"}, nil + })}) + if err != nil { + t.Fatal(err) + } + if err := broker.SaveRepositoryState(t.Context(), RepositoryState{Repo: repo, Keys: []protocol.Key{{User: "dev", Role: "developer"}}}); err != nil { + t.Fatal(err) + } + body := bytes.NewBufferString(`{"repo":{"provider":"file","bucket":"http.git","prefix":"","origin":"","logical":"http.git"},"type":"story","title":"From HTTP"}`) + request := httptest.NewRequest(http.MethodPost, "/issues/create", body) + response := httptest.NewRecorder() + broker.Handler().ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + issues, err := broker.ListIssues(t.Context(), repo, IssueFilter{Type: "story"}) + if err != nil || len(issues) != 1 || issues[0].Title != "From HTTP" { + t.Fatalf("issues=%#v err=%v", issues, err) + } +} + +func TestLoadsExistingBrokerStateLayout(t *testing.T) { + root := t.TempDir() + broker, err := New(Options{Root: root}) + if err != nil { + t.Fatal(err) + } + repo := protocol.Repository{Provider: "file", Bucket: "legacy.git", Logical: "legacy.git"} + state := RepositoryState{Repo: repo, Refs: map[string]string{"refs/heads/main": "0123456789abcdef0123456789abcdef01234567"}} + data, _ := json.Marshal(state) + path := filepath.Join(broker.BucketDir(repo.Bucket), filepath.FromSlash(repositoryStatePath)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + loaded, err := broker.LoadRepositoryState(t.Context(), repo) + if err != nil || loaded.Repo.Logical != "legacy.git" || loaded.Refs["refs/heads/main"] == "" { + t.Fatalf("loaded=%#v err=%v", loaded, err) + } +} + +func TestBoardServiceOrdersAndArchivesStories(t *testing.T) { + now := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC) + broker, err := New(Options{Root: t.TempDir(), Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + repo := protocol.Repository{Provider: "file", Bucket: "board.git", Logical: "board.git"} + one, err := broker.CreateIssue(t.Context(), repo, protocol.IssueRequest{Type: "story", Title: "One", Lane: "backlog"}, "owner") + if err != nil { + t.Fatal(err) + } + two, err := broker.CreateIssue(t.Context(), repo, protocol.IssueRequest{Type: "story", Title: "Two", Lane: "backlog"}, "owner") + if err != nil { + t.Fatal(err) + } + if err := broker.MoveStory(t.Context(), repo, two.ID, StoryMove{Lane: "backlog", Order: 1}, "owner"); err != nil { + t.Fatal(err) + } + issues, err := broker.ListIssues(t.Context(), repo, IssueFilter{Type: "story"}) + if err != nil || len(issues) != 2 || issues[0].ID != two.ID || issues[1].ID != one.ID || issues[0].Position != 1 || issues[1].Position != 2 { + t.Fatalf("ordered issues = %#v, %v", issues, err) + } + if err := broker.ArchiveIssue(t.Context(), repo, two.ID, true, "owner"); err != nil { + t.Fatal(err) + } + issues, err = broker.ListIssues(t.Context(), repo, IssueFilter{Type: "story"}) + if err != nil || len(issues) != 1 || issues[0].ID != one.ID { + t.Fatalf("active issues = %#v, %v", issues, err) + } + issues, err = broker.ListIssues(t.Context(), repo, IssueFilter{Type: "story", IncludeArchived: true}) + if err != nil || len(issues) != 2 { + t.Fatalf("all issues = %#v, %v", issues, err) + } +} + +func TestAuthorizeUsesInjectedIdentityAndRepositoryRole(t *testing.T) { + repo := protocol.Repository{Provider: "file", Bucket: "auth.git", Logical: "auth.git"} + broker, err := New(Options{Root: t.TempDir(), Verifier: IdentityVerifierFunc(func(context.Context, protocol.AuthRequest) (protocol.Identity, error) { + return protocol.Identity{User: "dev"}, nil + })}) + if err != nil { + t.Fatal(err) + } + if err := broker.SaveRepositoryState(t.Context(), RepositoryState{Repo: repo, Keys: []protocol.Key{{User: "dev", Role: "developer"}}}); err != nil { + t.Fatal(err) + } + response, err := broker.Authorize(t.Context(), protocol.AuthRequest{Repo: repo, Operation: "write"}) + if err != nil || !response.Allowed || response.Role != "developer" { + t.Fatalf("response=%#v err=%v", response, err) + } + response, err = broker.Authorize(t.Context(), protocol.AuthRequest{Repo: repo, Operation: "merge"}) + if err != nil || response.Allowed { + t.Fatalf("merge response=%#v err=%v", response, err) + } +} + +func TestFilesystemBrokerRejectsTraversal(t *testing.T) { + broker, err := New(Options{Root: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + repo := protocol.Repository{Provider: "file", Bucket: "repo.git", Logical: "repo.git"} + if err := broker.WriteObject(context.Background(), repo, "../escape", []byte("bad")); err == nil { + t.Fatal("traversal write succeeded") + } +} + +func TestUpdateRefPersistsGitRefAndRejectsStaleExpectedOID(t *testing.T) { + broker, err := New(Options{Root: t.TempDir(), Now: func() time.Time { + return time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC) + }}) + if err != nil { + t.Fatal(err) + } + repo := protocol.Repository{Provider: "file", Bucket: "refs.git", Logical: "refs.git"} + if err := broker.SaveRepositoryState(t.Context(), RepositoryState{Repo: repo, Refs: map[string]string{}}); err != nil { + t.Fatal(err) + } + const oid = "0123456789abcdef0123456789abcdef01234567" + request := protocol.RefUpdateRequest{Repo: repo, Ref: "refs/heads/main", New: oid} + if err := broker.UpdateRef(t.Context(), request, "owner"); err != nil { + t.Fatal(err) + } + refs, err := broker.Repository(repo).ListRefs(t.Context()) + if err != nil || refs[request.Ref] != oid { + t.Fatalf("refs=%#v err=%v", refs, err) + } + state, err := broker.LoadRepositoryState(t.Context(), repo) + if err != nil || state.Refs[request.Ref] != oid { + t.Fatalf("state=%#v err=%v", state, err) + } + recordPath := ".bucketgit/broker-state/v1/refs/cmVmcy9oZWFkcy9tYWlu.json" + var record map[string]any + if err := broker.LoadJSON(t.Context(), repo, recordPath, &record); err != nil || record["hash"] != oid { + t.Fatalf("record=%#v err=%v", record, err) + } + stale := request + stale.Old = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + stale.New = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + if err := broker.UpdateRef(t.Context(), stale, "owner"); !errors.Is(err, store.ErrConflict) { + t.Fatalf("stale update error=%v", err) + } + refs, err = broker.Repository(repo).ListRefs(t.Context()) + if err != nil || refs[request.Ref] != oid { + t.Fatalf("refs after stale update=%#v err=%v", refs, err) + } +} + +func TestReconcileRepositoryRepairsCachedRefs(t *testing.T) { + broker, err := New(Options{Root: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + repo := protocol.Repository{Provider: "file", Bucket: "reconcile.git", Logical: "reconcile.git"} + const oid = "0123456789abcdef0123456789abcdef01234567" + state := RepositoryState{Repo: repo, Refs: map[string]string{"refs/heads/stale": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}} + if err := broker.SaveRepositoryState(t.Context(), state); err != nil { + t.Fatal(err) + } + if err := broker.Repository(repo).CompareAndSwapRef(t.Context(), "refs/heads/main", "", oid); err != nil { + t.Fatal(err) + } + reconciled, err := broker.ReconcileRepository(t.Context(), repo, "repair") + if err != nil { + t.Fatal(err) + } + if len(reconciled.Refs) != 1 || reconciled.Refs["refs/heads/main"] != oid { + t.Fatalf("refs=%#v", reconciled.Refs) + } + loaded, err := broker.LoadRepositoryState(t.Context(), repo) + if err != nil || loaded.Refs["refs/heads/main"] != oid { + t.Fatalf("loaded=%#v err=%v", loaded, err) + } +} + +func TestOwnerAndRepositoryServicesPersistState(t *testing.T) { + broker, err := New(Options{Root: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + owners, err := broker.UpsertOwners(t.Context(), protocol.OwnerRequest{User: "owner", Role: "owner", PublicKeys: []string{"ssh-ed25519 AAAATEST comment"}}) + if err != nil || len(owners.Keys) != 1 || owners.Keys[0].PublicKey != "ssh-ed25519 AAAATEST" { + t.Fatalf("owners=%#v err=%v", owners, err) + } + repo := protocol.Repository{Provider: "file", Bucket: "admin.git", Logical: "admin.git"} + request := protocol.RepositoryAdminRequest{Repo: repo, User: "owner", Role: "developer"} + state, err := broker.CreateRepository(t.Context(), request, owners.Keys) + if err != nil || state.Repo.Logical != repo.Logical || len(state.Keys) != 1 { + t.Fatalf("state=%#v err=%v", state, err) + } + if _, err := broker.CreateRepository(t.Context(), request, owners.Keys); !errors.Is(err, protocol.ErrConflict) { + t.Fatalf("duplicate error=%v", err) + } + repositories, err := broker.ListRepositories(t.Context()) + if err != nil || len(repositories) != 1 || repositories[0].Repo.Logical != repo.Logical { + t.Fatalf("repositories=%#v err=%v", repositories, err) + } +} diff --git a/broker/local/http.go b/broker/local/http.go new file mode 100644 index 0000000..d1e7b6a --- /dev/null +++ b/broker/local/http.go @@ -0,0 +1,173 @@ +package local + +import ( + "encoding/base64" + "encoding/json" + "errors" + "io/fs" + "net/http" + "strings" + + "github.com/bucketgit/bgit/protocol" +) + +func (b *Broker) Handler() http.Handler { return http.HandlerFunc(b.serveHTTP) } + +func (b *Broker) serveHTTP(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + writeError(response, http.StatusMethodNotAllowed, "method not allowed") + return + } + decoder := json.NewDecoder(http.MaxBytesReader(response, request.Body, 8<<20)) + decoder.DisallowUnknownFields() + switch request.URL.Path { + case "/auth/check": + var value protocol.AuthRequest + if err := decoder.Decode(&value); err != nil { + writeError(response, http.StatusBadRequest, "invalid JSON") + return + } + result, err := b.Authorize(request.Context(), value) + writeResult(response, result, err) + return + case "/refs/list": + var value protocol.RefsRequest + if !decodeRequest(response, decoder, &value) { + return + } + if !b.requireHTTPAccess(response, request, value.Repo, "read") { + return + } + state, err := b.ReconcileRepository(request.Context(), value.Repo, "http-sync") + writeResult(response, protocol.RefsResponse{Refs: state.Refs}, err) + return + case "/refs/update": + var value protocol.RefUpdateRequest + if !decodeRequest(response, decoder, &value) { + return + } + identity, ok := b.httpIdentity(response, request, value.Repo, "write") + if !ok { + return + } + err := b.UpdateRef(request.Context(), value, identity.User) + writeResult(response, map[string]bool{"ok": err == nil}, err) + return + case "/objects/read", "/objects/list": + var value protocol.ObjectRequest + if !decodeRequest(response, decoder, &value) { + return + } + if !b.requireHTTPAccess(response, request, value.Repo, "read") { + return + } + if request.URL.Path == "/objects/read" { + data, err := b.ReadObject(request.Context(), value.Repo, value.Path) + writeResult(response, protocol.ObjectResponse{Data: base64.StdEncoding.EncodeToString(data)}, err) + } else { + paths, err := b.ListObjects(request.Context(), value.Repo, value.Prefix) + writeResult(response, protocol.ObjectResponse{Paths: paths}, err) + } + return + default: + if strings.HasPrefix(request.URL.Path, "/issues/") { + b.serveIssuesHTTP(response, request, decoder) + return + } + writeError(response, http.StatusNotFound, "unknown broker endpoint") + } +} + +func (b *Broker) serveIssuesHTTP(response http.ResponseWriter, request *http.Request, decoder *json.Decoder) { + var value protocol.IssueRequest + if !decodeRequest(response, decoder, &value) { + return + } + operation := "write" + if request.URL.Path == "/issues/list" || request.URL.Path == "/issues/view" { + operation = "read" + } + identity, ok := b.httpIdentity(response, request, value.Repo, operation) + if !ok { + return + } + var result any = map[string]bool{"ok": true} + var err error + switch request.URL.Path { + case "/issues/list": + var issues []protocol.Issue + issues, err = b.ListIssues(request.Context(), value.Repo, IssueFilter{Type: value.Type, IncludeArchived: value.IncludeArchived}) + result = map[string]any{"issues": issues} + case "/issues/view": + var issue protocol.Issue + issue, err = b.GetIssue(request.Context(), value.Repo, value.ID) + result = map[string]any{"issue": issue} + case "/issues/create": + var issue protocol.Issue + issue, err = b.CreateIssue(request.Context(), value.Repo, value, identity.User) + result = map[string]any{"issue": issue} + case "/issues/update": + err = b.UpdateIssue(request.Context(), value.Repo, value, identity.User) + case "/issues/comment": + err = b.CommentIssue(request.Context(), value.Repo, value.ID, value.Comment, identity.User) + case "/issues/close": + err = b.SetIssueClosed(request.Context(), value.Repo, value.ID, true, identity.User) + case "/issues/reopen": + err = b.SetIssueClosed(request.Context(), value.Repo, value.ID, false, identity.User) + case "/issues/reorder": + err = b.MoveStory(request.Context(), value.Repo, value.ID, StoryMove{Lane: value.Lane, AfterID: value.AfterID, Order: value.Order}, identity.User) + case "/issues/move": + if value.AfterID != nil { + err = b.MoveStory(request.Context(), value.Repo, value.ID, StoryMove{Lane: value.Lane, AfterID: value.AfterID}, identity.User) + } else { + err = b.MoveStoryToLaneEnd(request.Context(), value.Repo, value.ID, value.Lane, identity.User) + } + case "/issues/take": + err = b.TakeStory(request.Context(), value.Repo, value.ID, identity.User) + case "/issues/assign": + err = b.AssignIssue(request.Context(), value.Repo, value.ID, value.Assignee, identity.User) + case "/issues/archive": + err = b.ArchiveIssue(request.Context(), value.Repo, value.ID, value.Archived, identity.User) + default: + writeError(response, http.StatusNotFound, "unknown broker endpoint") + return + } + writeResult(response, result, err) +} + +func (b *Broker) requireHTTPAccess(response http.ResponseWriter, request *http.Request, repo protocol.Repository, operation string) bool { + _, ok := b.httpIdentity(response, request, repo, operation) + return ok +} +func (b *Broker) httpIdentity(response http.ResponseWriter, request *http.Request, repo protocol.Repository, operation string) (protocol.Identity, bool) { + authorized, err := b.Authorize(request.Context(), protocol.AuthRequest{Repo: repo, Operation: operation}) + if err != nil || !authorized.Allowed { + writeError(response, http.StatusForbidden, operation+" access denied") + return protocol.Identity{}, false + } + return protocol.Identity{User: authorized.User}, true +} +func decodeRequest(response http.ResponseWriter, decoder *json.Decoder, target any) bool { + if err := decoder.Decode(target); err != nil { + writeError(response, http.StatusBadRequest, "invalid JSON") + return false + } + return true +} +func writeResult(response http.ResponseWriter, value any, err error) { + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, fs.ErrNotExist) { + status = http.StatusNotFound + } + writeError(response, status, err.Error()) + return + } + response.Header().Set("content-type", "application/json") + _ = json.NewEncoder(response).Encode(value) +} +func writeError(response http.ResponseWriter, status int, message string) { + response.Header().Set("content-type", "application/json") + response.WriteHeader(status) + _ = json.NewEncoder(response).Encode(map[string]string{"error": message}) +} diff --git a/broker/local/issues.go b/broker/local/issues.go new file mode 100644 index 0000000..c8f14b8 --- /dev/null +++ b/broker/local/issues.go @@ -0,0 +1,450 @@ +package local + +import ( + "context" + "errors" + "sort" + "strings" + + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" +) + +const issuesPath = ".bucketgit/broker-state/v1/issues.json" + +type issueState struct { + NextID int `json:"next_id"` + Issues []protocol.Issue `json:"issues"` +} + +type IssueFilter struct { + Type string + IncludeArchived bool +} +type StoryMove struct { + Lane string + AfterID *int + Order int +} + +func (b *Broker) ListIssues(ctx context.Context, repo protocol.Repository, filter IssueFilter) ([]protocol.Issue, error) { + state, err := b.loadIssues(ctx, repo) + if err != nil { + return nil, err + } + issues := make([]protocol.Issue, 0, len(state.Issues)) + for _, issue := range state.Issues { + if filter.Type != "" && issue.Type != filter.Type { + continue + } + if issue.Archived && !filter.IncludeArchived { + continue + } + issues = append(issues, issue) + } + sortIssues(issues) + return issues, nil +} + +func (b *Broker) GetIssue(ctx context.Context, repo protocol.Repository, id int) (protocol.Issue, error) { + state, err := b.loadIssues(ctx, repo) + if err != nil { + return protocol.Issue{}, err + } + issue, err := findIssue(&state, id) + if err != nil { + return protocol.Issue{}, err + } + return *issue, nil +} + +func (b *Broker) CreateIssue(ctx context.Context, repo protocol.Repository, request protocol.IssueRequest, user string) (protocol.Issue, error) { + unlock := b.LockRepository(repo) + defer unlock() + state, expected, err := b.loadIssuesObserved(ctx, repo) + if err != nil { + return protocol.Issue{}, err + } + title, body := strings.TrimSpace(request.Title), strings.TrimSpace(request.Body) + if title == "" { + title = summary(body) + } + if title == "" { + return protocol.Issue{}, errors.New("issue title is required") + } + if state.NextID <= 0 { + state.NextID = nextIssueID(state.Issues) + } + issueType := first(request.Type, "issue") + lane, position := strings.TrimSpace(request.Lane), 0.0 + if issueType == "story" { + lane = normalizeLane(lane) + position = nextPosition(state.Issues, lane) + } + now := b.now().UTC().Format("2006-01-02T15:04:05Z07:00") + user = first(strings.TrimSpace(user), "owner") + issue := protocol.Issue{ID: state.NextID, Type: issueType, Title: title, Body: body, Status: "open", Lane: lane, Assignee: strings.TrimSpace(request.Assignee), Position: position, Author: user, CreatedAt: now, UpdatedAt: now, History: []protocol.IssueEvent{{User: user, Action: "created", At: now}}} + state.NextID++ + state.Issues = append(state.Issues, issue) + if issueType == "story" { + normalizeOrder(&state, lane, 0) + issue, _ = b.issueFromState(&state, issue.ID) + } + return issue, b.saveIssues(ctx, repo, expected, state) +} + +func (b *Broker) UpdateIssue(ctx context.Context, repo protocol.Repository, request protocol.IssueRequest, user string) error { + return b.mutateIssue(ctx, repo, request.ID, func(state *issueState, issue *protocol.Issue) error { + if request.Type != "" { + issue.Type = request.Type + } + if strings.TrimSpace(request.Title) != "" { + issue.Title = strings.TrimSpace(request.Title) + } + if request.Body != "" { + issue.Body = strings.TrimSpace(request.Body) + } + if issue.Title == "" && issue.Type == "story" { + issue.Title = summary(issue.Body) + } + b.event(issue, user, "edited", "", "", "") + return nil + }) +} + +func (b *Broker) CommentIssue(ctx context.Context, repo protocol.Repository, id int, comment, user string) error { + comment = strings.TrimSpace(comment) + if comment == "" { + return errors.New("comment is required") + } + return b.mutateIssue(ctx, repo, id, func(_ *issueState, issue *protocol.Issue) error { + now := b.now().UTC().Format("2006-01-02T15:04:05Z07:00") + issue.Comments = append(issue.Comments, protocol.IssueReply{User: first(user, "owner"), Body: comment, At: now}) + b.event(issue, user, "commented", "", "", "") + return nil + }) +} + +func (b *Broker) SetIssueClosed(ctx context.Context, repo protocol.Repository, id int, closed bool, user string) error { + return b.mutateIssue(ctx, repo, id, func(_ *issueState, issue *protocol.Issue) error { + status, action := "open", "reopened" + if closed { + status, action = "closed", "closed" + } + if issue.Status != status { + issue.Status = status + b.event(issue, user, action, "", "", "") + } + return nil + }) +} + +func (b *Broker) MoveStory(ctx context.Context, repo protocol.Repository, id int, move StoryMove, user string) error { + return b.mutateIssue(ctx, repo, id, func(state *issueState, issue *protocol.Issue) error { + lane, err := parseLane(first(move.Lane, issue.Lane)) + if err != nil { + return err + } + from := normalizeLane(issue.Lane) + applyOrder(state, id, lane, move.AfterID, move.Order) + if from != lane { + normalizeOrder(state, from, id) + } + issue, _ = findIssue(state, id) + b.event(issue, user, "reordered", from, lane, formatPosition(issue.Position)) + return nil + }) +} + +func (b *Broker) MoveStoryToLaneEnd(ctx context.Context, repo protocol.Repository, id int, lane, user string) error { + return b.mutateIssue(ctx, repo, id, func(state *issueState, issue *protocol.Issue) error { + target, err := parseLane(first(lane, issue.Lane)) + if err != nil { + return err + } + from := normalizeLane(issue.Lane) + issue.Lane = target + issue.Position = nextPosition(state.Issues, target) + if from != target { + normalizeOrder(state, from, id) + } + normalizeOrder(state, target, 0) + issue, _ = findIssue(state, id) + b.event(issue, user, "moved", from, target, formatPosition(issue.Position)) + return nil + }) +} + +func (b *Broker) TakeStory(ctx context.Context, repo protocol.Repository, id int, user string) error { + return b.mutateIssue(ctx, repo, id, func(state *issueState, issue *protocol.Issue) error { + fromAssignee, fromLane := issue.Assignee, normalizeLane(issue.Lane) + issue.Assignee = first(strings.TrimSpace(user), "owner") + if fromLane == "backlog" { + applyOrder(state, id, "doing", nil, 0) + normalizeOrder(state, fromLane, id) + issue, _ = findIssue(state, id) + } else if issue.Position == 0 { + normalizeOrder(state, fromLane, 0) + issue, _ = findIssue(state, id) + } + b.event(issue, user, "assigned", fromAssignee, issue.Assignee, "") + return nil + }) +} + +func (b *Broker) AssignIssue(ctx context.Context, repo protocol.Repository, id int, assignee, user string) error { + return b.mutateIssue(ctx, repo, id, func(_ *issueState, issue *protocol.Issue) error { + from := issue.Assignee + issue.Assignee = strings.TrimSpace(assignee) + b.event(issue, user, "assigned", from, issue.Assignee, "") + return nil + }) +} +func (b *Broker) ArchiveIssue(ctx context.Context, repo protocol.Repository, id int, archived bool, user string) error { + return b.mutateIssue(ctx, repo, id, func(_ *issueState, issue *protocol.Issue) error { + issue.Archived = archived + action := "unarchived" + if archived { + action = "archived" + } + b.event(issue, user, action, "", "", "") + return nil + }) +} + +func (b *Broker) mutateIssue(ctx context.Context, repo protocol.Repository, id int, change func(*issueState, *protocol.Issue) error) error { + unlock := b.LockRepository(repo) + defer unlock() + state, expected, err := b.loadIssuesObserved(ctx, repo) + if err != nil { + return err + } + issue, err := findIssue(&state, id) + if err != nil { + return err + } + if err := change(&state, issue); err != nil { + return err + } + return b.saveIssues(ctx, repo, expected, state) +} +func (b *Broker) loadIssues(ctx context.Context, repo protocol.Repository) (issueState, error) { + state, _, err := b.loadIssuesObserved(ctx, repo) + return state, err +} +func (b *Broker) loadIssuesObserved(ctx context.Context, repo protocol.Repository) (issueState, store.ObjectState, error) { + var state issueState + expected, err := b.LoadJSONState(ctx, repo, issuesPath, &state) + if err != nil { + return state, store.ObjectState{}, err + } + if !expected.Exists { + return issueState{NextID: 1}, expected, nil + } + if state.NextID <= 0 { + state.NextID = nextIssueID(state.Issues) + } + return state, expected, nil +} +func (b *Broker) saveIssues(ctx context.Context, repo protocol.Repository, expected store.ObjectState, state issueState) error { + if state.NextID <= 0 { + state.NextID = nextIssueID(state.Issues) + } + return b.CompareAndSwapJSON(ctx, repo, issuesPath, expected, state) +} +func (b *Broker) issueFromState(state *issueState, id int) (protocol.Issue, error) { + issue, err := findIssue(state, id) + if err != nil { + return protocol.Issue{}, err + } + return *issue, nil +} +func (b *Broker) event(issue *protocol.Issue, user, action, from, to, position string) { + now := b.now().UTC().Format("2006-01-02T15:04:05Z07:00") + issue.UpdatedAt = now + issue.History = append(issue.History, protocol.IssueEvent{User: first(user, "owner"), Action: action, From: from, To: to, Position: position, At: now}) +} + +func findIssue(state *issueState, id int) (*protocol.Issue, error) { + if id <= 0 { + return nil, errors.New("issue id is required") + } + for i := range state.Issues { + if state.Issues[i].ID == id { + return &state.Issues[i], nil + } + } + return nil, errors.New("issue not found") +} +func nextIssueID(issues []protocol.Issue) int { + next := 1 + for _, issue := range issues { + if issue.ID >= next { + next = issue.ID + 1 + } + } + return next +} +func nextPosition(issues []protocol.Issue, lane string) float64 { + count := 0 + for _, issue := range issues { + if issue.Type == "story" && !issue.Archived && normalizeLane(issue.Lane) == lane { + count++ + } + } + return float64(count + 1) +} +func applyOrder(state *issueState, id int, lane string, after *int, order int) { + target, err := findIssue(state, id) + if err != nil { + return + } + target.Lane = lane + stories := laneStories(state, lane, id) + index := len(stories) + if order > 0 { + index = order - 1 + if index < 0 { + index = 0 + } + if index > len(stories) { + index = len(stories) + } + } else if after != nil { + index = 0 + if *after > 0 { + index = len(stories) + for i, story := range stories { + if story.ID == *after { + index = i + 1 + break + } + } + } + } + stories = append(stories, nil) + copy(stories[index+1:], stories[index:]) + stories[index] = target + for i, story := range stories { + story.Lane = lane + story.Position = float64(i + 1) + } +} +func normalizeOrder(state *issueState, lane string, exclude int) { + for i, story := range laneStories(state, lane, exclude) { + story.Position = float64(i + 1) + } +} +func laneStories(state *issueState, lane string, exclude int) []*protocol.Issue { + var stories []*protocol.Issue + for i := range state.Issues { + issue := &state.Issues[i] + if issue.ID == exclude || issue.Type != "story" || issue.Archived || normalizeLane(issue.Lane) != lane { + continue + } + stories = append(stories, issue) + } + sort.SliceStable(stories, func(i, j int) bool { + if stories[i].Position != stories[j].Position { + if stories[i].Position == 0 { + return false + } + if stories[j].Position == 0 { + return true + } + return stories[i].Position < stories[j].Position + } + return stories[i].ID < stories[j].ID + }) + return stories +} +func sortIssues(issues []protocol.Issue) { + sort.SliceStable(issues, func(i, j int) bool { + left, right := laneIndex(normalizeLane(issues[i].Lane)), laneIndex(normalizeLane(issues[j].Lane)) + if left != right { + return left < right + } + if issues[i].Position != issues[j].Position { + if issues[i].Position == 0 { + return false + } + if issues[j].Position == 0 { + return true + } + return issues[i].Position < issues[j].Position + } + return issues[i].ID < issues[j].ID + }) +} +func laneIndex(lane string) int { + for i, value := range []string{"backlog", "ready", "doing", "review", "done"} { + if value == lane { + return i + } + } + return 5 +} +func normalizeLane(value string) string { + lane, err := parseLane(value) + if err != nil { + return "backlog" + } + return lane +} +func parseLane(value string) (string, error) { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "backlog": + return "backlog", nil + case "ready", "todo", "to-do": + return "ready", nil + case "doing", "in-progress", "in_progress", "progress": + return "doing", nil + case "review", "in-review", "in_review": + return "review", nil + case "done", "closed": + return "done", nil + default: + return "", errors.New("unknown board lane") + } +} +func summary(value string) string { + value = strings.Join(strings.Fields(value), " ") + runes := []rune(value) + if len(runes) <= 80 { + return value + } + return strings.TrimSpace(string(runes[:79])) + "..." +} +func first(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} +func formatPosition(value float64) string { + const digits = "0123456789" + whole := int64(value) + fraction := int64((value-float64(whole))*1_000_000 + 0.5) + result := intString(whole) + "." + divisor := int64(100000) + for divisor > 0 { + result += string(digits[(fraction/divisor)%10]) + divisor /= 10 + } + return result +} +func intString(value int64) string { + if value == 0 { + return "0" + } + var buffer [20]byte + index := len(buffer) + for value > 0 { + index-- + buffer[index] = byte('0' + value%10) + value /= 10 + } + return string(buffer[index:]) +} diff --git a/broker/local/json.go b/broker/local/json.go new file mode 100644 index 0000000..9b552b9 --- /dev/null +++ b/broker/local/json.go @@ -0,0 +1,6 @@ +package local + +import "encoding/json" + +func decodeJSON(data []byte, target any) error { return json.Unmarshal(data, target) } +func encodeJSON(value any) ([]byte, error) { return json.Marshal(value) } diff --git a/broker/local/refs.go b/broker/local/refs.go new file mode 100644 index 0000000..99f154f --- /dev/null +++ b/broker/local/refs.go @@ -0,0 +1,90 @@ +package local + +import ( + "context" + "encoding/base64" + "errors" + "io/fs" + "strings" + + "github.com/bucketgit/bgit/protocol" +) + +func (b *Broker) UpdateRef(ctx context.Context, request protocol.RefUpdateRequest, user string) error { + unlock := b.LockRepository(request.Repo) + defer unlock() + state, err := b.LoadRepositoryState(ctx, request.Repo) + if err != nil { + return err + } + if err := b.Repository(state.Repo).CompareAndSwapRef(ctx, request.Ref, request.Old, request.New); err != nil { + return err + } + if state.Refs == nil { + state.Refs = map[string]string{} + } + recordPath := ".bucketgit/broker-state/v1/refs/" + base64.RawURLEncoding.EncodeToString([]byte(request.Ref)) + ".json" + if IsZeroOID(request.New) { + delete(state.Refs, request.Ref) + _ = b.DeleteObject(ctx, state.Repo, recordPath) + } else { + state.Refs[request.Ref] = request.New + record := map[string]any{"ref": request.Ref, "hash": request.New, "updated_by": first(user, "owner"), "updated_at": b.now().UTC().Format("2006-01-02T15:04:05Z07:00")} + if err := b.SaveJSON(ctx, state.Repo, recordPath, record); err != nil { + return err + } + } + state.UpdatedAt = b.now().UTC().Format("2006-01-02T15:04:05Z07:00") + return b.SaveRepositoryState(ctx, state) +} + +// ReconcileRepository repairs the broker's cached ref map and per-ref records +// from the storage backend's authoritative Git refs. It is safe to call after +// an interrupted update or when another local-broker process updated a shared +// bucket. +func (b *Broker) ReconcileRepository(ctx context.Context, repo protocol.Repository, user string) (RepositoryState, error) { + unlock := b.LockRepository(repo) + defer unlock() + state, err := b.LoadRepositoryState(ctx, repo) + if err != nil { + return RepositoryState{}, err + } + refs, err := b.Repository(state.Repo).ListRefs(ctx) + if err != nil { + return RepositoryState{}, err + } + recordPrefix := ".bucketgit/broker-state/v1/refs/" + records, err := b.ListObjects(ctx, state.Repo, recordPrefix) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return RepositoryState{}, err + } + wanted := make(map[string]struct{}, len(refs)) + for ref, oid := range refs { + recordPath := recordPrefix + base64.RawURLEncoding.EncodeToString([]byte(ref)) + ".json" + wanted[recordPath] = struct{}{} + record := map[string]any{"ref": ref, "hash": oid, "updated_by": first(user, "reconcile"), "updated_at": b.now().UTC().Format("2006-01-02T15:04:05Z07:00")} + if err := b.SaveJSON(ctx, state.Repo, recordPath, record); err != nil { + return RepositoryState{}, err + } + } + for _, recordPath := range records { + if !strings.HasPrefix(recordPath, recordPrefix) { + continue + } + if _, ok := wanted[recordPath]; !ok { + if err := b.DeleteObject(ctx, state.Repo, recordPath); err != nil { + return RepositoryState{}, err + } + } + } + state.Refs = refs + state.UpdatedAt = b.now().UTC().Format("2006-01-02T15:04:05Z07:00") + if err := b.SaveRepositoryState(ctx, state); err != nil { + return RepositoryState{}, err + } + return state, nil +} + +func IsZeroOID(value string) bool { + return value == "" || value == "0000000000000000000000000000000000000000" +} diff --git a/broker/local/state.go b/broker/local/state.go new file mode 100644 index 0000000..9bdae68 --- /dev/null +++ b/broker/local/state.go @@ -0,0 +1,135 @@ +package local + +import ( + "context" + "errors" + "io/fs" + "sort" + "strings" + + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" + fsstore "github.com/bucketgit/bgit/store/fs" +) + +const repositoryStatePath = ".bucketgit/broker-state/v1/repo.json" + +type RepositoryState struct { + Repo protocol.Repository `json:"repo"` + Keys []protocol.Key `json:"keys"` + Refs map[string]string `json:"refs,omitempty"` + Teams []protocol.RepositoryTeamGrant `json:"teams,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} +type Owners struct { + Keys []protocol.Key `json:"keys"` +} +type RepositoryIndex struct { + Repos []protocol.Repository `json:"repos"` +} + +func (b *Broker) LoadRepositoryState(ctx context.Context, repo protocol.Repository) (RepositoryState, error) { + var state RepositoryState + err := b.LoadJSON(ctx, repo, repositoryStatePath, &state) + return state, err +} +func (b *Broker) SaveRepositoryState(ctx context.Context, state RepositoryState) error { + if err := b.SaveJSON(ctx, state.Repo, repositoryStatePath, state); err != nil { + return err + } + return b.UpsertRepository(ctx, state.Repo) +} + +func (b *Broker) LoadRepositoryIndex(ctx context.Context) (RepositoryIndex, error) { + backend, err := fsstore.New(b.root) + if err != nil { + return RepositoryIndex{}, err + } + var index RepositoryIndex + data, err := backend.Read(ctx, "repos.json") + if errors.Is(err, fs.ErrNotExist) { + return index, nil + } + if err != nil { + return index, err + } + if err := decodeJSON(data, &index); err != nil { + return index, err + } + return index, nil +} +func (b *Broker) FindRepository(ctx context.Context, logical string) (protocol.Repository, bool, error) { + index, err := b.LoadRepositoryIndex(ctx) + if err != nil { + return protocol.Repository{}, false, err + } + for _, repo := range index.Repos { + if strings.EqualFold(strings.TrimSpace(repo.Logical), strings.TrimSpace(logical)) { + return repo, true, nil + } + } + return protocol.Repository{}, false, nil +} +func (b *Broker) UpsertRepository(ctx context.Context, repo protocol.Repository) error { + if strings.TrimSpace(repo.Logical) == "" { + return nil + } + unlock := b.LockRepository(protocol.Repository{Logical: "__index__"}) + defer unlock() + backend, err := fsstore.New(b.root) + if err != nil { + return err + } + for attempt := 0; attempt < 16; attempt++ { + var index RepositoryIndex + expected, err := loadJSONState(ctx, backend, "repos.json", &index) + if err != nil { + return err + } + found := false + for i := range index.Repos { + if strings.EqualFold(strings.TrimSpace(index.Repos[i].Logical), strings.TrimSpace(repo.Logical)) { + index.Repos[i] = repo + found = true + break + } + } + if !found { + index.Repos = append(index.Repos, repo) + } + sort.Slice(index.Repos, func(i, j int) bool { return index.Repos[i].Logical < index.Repos[j].Logical }) + if err := compareAndSwapJSON(ctx, backend, "repos.json", expected, index); errors.Is(err, store.ErrConflict) { + continue + } else { + return err + } + } + return store.ErrConflict +} + +func loadJSONState(ctx context.Context, backend store.Writer, path string, target any) (store.ObjectState, error) { + data, err := backend.Read(ctx, path) + if errors.Is(err, fs.ErrNotExist) { + return store.ObjectState{}, nil + } + if err != nil { + return store.ObjectState{}, err + } + if err := decodeJSON(data, target); err != nil { + return store.ObjectState{}, err + } + return store.ObjectState{Exists: true, Data: data}, nil +} + +func compareAndSwapJSON(ctx context.Context, backend store.Writer, path string, expected store.ObjectState, value any) error { + cas, ok := backend.(store.CompareAndSwapper) + if !ok { + return store.ErrNotSupported + } + data, err := encodeJSON(value) + if err != nil { + return err + } + return cas.CompareAndSwap(ctx, path, expected, data) +} diff --git a/broker/test_support/signing_fixture_test.js b/broker/test_support/signing_fixture_test.js new file mode 100644 index 0000000..3541bd4 --- /dev/null +++ b/broker/test_support/signing_fixture_test.js @@ -0,0 +1,35 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const fixture = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'spec', 'testdata', 'signing-v2.json'), 'utf8')); + +function canonicalGCP(vector) { + return [ + 'bgit-broker-v2', + String(vector.method || '').trim().toUpperCase(), + String(vector.path || '/').trim(), + String(vector.host || '').trim().toLowerCase(), + String(vector.timestamp || '').trim(), + String(vector.nonce || '').trim(), + crypto.createHash('sha256').update(Buffer.from(vector.body)).digest('hex'), + ].join('\n'); +} + +function canonicalAWS(vector) { + const digest = crypto.createHash('sha256').update(Buffer.from(vector.body)).digest('hex'); + return ['bgit-broker-v2', vector.method.toUpperCase(), vector.path, vector.host.toLowerCase(), vector.timestamp, vector.nonce, digest].join('\n'); +} + +for (const vector of fixture.vectors) { + for (const [runtime, canonical] of [['gcp', canonicalGCP], ['aws', canonicalAWS]]) { + const actual = canonical(vector); + if (actual !== vector.message) { + throw new Error(`${runtime} signing fixture ${vector.name} mismatch\nactual: ${actual}\nexpected: ${vector.message}`); + } + } +} + +process.stdout.write(`validated ${fixture.vectors.length} signing fixture(s) for AWS and GCP\n`); diff --git a/broker_assets.go b/broker_assets.go deleted file mode 100644 index 240be39..0000000 --- a/broker_assets.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "embed" - "os" - "path/filepath" - "strings" -) - -//go:embed broker/gcp/package.json broker/gcp/index.js broker/gcp/materializer.js broker/aws/template.yaml broker/test_support/sqlite_broker.js -var brokerAssets embed.FS - -func writeGCPBrokerSource(dir string) error { - for _, name := range []string{"package.json", "index.js"} { - data, err := brokerAssets.ReadFile("broker/gcp/" + name) - if err != nil { - return err - } - body := strings.ReplaceAll(string(data), "{{BROKER_VERSION}}", brokerVersion()) - if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { - return err - } - } - return nil -} - -func writeGCPMaterializerSource(dir string) error { - for _, name := range []string{"package.json", "materializer.js"} { - data, err := brokerAssets.ReadFile("broker/gcp/" + name) - if err != nil { - return err - } - body := strings.ReplaceAll(string(data), "{{BROKER_VERSION}}", brokerVersion()) - target := name - if name == "materializer.js" { - target = "index.js" - } - if err := os.WriteFile(filepath.Join(dir, target), []byte(body), 0o644); err != nil { - return err - } - } - return nil -} - -func awsBrokerCloudFormationTemplate() string { - data, err := brokerAssets.ReadFile("broker/aws/template.yaml") - if err != nil { - return "" - } - return strings.ReplaceAll(string(data), "{{BROKER_VERSION}}", brokerVersion()) -} diff --git a/broker_data_path.go b/broker_data_path.go deleted file mode 100644 index 062f492..0000000 --- a/broker_data_path.go +++ /dev/null @@ -1,240 +0,0 @@ -package main - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/fs" - "net/http" - "strings" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" -) - -type brokerObjectCapabilityRequest struct { - Repo brokerRepo `json:"repo"` - Path string `json:"path"` - Operation string `json:"operation"` - Size int64 `json:"size,omitempty"` - Resumable bool `json:"resumable,omitempty"` -} - -type brokerObjectCapabilityResponse struct { - Provider string `json:"provider"` - Mode string `json:"mode"` - Method string `json:"method,omitempty"` - URL string `json:"url,omitempty"` - Headers map[string]string `json:"headers,omitempty"` - Bucket string `json:"bucket,omitempty"` - Prefix string `json:"prefix,omitempty"` - Object string `json:"object,omitempty"` - Profile string `json:"profile,omitempty"` - Region string `json:"region,omitempty"` - Credentials brokerObjectAWSCredentials `json:"credentials,omitempty"` -} - -type brokerObjectAWSCredentials struct { - AccessKeyID string `json:"access_key_id"` - SecretAccessKey string `json:"secret_access_key"` - SessionToken string `json:"session_token"` -} - -type brokerRefsRequest struct { - Repo brokerRepo `json:"repo"` -} - -type brokerRefsResponse struct { - Refs map[string]string `json:"refs"` -} - -func (s *brokerGitStore) listRefs(ctx context.Context) (map[string]string, error) { - var resp brokerRefsResponse - if err := brokerPostContext(ctx, s.brokerURL, "/refs/list", brokerRefsRequest{Repo: repoForBroker(s.cfg)}, &resp); err != nil { - return nil, err - } - refs := map[string]string{} - for ref, hash := range resp.Refs { - if strings.HasPrefix(ref, "refs/") && isHexHash(strings.TrimSpace(hash)) { - refs[ref] = strings.TrimSpace(hash) - } - } - return refs, nil -} - -func (s *brokerGitStore) write(ctx context.Context, objectPath string, data []byte) error { - capability, err := s.objectCapability(ctx, objectPath, "write", int64(len(data))) - if err != nil { - return err - } - return s.writeWithCapability(ctx, capability, data) -} - -func (s *brokerGitStore) delete(ctx context.Context, objectPath string) error { - capability, err := s.objectCapability(ctx, objectPath, "delete", 0) - if err != nil { - return err - } - return s.deleteWithCapability(ctx, capability) -} - -func (s *brokerGitStore) readWithCapability(ctx context.Context, objectPath string) ([]byte, bool, error) { - capability, err := s.objectCapability(ctx, objectPath, "read", 0) - if err != nil { - if isBrokerNotFoundError(err) { - return nil, true, fs.ErrNotExist - } - if isBrokerCapabilityUnsupported(err) { - return nil, false, nil - } - return nil, true, err - } - data, err := s.getWithCapability(ctx, capability) - if errors.Is(err, fs.ErrNotExist) { - return nil, true, fs.ErrNotExist - } - return data, true, err -} - -func (s *brokerGitStore) objectCapability(ctx context.Context, objectPath, operation string, size int64) (brokerObjectCapabilityResponse, error) { - var resp brokerObjectCapabilityResponse - req := brokerObjectCapabilityRequest{ - Repo: repoForBroker(s.cfg), - Path: strings.TrimPrefix(objectPath, "/"), - Operation: operation, - Size: size, - Resumable: s.cfg.provider == "gcs" && operation == "write" && size > 32*1024*1024, - } - err := brokerPostContext(ctx, s.brokerURL, "/objects/capability", req, &resp) - return resp, err -} - -func (s *brokerGitStore) getWithCapability(ctx context.Context, capability brokerObjectCapabilityResponse) ([]byte, error) { - if capability.Mode == "local" { - return localBrokerCapabilityRead(ctx, capability) - } - if capability.Mode == "sts" || capability.Provider == "s3" { - client := s3ClientForBrokerCapability(capability) - out, err := client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(capability.Bucket), - Key: aws.String(capability.Object), - }) - if err != nil { - if isS3NotFound(err) { - return nil, fs.ErrNotExist - } - return nil, err - } - defer out.Body.Close() - return io.ReadAll(out.Body) - } - httpReq, err := http.NewRequestWithContext(ctx, firstNonEmpty(capability.Method, http.MethodGet), capability.URL, nil) - if err != nil { - return nil, err - } - for key, value := range capability.Headers { - httpReq.Header.Set(key, value) - } - httpResp, err := http.DefaultClient.Do(httpReq) - if err != nil { - return nil, err - } - defer httpResp.Body.Close() - if httpResp.StatusCode == http.StatusNotFound { - return nil, fs.ErrNotExist - } - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - body, _ := io.ReadAll(httpResp.Body) - return nil, fmt.Errorf("broker object GET: %s %s", httpResp.Status, strings.TrimSpace(string(body))) - } - return io.ReadAll(httpResp.Body) -} - -func (s *brokerGitStore) writeWithCapability(ctx context.Context, capability brokerObjectCapabilityResponse, data []byte) error { - if capability.Mode == "local" { - return localBrokerCapabilityWrite(ctx, capability, data) - } - if capability.Mode == "sts" || capability.Provider == "s3" { - client := s3ClientForBrokerCapability(capability) - _, err := client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(capability.Bucket), - Key: aws.String(capability.Object), - Body: bytes.NewReader(data), - }) - return err - } - method := firstNonEmpty(capability.Method, http.MethodPut) - httpReq, err := http.NewRequestWithContext(ctx, method, capability.URL, bytes.NewReader(data)) - if err != nil { - return err - } - for key, value := range capability.Headers { - httpReq.Header.Set(key, value) - } - httpResp, err := http.DefaultClient.Do(httpReq) - if err != nil { - return err - } - defer httpResp.Body.Close() - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - body, _ := io.ReadAll(httpResp.Body) - return fmt.Errorf("broker object %s: %s %s", method, httpResp.Status, strings.TrimSpace(string(body))) - } - return nil -} - -func (s *brokerGitStore) deleteWithCapability(ctx context.Context, capability brokerObjectCapabilityResponse) error { - if capability.Mode == "local" { - return localBrokerCapabilityDelete(ctx, capability) - } - if capability.Mode == "sts" || capability.Provider == "s3" { - client := s3ClientForBrokerCapability(capability) - _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(capability.Bucket), - Key: aws.String(capability.Object), - }) - return err - } - httpReq, err := http.NewRequestWithContext(ctx, firstNonEmpty(capability.Method, http.MethodDelete), capability.URL, nil) - if err != nil { - return err - } - for key, value := range capability.Headers { - httpReq.Header.Set(key, value) - } - httpResp, err := http.DefaultClient.Do(httpReq) - if err != nil { - return err - } - defer httpResp.Body.Close() - if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { - body, _ := io.ReadAll(httpResp.Body) - return fmt.Errorf("broker object DELETE: %s %s", httpResp.Status, strings.TrimSpace(string(body))) - } - return nil -} - -func s3ClientForBrokerCapability(capability brokerObjectCapabilityResponse) *s3.Client { - region := firstNonEmpty(capability.Region, defaultAWSRegion()) - creds := credentials.NewStaticCredentialsProvider( - capability.Credentials.AccessKeyID, - capability.Credentials.SecretAccessKey, - capability.Credentials.SessionToken, - ) - return s3.New(s3.Options{ - Region: region, - Credentials: aws.NewCredentialsCache(creds), - }) -} - -func isBrokerCapabilityUnsupported(err error) bool { - if err == nil { - return false - } - message := err.Error() - return strings.Contains(message, "unknown broker endpoint") || - strings.Contains(message, "404") -} diff --git a/cmd/bgit/main.go b/cmd/bgit/main.go new file mode 100644 index 0000000..a60bb5c --- /dev/null +++ b/cmd/bgit/main.go @@ -0,0 +1,8 @@ +package main + +import ( + bgit "github.com/bucketgit/bgit" + "github.com/bucketgit/bgit/internal/cli" +) + +func main() { cli.Main(bgit.RunExecutable) } diff --git a/examples_sdk_test.go b/examples_sdk_test.go new file mode 100644 index 0000000..a1c8f9c --- /dev/null +++ b/examples_sdk_test.go @@ -0,0 +1,106 @@ +package bgit_test + +import ( + "bytes" + "context" + "io/fs" + "net/http" + + "github.com/bucketgit/bgit/broker/capability" + brokerclient "github.com/bucketgit/bgit/broker/client" + localbroker "github.com/bucketgit/bgit/broker/local" + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/repository" + "github.com/bucketgit/bgit/store" + gcsstore "github.com/bucketgit/bgit/store/gcs" + s3store "github.com/bucketgit/bgit/store/s3" + "github.com/bucketgit/bgit/transport" +) + +func Example_s3Repository() { + ctx := context.Background() + objects, _ := s3store.Load(ctx, s3store.Options{Bucket: "123456789012-demo", Profile: "work", Region: "eu-west-1"}) + _ = repository.Open(objects, objects) +} + +func Example_gcsRepository() { + ctx := context.Background() + objects, _ := gcsstore.Load(ctx, gcsstore.Options{Bucket: "project-demo"}) + if objects != nil { + defer objects.Close() + } + _ = repository.Open(objects, objects) +} + +func Example_remoteBrokerRepository() { + ctx := context.Background() + signatures := brokerclient.SignatureProviderFunc(func(context.Context, string, string, []byte) ([]http.Header, error) { + return []http.Header{{}}, nil + }) + client, _ := brokerclient.New("https://broker.example.com", brokerclient.Options{HTTPClient: http.DefaultClient, Signatures: signatures}) + repoInfo := protocol.Repository{Provider: "gcp", Bucket: "project-demo", Prefix: "repo.git", Logical: "demo.git"} + objects, _ := capability.New(client, repoInfo, capability.Options{HTTPClient: http.DefaultClient}) + _ = repository.Open(objects, objects) + _ = ctx +} + +func Example_inProcessBroker() { + controlPlane, _ := localbroker.New(localbroker.Options{Root: "/srv/bgit"}) + repoInfo := protocol.Repository{Provider: "file", Bucket: "demo.git", Logical: "demo.git"} + objects := controlPlane.Repository(repoInfo) + _ = repository.Open(objects, objects) +} + +func Example_transportServer() { + ctx := context.Background() + objects := &exampleStore{objects: map[string][]byte{}, refs: map[string]string{}} + repo := repository.Open(objects, objects) + var input, output bytes.Buffer + _ = transport.ServeUploadPack(ctx, repo, &input, &output) + _ = transport.ServeReceivePack(ctx, repo, objects, &input, &output) +} + +func Example_customStore() { + objects := &exampleStore{objects: map[string][]byte{}, refs: map[string]string{}} + _ = repository.Open(objects, objects) +} + +type exampleStore struct { + objects map[string][]byte + refs map[string]string +} + +func (s *exampleStore) Read(_ context.Context, path string) ([]byte, error) { + value, ok := s.objects[path] + if !ok { + return nil, fs.ErrNotExist + } + return append([]byte(nil), value...), nil +} +func (s *exampleStore) List(context.Context, string) ([]string, error) { return nil, nil } +func (s *exampleStore) Write(_ context.Context, path string, value []byte) error { + s.objects[path] = append([]byte(nil), value...) + return nil +} +func (s *exampleStore) Delete(_ context.Context, path string) error { + delete(s.objects, path) + return nil +} +func (s *exampleStore) ListRefs(context.Context) (map[string]string, error) { + result := map[string]string{} + for ref, oid := range s.refs { + result[ref] = oid + } + return result, nil +} +func (s *exampleStore) CompareAndSwapRef(_ context.Context, ref, oldOID, newOID string) error { + if s.refs[ref] != oldOID { + return store.ErrConflict + } + if newOID == "" { + delete(s.refs, ref) + } else { + s.refs[ref] = newOID + } + return nil +} diff --git a/examples_test.go b/examples_test.go new file mode 100644 index 0000000..d9a9867 --- /dev/null +++ b/examples_test.go @@ -0,0 +1,18 @@ +package bgit_test + +import ( + "context" + "log" + + "github.com/bucketgit/bgit/repository" + fsstore "github.com/bucketgit/bgit/store/fs" +) + +func Example() { + objects, err := fsstore.New("/srv/git/example.git") + if err != nil { + log.Fatal(err) + } + repo := repository.Open(objects, nil) + _, _ = repo.Resolve(context.Background(), "main") +} diff --git a/git_receive.go b/git_receive.go deleted file mode 100644 index a4407f9..0000000 --- a/git_receive.go +++ /dev/null @@ -1,383 +0,0 @@ -package main - -import ( - "bufio" - "context" - "errors" - "fmt" - "io" - "io/fs" - "strings" -) - -type receivePackCommand struct { - old string - new string - ref string - caps map[string]bool -} - -type receivePackRequest struct { - commands []receivePackCommand - caps map[string]bool - pushOptions []string - pack io.Reader -} - -func readReceivePackRequest(stdin io.Reader) (receivePackRequest, error) { - r := bufio.NewReader(stdin) - var req receivePackRequest - for { - line, err := readPktLine(r) - if err != nil { - return req, err - } - switch line.kind { - case pktLineFlush: - if req.caps["push-options"] { - options, err := readReceivePackPushOptions(r) - if err != nil { - return req, err - } - req.pushOptions = options - } - req.pack = r - return req, nil - case pktLineData: - default: - continue - } - cmd, caps, err := parseReceivePackCommandLine(string(line.data), len(req.commands) == 0) - if err != nil { - return req, err - } - req.commands = append(req.commands, cmd) - if caps != nil { - req.caps = caps - } - } -} - -func readReceivePackPushOptions(r *bufio.Reader) ([]string, error) { - var options []string - for { - line, err := readPktLine(r) - if err != nil { - return nil, err - } - switch line.kind { - case pktLineFlush: - return options, nil - case pktLineData: - options = append(options, strings.TrimRight(string(line.data), "\n")) - default: - return nil, fmt.Errorf("invalid push-options pkt-line") - } - } -} - -func parseReceivePackCommandLine(line string, first bool) (receivePackCommand, map[string]bool, error) { - line = strings.TrimRight(line, "\n") - capText := "" - if first { - before, after, ok := strings.Cut(line, "\x00") - if ok { - line = before - capText = after - } - } - fields := strings.Fields(line) - if len(fields) != 3 { - return receivePackCommand{}, nil, fmt.Errorf("invalid receive-pack command %q", line) - } - if !isHexHash(fields[0]) || !isHexHash(fields[1]) { - return receivePackCommand{}, nil, fmt.Errorf("invalid receive-pack object id in %q", line) - } - if !strings.HasPrefix(fields[2], "refs/") { - return receivePackCommand{}, nil, fmt.Errorf("invalid receive-pack ref %q", fields[2]) - } - cmd := receivePackCommand{old: fields[0], new: fields[1], ref: fields[2]} - caps := map[string]bool{} - if capText != "" { - for _, cap := range strings.Fields(capText) { - caps[cap] = true - if name, _, ok := strings.Cut(cap, "="); ok { - caps[name] = true - } - } - cmd.caps = caps - } - return cmd, caps, nil -} - -func zeroObjectID() string { - return "0000000000000000000000000000000000000000" -} - -func (c receivePackCommand) action() string { - switch { - case c.old == zeroObjectID() && c.new != zeroObjectID(): - return "create" - case c.old != zeroObjectID() && c.new == zeroObjectID(): - return "delete" - case c.old != zeroObjectID() && c.new != zeroObjectID(): - return "update" - default: - return "noop" - } -} - -func writeReceivePackReportStatus(w io.Writer, commands []receivePackCommand, unpackErr error, commandErrs map[string]error) error { - if unpackErr != nil { - if err := writePktLineString(w, "unpack "+unpackErr.Error()+"\n"); err != nil { - return err - } - } else if err := writePktLineString(w, "unpack ok\n"); err != nil { - return err - } - for _, cmd := range commands { - if err := commandErrs[cmd.ref]; err != nil { - if err := writePktLineString(w, "ng "+cmd.ref+" "+err.Error()+"\n"); err != nil { - return err - } - continue - } - if err := writePktLineString(w, "ok "+cmd.ref+"\n"); err != nil { - return err - } - } - return writePktFlush(w) -} - -func serveReceivePack(ctx context.Context, repo *nativeGitRepo, stdin io.Reader, stdout io.Writer) error { - req, err := readReceivePackRequest(stdin) - if err != nil { - return err - } - if len(req.commands) == 0 { - return nil - } - store, ok := repo.store.(writableGitRemoteStore) - if !ok { - return errors.New("receive-pack requires a writable store") - } - - var received map[string]gitObject - var unpackErr error - if receivePackNeedsPack(req.commands) { - received, unpackErr = ingestReceivePack(ctx, repo, store, req.pack) - } - if unpackErr != nil { - if req.caps["report-status"] || req.caps["report-status-v2"] { - _ = writeReceivePackReport(stdout, req, unpackErr, map[string]error{}) - } - return unpackErr - } - - brokerURL := "" - if repo.cfg.origin != "" { - var err error - brokerURL, err = brokerURLForSSHService(repo.cfg) - if err != nil { - return err - } - } - commandErrs, applyErr := applyReceivePackCommands(ctx, repo, store, req.commands, req.caps["atomic"], brokerURL, received) - if req.caps["report-status"] || req.caps["report-status-v2"] { - if err := writeReceivePackReport(stdout, req, nil, commandErrs); err != nil && applyErr == nil { - return err - } - } - return applyErr -} - -func writeReceivePackReport(stdout io.Writer, req receivePackRequest, unpackErr error, commandErrs map[string]error) error { - if !req.caps["side-band-64k"] { - return writeReceivePackReportStatus(stdout, req.commands, unpackErr, commandErrs) - } - var report strings.Builder - if err := writeReceivePackReportStatus(&report, req.commands, unpackErr, commandErrs); err != nil { - return err - } - if err := writeSideband(stdout, 1, []byte(report.String()), 65516); err != nil { - return err - } - return writePktFlush(stdout) -} - -func receivePackNeedsPack(commands []receivePackCommand) bool { - for _, cmd := range commands { - if cmd.new != zeroObjectID() { - return true - } - } - return false -} - -func applyReceivePackCommands(ctx context.Context, repo *nativeGitRepo, store writableGitRemoteStore, commands []receivePackCommand, atomic bool, brokerURL string, received map[string]gitObject) (map[string]error, error) { - refs, err := repo.refs(ctx) - if err != nil { - return nil, err - } - commandErrs := map[string]error{} - var firstErr error - for _, cmd := range commands { - if err := validateReceivePackCommand(ctx, repo, refs, cmd, received); err != nil { - commandErrs[cmd.ref] = err - if firstErr == nil { - firstErr = err - } - } - } - if atomic && firstErr != nil { - for _, cmd := range commands { - if commandErrs[cmd.ref] == nil { - commandErrs[cmd.ref] = errors.New("atomic push failed") - } - } - return commandErrs, firstErr - } - for _, cmd := range commands { - if commandErrs[cmd.ref] != nil { - continue - } - switch cmd.action() { - case "create", "update": - if brokerURL != "" { - if err := brokerUpdateRef(brokerURL, repo.cfg, cmd.ref, cmd.old, cmd.new); err != nil { - commandErrs[cmd.ref] = err - if firstErr == nil { - firstErr = err - } - continue - } - refs[cmd.ref] = cmd.new - continue - } - if err := store.write(ctx, cmd.ref, []byte(cmd.new+"\n")); err != nil { - commandErrs[cmd.ref] = err - if firstErr == nil { - firstErr = err - } - } else { - refs[cmd.ref] = cmd.new - } - case "delete": - if brokerURL != "" { - if err := brokerUpdateRef(brokerURL, repo.cfg, cmd.ref, cmd.old, zeroObjectID()); err != nil { - commandErrs[cmd.ref] = err - if firstErr == nil { - firstErr = err - } - continue - } - delete(refs, cmd.ref) - if strings.HasPrefix(cmd.ref, "refs/heads/") { - _ = unsetGitBranchTracking(".", strings.TrimPrefix(cmd.ref, "refs/heads/")) - } - continue - } - if err := store.delete(ctx, cmd.ref); err != nil { - commandErrs[cmd.ref] = err - if firstErr == nil { - firstErr = err - } - } else { - delete(refs, cmd.ref) - if strings.HasPrefix(cmd.ref, "refs/heads/") { - _ = unsetGitBranchTracking(".", strings.TrimPrefix(cmd.ref, "refs/heads/")) - } - } - case "noop": - default: - err := fmt.Errorf("unsupported ref update %s", cmd.action()) - commandErrs[cmd.ref] = err - if firstErr == nil { - firstErr = err - } - } - } - return commandErrs, firstErr -} - -func validateReceivePackCommand(ctx context.Context, repo *nativeGitRepo, refs map[string]string, cmd receivePackCommand, received map[string]gitObject) error { - current, exists := refs[cmd.ref] - switch cmd.action() { - case "create": - if exists { - return fmt.Errorf("ref already exists") - } - case "update": - if !exists { - return fmt.Errorf("ref does not exist") - } - if current != cmd.old { - return fmt.Errorf("stale ref") - } - if strings.HasPrefix(cmd.ref, "refs/heads/") { - ancestor, err := isAncestorWithReceived(ctx, repo, received, current, cmd.new) - if err != nil { - return err - } - if !ancestor { - return fmt.Errorf("non-fast-forward update") - } - } - case "delete": - if !exists { - return fmt.Errorf("ref does not exist") - } - if current != cmd.old { - return fmt.Errorf("stale ref") - } - case "noop": - return nil - default: - return fmt.Errorf("unsupported ref update %s", cmd.action()) - } - if cmd.new == zeroObjectID() { - return nil - } - if _, ok := received[cmd.new]; ok { - return nil - } - if _, err := repo.object(ctx, cmd.new); err != nil { - if errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("missing object %s", cmd.new) - } - return err - } - return nil -} - -func isAncestorWithReceived(ctx context.Context, repo *nativeGitRepo, received map[string]gitObject, ancestor, descendant string) (bool, error) { - stack := []string{descendant} - seen := map[string]struct{}{} - for len(stack) > 0 { - hash := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if hash == ancestor { - return true, nil - } - if _, ok := seen[hash]; ok { - continue - } - seen[hash] = struct{}{} - commit, err := commitWithReceived(ctx, repo, received, hash) - if err != nil { - return false, err - } - stack = append(stack, commit.parents...) - } - return false, nil -} - -func commitWithReceived(ctx context.Context, repo *nativeGitRepo, received map[string]gitObject, hash string) (commitObject, error) { - if obj, ok := received[hash]; ok { - if obj.typ != "commit" { - return commitObject{}, fmt.Errorf("object %s is %s, not commit", hash, obj.typ) - } - return parseCommit(hash, obj.data) - } - return repo.commit(ctx, hash) -} diff --git a/go.mod b/go.mod index 5c16c94..6a481ae 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module bgit +module github.com/bucketgit/bgit go 1.25.0 @@ -15,6 +15,7 @@ require ( github.com/aws/smithy-go v1.24.2 golang.org/x/crypto v0.52.0 golang.org/x/oauth2 v0.34.0 + golang.org/x/sys v0.45.0 golang.org/x/term v0.43.0 google.golang.org/api v0.191.0 gopkg.in/yaml.v3 v3.0.1 @@ -54,7 +55,6 @@ require ( go.opentelemetry.io/otel/trace v1.41.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect diff --git a/admin.go b/internal/app/admin.go similarity index 98% rename from admin.go rename to internal/app/admin.go index 931f85e..f8ef559 100644 --- a/admin.go +++ b/internal/app/admin.go @@ -1,4 +1,4 @@ -package main +package app import ( "context" @@ -378,9 +378,7 @@ func (p *s3BucketPolicy) addBucketGitGrant(bucket, prefix, action, principal str prefix = strings.Trim(prefix, "/") baseSid := s3GrantSid(action, principal, prefix) p.removeStatements(baseSid) - for _, statement := range s3GrantStatements(bucket, prefix, action, principal, baseSid) { - p.Statement = append(p.Statement, statement) - } + p.Statement = append(p.Statement, s3GrantStatements(bucket, prefix, action, principal, baseSid)...) } func (p *s3BucketPolicy) removeStatements(baseSid string) { @@ -464,7 +462,10 @@ func s3ListStatement(bucketARN, prefix string, principal any, baseSid string) s3 func s3GrantSid(action, principal, prefix string) string { sum := sha1.Sum([]byte(action + "\x00" + principal + "\x00" + prefix)) - return "BucketGit" + strings.Title(action) + hex.EncodeToString(sum[:])[:12] + if action != "" { + action = strings.ToUpper(action[:1]) + action[1:] + } + return "BucketGit" + action + hex.EncodeToString(sum[:])[:12] } func normalizeAWSPrincipal(raw string) (string, error) { diff --git a/auth_capabilities.go b/internal/app/auth_capabilities.go similarity index 82% rename from auth_capabilities.go rename to internal/app/auth_capabilities.go index 3558190..4200a5c 100644 --- a/auth_capabilities.go +++ b/internal/app/auth_capabilities.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" @@ -15,13 +15,10 @@ import ( "strings" "time" + "github.com/bucketgit/bgit/protocol" "golang.org/x/crypto/ssh" ) -type brokerAuthStatusRequest struct { - Repo brokerRepo `json:"repo"` -} - type repoAuthCache struct { BrokerURL string `json:"broker_url,omitempty"` Repo string `json:"repo,omitempty"` @@ -29,27 +26,6 @@ type repoAuthCache struct { UpdatedAt string `json:"updated_at,omitempty"` } -type brokerAuthStatus struct { - BrokerURL string `json:"broker_url,omitempty"` - BrokerVersion string `json:"broker_version,omitempty"` - Repo brokerRepo `json:"repo,omitempty"` - Identity brokerIdentity `json:"identity,omitempty"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - Capabilities map[string]bool `json:"capabilities,omitempty"` - ResolvedAt string `json:"resolved_at,omitempty"` - CachedAt string `json:"cached_at,omitempty"` - Stale bool `json:"stale,omitempty"` - Error string `json:"error,omitempty"` -} - -type brokerIdentity struct { - User string `json:"user,omitempty"` - Source string `json:"source,omitempty"` - KeyFingerprint string `json:"key_fingerprint,omitempty"` - PublicKey string `json:"public_key,omitempty"` -} - func whoamiCommand(ctx context.Context, cfg config, args []string, stdout io.Writer) error { jsonOut := false refresh := false @@ -118,7 +94,7 @@ func whoamiAllCommand(ctx context.Context, cfg config, jsonOut bool, stdout io.W if err != nil { return err } - resp := brokerReposMineResponse{Repos: repos} + resp := protocol.RepositoriesMineResponse{Repos: repos} if jsonOut { data, err := json.MarshalIndent(resp, "", " ") if err != nil { @@ -156,16 +132,16 @@ func reposCommand(ctx context.Context, cfg config, args []string, stdout io.Writ return whoamiAllCommand(ctx, cfg, jsonOut, stdout) } -func brokerReposMineAllKeys(ctx context.Context, brokerURL string) ([]brokerRepoMembership, error) { +func brokerReposMineAllKeys(ctx context.Context, brokerURL string) ([]protocol.RepositoryMembership, error) { data := []byte(`{}`) headerSets := brokerSignatureHeaderSetsForBroker(brokerURL, "/repos/mine", data) if len(headerSets) == 0 { return nil, errors.New("no SSH agent keys available") } - merged := map[string]brokerRepoMembership{} + merged := map[string]protocol.RepositoryMembership{} var lastErr error for _, headers := range headerSets { - var resp brokerReposMineResponse + var resp protocol.RepositoriesMineResponse if err := brokerPostContextWithHeaders(ctx, brokerURL, "/repos/mine", data, headers, &resp); err != nil { lastErr = err continue @@ -178,7 +154,7 @@ func brokerReposMineAllKeys(ctx context.Context, brokerURL string) ([]brokerRepo if len(merged) == 0 && lastErr != nil { return nil, lastErr } - var repos []brokerRepoMembership + var repos []protocol.RepositoryMembership for _, repo := range merged { repos = append(repos, repo) } @@ -193,8 +169,8 @@ func brokerReposMineAllKeys(ctx context.Context, brokerURL string) ([]brokerRepo return repos, nil } -func repoMembershipWarnings(repos []brokerRepoMembership) []string { - byRepo := map[string][]brokerRepoMembership{} +func repoMembershipWarnings(repos []protocol.RepositoryMembership) []string { + byRepo := map[string][]protocol.RepositoryMembership{} for _, repo := range repos { name := firstNonEmpty(repo.Logical, repo.RepoID, repo.Repo.Logical) if name == "" { @@ -257,23 +233,7 @@ func brokerPostContextWithHeaders(ctx context.Context, brokerURL, path string, d return nil } -type brokerReposMineResponse struct { - Repos []brokerRepoMembership `json:"repos"` -} - -type brokerRepoMembership struct { - RepoID string `json:"repo_id,omitempty"` - Logical string `json:"logical,omitempty"` - Repo brokerRepo `json:"repo,omitempty"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - Source string `json:"source,omitempty"` - KeyFingerprint string `json:"key_fingerprint,omitempty"` - Suspended bool `json:"suspended,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` -} - -func brokerWhoami(ctx context.Context, cfg config, refresh bool) (brokerAuthStatus, error) { +func brokerWhoami(ctx context.Context, cfg config, refresh bool) (protocol.AuthStatus, error) { if !refresh { if cached, err := readWhoamiCache(cfg.brokerURL); err == nil && cached.BrokerURL != "" { if firstNonEmpty(cached.Repo.Logical, cached.Repo.Prefix) == firstNonEmpty(cfg.logicalRepo, cfg.prefix) { @@ -281,9 +241,13 @@ func brokerWhoami(ctx context.Context, cfg config, refresh bool) (brokerAuthStat } } } - var status brokerAuthStatus - if err := brokerPostContext(ctx, cfg.brokerURL, "/auth/status", brokerAuthStatusRequest{Repo: repoForBroker(cfg)}, &status); err != nil { - return brokerAuthStatus{}, err + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return protocol.AuthStatus{}, err + } + status, err := endpoints.AuthStatus(ctx, protocol.AuthStatusRequest{Repo: repoForBroker(cfg)}) + if err != nil { + return protocol.AuthStatus{}, err } status.BrokerURL = cfg.brokerURL if status.Repo.Logical == "" && status.Repo.Prefix == "" { @@ -318,23 +282,23 @@ func brokerRequirePush(ctx context.Context, cfg config) error { } } -func readWhoamiCache(brokerURL string) (brokerAuthStatus, error) { +func readWhoamiCache(brokerURL string) (protocol.AuthStatus, error) { path, err := whoamiCachePath(brokerURL) if err != nil { - return brokerAuthStatus{}, err + return protocol.AuthStatus{}, err } data, err := os.ReadFile(path) if err != nil { - return brokerAuthStatus{}, err + return protocol.AuthStatus{}, err } - var status brokerAuthStatus + var status protocol.AuthStatus if err := json.Unmarshal(data, &status); err != nil { - return brokerAuthStatus{}, err + return protocol.AuthStatus{}, err } return status, nil } -func writeWhoamiCache(brokerURL string, status brokerAuthStatus) error { +func writeWhoamiCache(brokerURL string, status protocol.AuthStatus) error { path, err := whoamiCachePath(brokerURL) if err != nil { return err @@ -435,7 +399,7 @@ func repoAuthCachePath() (string, error) { func repoNameFromBrokerPayload(payload []byte) string { var raw struct { - Repo brokerRepo `json:"repo"` + Repo protocol.Repository `json:"repo"` } if err := json.Unmarshal(payload, &raw); err != nil { return "" diff --git a/auth_capabilities_test.go b/internal/app/auth_capabilities_test.go similarity index 89% rename from auth_capabilities_test.go rename to internal/app/auth_capabilities_test.go index 608ec18..eac0112 100644 --- a/auth_capabilities_test.go +++ b/internal/app/auth_capabilities_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" @@ -14,6 +14,8 @@ import ( "strings" "testing" + internalconfig "github.com/bucketgit/bgit/internal/config" + "github.com/bucketgit/bgit/protocol" "golang.org/x/crypto/ssh" ) @@ -24,10 +26,10 @@ func TestWhoamiCommandWritesGlobalCache(t *testing.T) { if r.URL.Path != "/auth/status" { t.Fatalf("path = %s", r.URL.Path) } - _ = json.NewEncoder(w).Encode(brokerAuthStatus{ + _ = json.NewEncoder(w).Encode(protocol.AuthStatus{ BrokerVersion: "1.0.0-dev", - Repo: brokerRepo{Provider: "gcs", Logical: "foo.git"}, - Identity: brokerIdentity{User: "dennis", KeyFingerprint: "SHA256:test"}, + Repo: protocol.Repository{Provider: "gcs", Logical: "foo.git"}, + Identity: protocol.Identity{User: "dennis", KeyFingerprint: "SHA256:test"}, User: "dennis", Role: "admin", Capabilities: map[string]bool{"read": true, "push": true, "admin_keys": true}, @@ -111,7 +113,7 @@ func TestPreferredBrokerKeyRankUsesConfiguredThenCachedKeys(t *testing.T) { } func TestRepoMembershipWarningsShowAmbiguousKeys(t *testing.T) { - warnings := repoMembershipWarnings([]brokerRepoMembership{ + warnings := repoMembershipWarnings([]protocol.RepositoryMembership{ {Logical: "foo.git", User: "dennis", Role: "admin", KeyFingerprint: "SHA256:a"}, {Logical: "foo.git", User: "dennis", Role: "read", KeyFingerprint: "SHA256:b"}, {Logical: "bar.git", User: "work", Role: "read", KeyFingerprint: "SHA256:c"}, @@ -130,12 +132,12 @@ func TestExplicitProfileSelectionAppliesToRepositoryDiscovery(t *testing.T) { home := t.TempDir() setTestHome(t, home) path := filepath.Join(home, ".bgit", "config.yaml") - if err := writeGlobalConfig(path, globalConfig{ + if err := writeGlobalConfig(path, internalconfig.Global{ Version: globalConfigVersion, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", ProjectID: "example-test-123456", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: "https://gcp.example.test", }}, diff --git a/internal/app/broker_assets.go b/internal/app/broker_assets.go new file mode 100644 index 0000000..2b6b98f --- /dev/null +++ b/internal/app/broker_assets.go @@ -0,0 +1,35 @@ +package app + +import ( + "io/fs" + + internalsetup "github.com/bucketgit/bgit/internal/setup" +) + +type assetFileSystem struct{ fs.FS } + +func (a assetFileSystem) ReadFile(name string) ([]byte, error) { + return fs.ReadFile(a.FS, name) +} + +var brokerAssets assetFileSystem + +func ConfigureBrokerAssets(assets fs.FS) { + brokerAssets = assetFileSystem{FS: assets} +} + +func writeGCPBrokerSource(dir string) error { + return internalsetup.WriteGCPBrokerSource(brokerAssets, "broker/gcp", dir, brokerVersion()) +} + +func writeGCPMaterializerSource(dir string) error { + return internalsetup.WriteGCPMaterializerSource(brokerAssets, "broker/gcp", dir, brokerVersion()) +} + +func awsBrokerCloudFormationTemplate() string { + template, err := internalsetup.AWSTemplate(brokerAssets, "broker/aws/template.yaml", brokerVersion()) + if err != nil { + return "" + } + return template +} diff --git a/broker_commands.go b/internal/app/broker_commands.go similarity index 82% rename from broker_commands.go rename to internal/app/broker_commands.go index ca39523..835d835 100644 --- a/broker_commands.go +++ b/internal/app/broker_commands.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -19,6 +19,8 @@ import ( "strings" "time" + internalconfig "github.com/bucketgit/bgit/internal/config" + "github.com/bucketgit/bgit/protocol" "golang.org/x/crypto/ssh" ) @@ -37,6 +39,7 @@ func brokerAdminCommand(cfg config, args []string, stdout io.Writer) error { func brokerAdminCommandWithInput(cfg config, args []string, stdin io.Reader, stdout io.Writer) error { if len(args) == 0 { + //lint:ignore ST1005 CLI usage errors intentionally include formatted guidance. return errors.New("usage: bgit admin keys|broker|broker-users|teams|repo create|repo|owner|protect|ci|members|confirm-ownership-transfer|accept-ownership-transfer|cancel-ownership-transfer|invite-user|accept-invite|cancel-invite|invite-broker-user|accept-broker-invite|cancel-broker-invite [args]\n\nCloud IAM administration moved to bgit direct admin.") } switch args[0] { @@ -162,7 +165,7 @@ func brokerAdminBrokerUpgradeCommand(cfg config, args []string, stdin io.Reader, return err } -func brokerProvisionResolvedTarget(path string, target brokerProfile, global *globalConfig, stdout io.Writer) (provisionedBroker, error) { +func brokerProvisionResolvedTarget(path string, target brokerProfile, global *internalconfig.Global, stdout io.Writer) (provisionedBroker, error) { cfg := config{ provider: target.Provider, gcloudConfiguration: target.Name, @@ -184,12 +187,12 @@ func brokerProvisionResolvedTarget(path string, target brokerProfile, global *gl if serviceAccount == "" && strings.TrimSpace(existing.ProjectID) != "" { serviceAccount = gcpBrokerServiceAccountEmail(existing.ProjectID) } - *global = upsertGlobalGCPProfile(*global, globalGCPProfile{ + *global = upsertGlobalGCPProfile(*global, internalconfig.GCPProfile{ Name: target.Name, ProjectID: existing.ProjectID, Account: existing.Account, ServiceAccount: serviceAccount, - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: target.Region, BrokerURL: provisioned.URL, BrokerVersion: brokerVersion(), @@ -198,11 +201,11 @@ func brokerProvisionResolvedTarget(path string, target brokerProfile, global *gl }) case "s3": existing := findGlobalAWSProfile(*global, target.Name) - *global = upsertGlobalAWSProfile(*global, globalAWSProfile{ + *global = upsertGlobalAWSProfile(*global, internalconfig.AWSProfile{ Name: target.Name, AccountID: existing.AccountID, ARN: existing.ARN, - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: target.Region, BrokerURL: provisioned.URL, BrokerVersion: brokerVersion(), @@ -219,25 +222,25 @@ func brokerProvisionResolvedTarget(path string, target brokerProfile, global *gl return provisioned, nil } -func findGlobalGCPProfile(cfg globalConfig, name string) globalGCPProfile { +func findGlobalGCPProfile(cfg internalconfig.Global, name string) internalconfig.GCPProfile { for _, profile := range cfg.GCPProfiles { if profile.Name == name { return profile } } - return globalGCPProfile{Name: name} + return internalconfig.GCPProfile{Name: name} } -func findGlobalAWSProfile(cfg globalConfig, name string) globalAWSProfile { +func findGlobalAWSProfile(cfg internalconfig.Global, name string) internalconfig.AWSProfile { for _, profile := range cfg.AWSProfiles { if profile.Name == name { return profile } } - return globalAWSProfile{Name: name} + return internalconfig.AWSProfile{Name: name} } -func brokerUpgradeTargetForCurrentRepo(cfg config, global globalConfig) (brokerProfile, error) { +func brokerUpgradeTargetForCurrentRepo(cfg config, global internalconfig.Global) (brokerProfile, error) { profiles := brokerProfilesFromGlobalConfig(global) if len(profiles) == 0 { return brokerProfile{}, errors.New("no configured brokers found; run bgit setup first") @@ -267,122 +270,33 @@ func brokerUpgradeTargetForCurrentRepo(cfg config, global globalConfig) (brokerP return brokerProfile{}, errors.New("current repository is not attached to a configured broker profile") } -type brokerRepoAdminRequest struct { - Repo brokerRepo `json:"repo"` - Description string `json:"description,omitempty"` - DefaultBranch string `json:"default_branch,omitempty"` - Visibility string `json:"visibility,omitempty"` - ReadOnly *bool `json:"read_only,omitempty"` - IssuesEnabled *bool `json:"issues_enabled,omitempty"` - Logical string `json:"logical,omitempty"` - TeamID string `json:"team_id,omitempty"` - Name string `json:"name,omitempty"` - UserID string `json:"user_id,omitempty"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - BrokerRole string `json:"broker_role,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` - Suspended bool `json:"suspended,omitempty"` - BrokerURL string `json:"broker_url,omitempty"` - Token string `json:"token,omitempty"` -} - -type brokerRepoListResponse struct { - Repos []brokerRepoInfo `json:"repos"` -} - -type brokerRepoInfo struct { - Repo brokerRepo `json:"repo"` - Logical string `json:"logical,omitempty"` - Teams []brokerRepoTeamGrant `json:"teams,omitempty"` -} - -type brokerAdminRepoInfoResponse struct { - Repo brokerRepo `json:"repo"` - Description string `json:"description,omitempty"` - DefaultBranch string `json:"default_branch,omitempty"` - Visibility string `json:"visibility,omitempty"` - ReadOnly bool `json:"read_only,omitempty"` - IssuesEnabled bool `json:"issues_enabled,omitempty"` -} - -type brokerRepoTeamsResponse struct { - Teams []brokerRepoTeamGrant `json:"teams"` -} - -type brokerRepoUsersResponse struct { - Users []brokerRepoUserGrant `json:"users"` -} - -type brokerRepoUserGrant struct { - UserID string `json:"user_id,omitempty"` - User string `json:"user,omitempty"` - Username string `json:"username,omitempty"` - Role string `json:"role,omitempty"` -} - -type brokerUsersResponse struct { - Users []brokerUserInfo `json:"users"` -} - -type brokerUserInfo struct { - ID string `json:"id"` - Username string `json:"username"` - BrokerRole string `json:"broker_role"` - Keys []brokerKey `json:"keys,omitempty"` - Suspended bool `json:"suspended,omitempty"` - Pending bool `json:"pending,omitempty"` -} - -type brokerRepoInvitesResponse struct { - Invites []brokerRepoInviteInfo `json:"invites"` -} - -type brokerRepoInviteInfo struct { - User string `json:"user"` - Role string `json:"role"` - ExpiresAt string `json:"expires_at"` -} - -type brokerTeamsResponse struct { - Teams []brokerTeamInfo `json:"teams"` -} - -type brokerTeamInfo struct { - ID string `json:"id"` - Name string `json:"name"` - Members []brokerTeamMember `json:"members,omitempty"` -} - -type brokerTeamMember struct { - UserID string `json:"user_id,omitempty"` - Username string `json:"username,omitempty"` - Role string `json:"role"` -} - func brokerUsersCommand(cfg config, args []string, stdout io.Writer) error { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return err } + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } if len(args) == 1 && args[0] == "list" { - var resp brokerUsersResponse - if err := brokerPost(brokerURL, "/broker/users/list", brokerRepoAdminRequest{}, &resp); err != nil { + users, err := endpoints.ListUsers(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { return err } - printBrokerUsers(stdout, resp.Users) + printBrokerUsers(stdout, users) return nil } if len(args) == 2 && args[0] == "delete" { - req := brokerRepoAdminRequest{User: args[1]} - if err := brokerPost(brokerURL, "/broker/users/delete", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{User: args[1]} + if err := endpoints.DeleteUser(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "deleted broker user %s\n", req.User) return nil } if len(args) >= 2 && args[0] == "upsert" { - req := brokerRepoAdminRequest{User: args[1], BrokerRole: "user"} + req := protocol.RepositoryAdminRequest{User: args[1], BrokerRole: "user"} for i := 2; i < len(args); i++ { name, value, hasValue := strings.Cut(args[i], "=") switch name { @@ -418,22 +332,20 @@ func brokerUsersCommand(cfg config, args []string, stdout io.Writer) error { return fmt.Errorf("unsupported broker-users upsert option %s", args[i]) } } - var resp struct { - User brokerUserInfo `json:"user"` - } if !validBrokerUserRole(req.BrokerRole) { return fmt.Errorf("invalid broker role %q", req.BrokerRole) } - if err := brokerPost(brokerURL, "/broker/users/upsert", req, &resp); err != nil { + user, err := endpoints.UpsertUser(context.Background(), req) + if err != nil { return err } - fmt.Fprintf(stdout, "upserted broker user %s as %s\n", resp.User.Username, resp.User.BrokerRole) + fmt.Fprintf(stdout, "upserted broker user %s as %s\n", user.Username, user.BrokerRole) return nil } return errors.New("usage: bgit admin broker-users list | upsert USER [--role admin|user] [--key PATH_OR_PUBLIC_KEY] [--suspended true|false] | delete USER") } -func printBrokerUsers(stdout io.Writer, users []brokerUserInfo) { +func printBrokerUsers(stdout io.Writer, users []protocol.UserInfo) { fmt.Fprintf(stdout, "%-18s %-28s %-8s %-9s\n", "ID", "Username", "Role", "Status") fmt.Fprintf(stdout, "%-18s %-28s %-8s %-9s\n", strings.Repeat("-", 2), strings.Repeat("-", 8), strings.Repeat("-", 4), strings.Repeat("-", 6)) sort.Slice(users, func(i, j int) bool { @@ -480,12 +392,16 @@ func brokerTeamsCommand(cfg config, args []string, stdout io.Writer) error { if err != nil { return err } + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } if len(args) == 1 && args[0] == "list" { - var resp brokerTeamsResponse - if err := brokerPost(brokerURL, "/teams/list", brokerRepoAdminRequest{}, &resp); err != nil { + teams, err := endpoints.ListTeams(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { return err } - for _, team := range resp.Teams { + for _, team := range teams { members := make([]string, 0, len(team.Members)) for _, member := range team.Members { members = append(members, firstNonEmpty(member.Username, member.UserID)+":"+member.Role) @@ -499,26 +415,24 @@ func brokerTeamsCommand(cfg config, args []string, stdout io.Writer) error { return nil } if len(args) >= 2 && args[0] == "create" { - req := brokerRepoAdminRequest{Name: args[1]} - var resp struct { - Team brokerTeamInfo `json:"team"` - } - if err := brokerPost(brokerURL, "/teams/create", req, &resp); err != nil { + req := protocol.RepositoryAdminRequest{Name: args[1]} + team, err := endpoints.CreateTeam(context.Background(), req) + if err != nil { return err } - fmt.Fprintf(stdout, "created team %s (%s)\n", resp.Team.Name, resp.Team.ID) + fmt.Fprintf(stdout, "created team %s (%s)\n", team.Name, team.ID) return nil } if len(args) == 2 && args[0] == "delete" { - req := brokerRepoAdminRequest{TeamID: args[1]} - if err := brokerPost(brokerURL, "/teams/delete", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{TeamID: args[1]} + if err := endpoints.DeleteTeam(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "deleted team %s\n", req.TeamID) return nil } if len(args) >= 4 && args[0] == "member" && args[1] == "add" { - req := brokerRepoAdminRequest{TeamID: args[2], User: args[3], Role: "read"} + req := protocol.RepositoryAdminRequest{TeamID: args[2], User: args[3], Role: "read"} for i := 4; i < len(args); i++ { name, value, hasValue := strings.Cut(args[i], "=") if name != "--role" { @@ -534,15 +448,15 @@ func brokerTeamsCommand(cfg config, args []string, stdout io.Writer) error { if !validRepoRole(req.Role) { return fmt.Errorf("invalid team member role %q", req.Role) } - if err := brokerPost(brokerURL, "/teams/member/upsert", req, nil); err != nil { + if err := endpoints.UpsertTeamMember(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "added %s to team %s as %s\n", req.User, req.TeamID, req.Role) return nil } if len(args) == 4 && args[0] == "member" && args[1] == "remove" { - req := brokerRepoAdminRequest{TeamID: args[2], User: args[3]} - if err := brokerPost(brokerURL, "/teams/member/remove", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{TeamID: args[2], User: args[3]} + if err := endpoints.RemoveTeamMember(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "removed %s from team %s\n", req.User, req.TeamID) @@ -553,11 +467,15 @@ func brokerTeamsCommand(cfg config, args []string, stdout io.Writer) error { if err != nil { return err } - var resp brokerRepoTeamsResponse - if err := brokerPost(cfg.brokerURL, "/repo/teams/list", brokerRepoAdminRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { + cfgEndpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } + teams, err := cfgEndpoints.ListRepositoryTeams(context.Background(), protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg)}) + if err != nil { return err } - for _, team := range resp.Teams { + for _, team := range teams { fmt.Fprintf(stdout, "%s\t%s\n", firstNonEmpty(team.ID, team.TeamID), team.Role) } return nil @@ -571,8 +489,12 @@ func brokerTeamsCommand(cfg config, args []string, stdout io.Writer) error { if !validRepoRole(role) { return fmt.Errorf("invalid repo team role %q", args[3]) } - req := brokerRepoAdminRequest{Repo: repoForBroker(cfg), TeamID: args[2], Role: role} - if err := brokerPost(cfg.brokerURL, "/repo/teams/upsert", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg), TeamID: args[2], Role: role} + cfgEndpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } + if err := cfgEndpoints.UpsertRepositoryTeam(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "attached team %s to %s as %s\n", req.TeamID, cfg.logicalRepo, req.Role) @@ -583,8 +505,12 @@ func brokerTeamsCommand(cfg config, args []string, stdout io.Writer) error { if err != nil { return err } - req := brokerRepoAdminRequest{Repo: repoForBroker(cfg), TeamID: args[2]} - if err := brokerPost(cfg.brokerURL, "/repo/teams/remove", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg), TeamID: args[2]} + cfgEndpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } + if err := cfgEndpoints.RemoveRepositoryTeam(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "detached team %s from %s\n", req.TeamID, cfg.logicalRepo) @@ -615,11 +541,15 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { if err != nil { return err } - var resp brokerRepoListResponse - if err := brokerPost(brokerURL, "/repos/list", brokerRepoAdminRequest{}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + repos, err := endpoints.ListRepositories(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { return err } - for _, repo := range resp.Repos { + for _, repo := range repos { teamIDs := make([]string, 0, len(repo.Teams)) for _, team := range repo.Teams { teamIDs = append(teamIDs, firstNonEmpty(team.ID, team.TeamID)) @@ -633,13 +563,17 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { if err != nil { return err } + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } switch args[0] { case "info": if len(args) != 1 { return errors.New("usage: bgit admin repo info") } - var resp brokerAdminRepoInfoResponse - if err := brokerPost(cfg.brokerURL, "/repo/info", brokerRepoAdminRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { + resp, err := endpoints.RepositoryInfo(context.Background(), protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg)}) + if err != nil { return err } fmt.Fprintf(stdout, "repository: %s\n", logicalRepoDisplayName(resp.Repo.Logical)) @@ -657,8 +591,8 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { if len(args) != 2 || (args[1] != "public" && args[1] != "private") { return errors.New("usage: bgit admin repo visibility public|private") } - req := brokerRepoAdminRequest{Repo: repoForBroker(cfg), Visibility: args[1]} - if err := brokerPost(cfg.brokerURL, "/repo/update", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg), Visibility: args[1]} + if err := endpoints.UpdateRepository(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "set repository visibility to %s\n", args[1]) @@ -668,8 +602,8 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { return errors.New("usage: bgit admin repo readonly on|off") } readOnly := args[1] == "on" - req := brokerRepoAdminRequest{Repo: repoForBroker(cfg), ReadOnly: &readOnly} - if err := brokerPost(cfg.brokerURL, "/repo/update", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg), ReadOnly: &readOnly} + if err := endpoints.UpdateRepository(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "set repository read-only to %t\n", readOnly) @@ -679,8 +613,8 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { return errors.New("usage: bgit admin repo issues on|off") } issuesEnabled := args[1] == "on" - req := brokerRepoAdminRequest{Repo: repoForBroker(cfg), IssuesEnabled: &issuesEnabled} - if err := brokerPost(cfg.brokerURL, "/repo/update", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg), IssuesEnabled: &issuesEnabled} + if err := endpoints.UpdateRepository(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "set repository issues to %t\n", issuesEnabled) @@ -693,7 +627,7 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/repo/rename", brokerRepoAdminRequest{Repo: repoForBroker(cfg), Logical: logical}, nil); err != nil { + if err := endpoints.RenameRepository(context.Background(), protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg), Logical: logical}); err != nil { return err } _, _ = runGit(".", "config", "--local", "bucketgit.logicalRepo", logical) @@ -704,7 +638,7 @@ func brokerAdminRepoCommand(cfg config, args []string, stdout io.Writer) error { if len(args) != 2 || args[1] != "--yes" { return errors.New("usage: bgit admin repo delete --yes") } - if err := brokerPost(cfg.brokerURL, "/repo/delete", brokerRepoAdminRequest{Repo: repoForBroker(cfg)}, nil); err != nil { + if err := endpoints.DeleteRepository(context.Background(), protocol.RepositoryAdminRequest{Repo: repoForBroker(cfg)}); err != nil { return err } fmt.Fprintln(stdout, "deleted repository") @@ -773,8 +707,12 @@ func brokerAdminRepoCreateCommand(cfg config, args []string, stdout io.Writer) e return err } repo.TeamName = team - req := brokerRepoAdminRequest{Repo: repo, Role: role} - if err := brokerPost(brokerURL, "/repos/create", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repo, Role: role} + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + if _, err := endpoints.CreateRepository(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "created repository %s in team %s\n", logicalRepoDisplayName(repo.Logical), team) @@ -812,7 +750,11 @@ func janitorMembersReindex(cfg config, stdout io.Writer) error { return err } } - if err := brokerPost(brokerURL, "/members/reindex", brokerKeyRequest{Repo: repoForBroker(cfg)}, nil); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + if err := endpoints.ReindexRepositoryMembers(context.Background(), protocol.KeyRequest{Repo: repoForBroker(cfg)}); err != nil { return err } fmt.Fprintln(stdout, "reindexed broker membership") @@ -1057,11 +999,11 @@ func brokerCloneWithEphemeralLocalBroker(opts brokerInitOptions, repoName string return err } defer managed.Close() - global = upsertGlobalLocalProfile(global, globalLocalProfile{ + global = upsertGlobalLocalProfile(global, internalconfig.LocalProfile{ Name: profile.Name, Root: profile.Root, Autostart: true, - Regions: []globalProfileRegion{region}, + Regions: []internalconfig.ProfileRegion{region}, }) if err := writeGlobalConfig(path, global); err != nil { return err @@ -1097,12 +1039,12 @@ func brokerCloneWithEphemeralLocalBroker(opts brokerInitOptions, repoName string return nil } -func startDefaultLocalBroker(global globalConfig) (globalLocalProfile, globalProfileRegion, *managedLocalBroker, error) { +func startDefaultLocalBroker(global internalconfig.Global) (internalconfig.LocalProfile, internalconfig.ProfileRegion, *managedLocalBroker, error) { profile := ensureDefaultLocalProfile(global) region := ensureDefaultLocalRegion(profile) managed, err := ensureManagedLocalBroker(context.Background(), profile, region) if err != nil { - return globalLocalProfile{}, globalProfileRegion{}, nil, err + return internalconfig.LocalProfile{}, internalconfig.ProfileRegion{}, nil, err } profile.Root = managed.Root region.BrokerURL = managed.URL @@ -1111,21 +1053,22 @@ func startDefaultLocalBroker(global globalConfig) (globalLocalProfile, globalPro return profile, region, managed, nil } -func prepareLocalBrokerRepository(brokerURL, bootstrapToken string, repo brokerRepo, keys []string) (brokerRepo, error) { +func prepareLocalBrokerRepository(brokerURL, bootstrapToken string, repo protocol.Repository, keys []string) (protocol.Repository, error) { if len(keys) == 0 { - return brokerRepo{}, errors.New("local broker requires at least one local SSH public key") + return protocol.Repository{}, errors.New("local broker requires at least one local SSH public key") } if err := brokerUpsertOwners(brokerURL, bootstrapToken, keys); err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } if err := brokerEnsureCoreTeam(brokerURL); err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } repo.TeamName = coreTeamName - var resp struct { - Repo brokerRepo `json:"repo"` + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return protocol.Repository{}, err } - err := brokerPost(brokerURL, "/repos/create", brokerRepoAdminRequest{Repo: repo, Role: "developer", User: "owner", PublicKeys: keys}, &resp) + resp, err := endpoints.CreateRepository(context.Background(), protocol.RepositoryAdminRequest{Repo: repo, Role: "developer", User: "owner", PublicKeys: keys}) if err == nil { if strings.TrimSpace(resp.Repo.Logical) != "" { return resp.Repo, nil @@ -1133,14 +1076,15 @@ func prepareLocalBrokerRepository(brokerURL, bootstrapToken string, repo brokerR return repo, nil } if !strings.Contains(err.Error(), "already exists") { - return brokerRepo{}, err + return protocol.Repository{}, err } - if getErr := brokerPost(brokerURL, "/repos/get", brokerRepoRequest{Repo: repo}, &resp); getErr != nil { - return brokerRepo{}, err + resp, getErr := endpoints.GetRepository(context.Background(), protocol.RepositoryRequest{Repo: repo}) + if getErr != nil { + return protocol.Repository{}, err } if strings.TrimSpace(resp.Repo.Logical) != "" { if !localBrokerStorageCompatible(repo, resp.Repo) { - return brokerRepo{}, fmt.Errorf("repository %s already exists with %s storage; requested %s storage", resp.Repo.Logical, localBrokerStorageDescription(resp.Repo), localBrokerStorageDescription(repo)) + return protocol.Repository{}, fmt.Errorf("repository %s already exists with %s storage; requested %s storage", resp.Repo.Logical, localBrokerStorageDescription(resp.Repo), localBrokerStorageDescription(repo)) } return resp.Repo, nil } @@ -1156,7 +1100,7 @@ func brokerInitWithEphemeralLocalBroker(opts brokerInitOptions, repoName string, return brokerInitWithLocalProfile(opts, repoName, profile, global, path, "", "", stdout) } -func brokerInitWithLocalProfile(opts brokerInitOptions, repoName string, profile brokerProfile, global globalConfig, path, identityName, identityEmail string, stdout io.Writer) error { +func brokerInitWithLocalProfile(opts brokerInitOptions, repoName string, profile brokerProfile, global internalconfig.Global, path, identityName, identityEmail string, stdout io.Writer) error { localProfile := ensureDefaultLocalProfile(global) if profile.Name != "" { localProfile.Name = profile.Name @@ -1174,11 +1118,11 @@ func brokerInitWithLocalProfile(opts brokerInitOptions, repoName string, profile localRegion.BrokerURL = managed.URL localRegion.BrokerVersion = brokerVersion() localRegion.LastSetupAt = time.Now().UTC().Format(time.RFC3339) - global = upsertGlobalLocalProfile(global, globalLocalProfile{ + global = upsertGlobalLocalProfile(global, internalconfig.LocalProfile{ Name: localProfile.Name, Root: localProfile.Root, Autostart: true, - Regions: []globalProfileRegion{localRegion}, + Regions: []internalconfig.ProfileRegion{localRegion}, }) if err := writeGlobalConfig(path, global); err != nil { return err @@ -1204,7 +1148,7 @@ func brokerInitWithLocalProfile(opts brokerInitOptions, repoName string, profile return initBrokerWorktreeWithLocalRepo(firstNonEmpty(opts.directory, "."), repo, initProfile, identityName, identityEmail, stdout) } -func ensureDefaultLocalProfile(global globalConfig) globalLocalProfile { +func ensureDefaultLocalProfile(global internalconfig.Global) internalconfig.LocalProfile { for _, profile := range global.LocalProfiles { if profile.Name == "default" { return profile @@ -1214,10 +1158,10 @@ func ensureDefaultLocalProfile(global globalConfig) globalLocalProfile { if err != nil { root = "~/.bgit/local-broker" } - return globalLocalProfile{Name: "default", Root: root, Autostart: true} + return internalconfig.LocalProfile{Name: "default", Root: root, Autostart: true} } -func ensureDefaultLocalRegion(profile globalLocalProfile) globalProfileRegion { +func ensureDefaultLocalRegion(profile internalconfig.LocalProfile) internalconfig.ProfileRegion { for _, region := range profile.Regions { if region.Name == "default" { return region @@ -1226,7 +1170,7 @@ func ensureDefaultLocalRegion(profile globalLocalProfile) globalProfileRegion { if len(profile.Regions) > 0 { return profile.Regions[0] } - return globalProfileRegion{Name: "default"} + return internalconfig.ProfileRegion{Name: "default"} } func defaultOwnerPublicKeys() []string { @@ -1244,7 +1188,7 @@ func defaultOwnerPublicKeys() []string { return uniqueStrings(out) } -func brokerRepoForAdminCreateTarget(cfg config, brokerURL, repoName, teamID string) (brokerRepo, error) { +func brokerRepoForAdminCreateTarget(cfg config, brokerURL, repoName, teamID string) (protocol.Repository, error) { if cfg.provider == "local" { local := cfg local.brokerURL = firstNonEmpty(strings.TrimSpace(local.brokerURL), strings.TrimSpace(brokerURL)) @@ -1253,7 +1197,7 @@ func brokerRepoForAdminCreateTarget(cfg config, brokerURL, repoName, teamID stri return brokerRepoForAdminTarget(cfg, repoName, teamID) } -func localBrokerRepoForTarget(cfg config, repoName, teamID string) (brokerRepo, error) { +func localBrokerRepoForTarget(cfg config, repoName, teamID string) (protocol.Repository, error) { raw := strings.TrimSpace(repoName) scheme, _, _, _, ok := storageTargetParts(raw) if !ok || scheme == "" { @@ -1261,7 +1205,7 @@ func localBrokerRepoForTarget(cfg config, repoName, teamID string) (brokerRepo, } logical, err := logicalRepoFromStorageTarget(raw) if err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } storageProvider := mapLocalStorageSchemeToProvider(scheme) storageProfile := "" @@ -1270,7 +1214,7 @@ func localBrokerRepoForTarget(cfg config, repoName, teamID string) (brokerRepo, if scheme == "s3" || scheme == "gs" { physical, storageProvider, storageProfile, storageRegion, err = deterministicLocalBrokerCloudStorageTarget(context.Background(), cfg, raw, logical) if err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } } local := cfg @@ -1321,7 +1265,7 @@ func localBrokerCloudIdentityFromConfig(ctx context.Context, scheme, profile str global, err := readGlobalConfig(path) if err != nil { if errors.Is(err, os.ErrNotExist) { - global = globalConfig{Version: globalConfigVersion} + global = internalconfig.Global{Version: globalConfigVersion} } else { return "", fmt.Errorf("read bgit config for local broker cloud identity: %w", err) } @@ -1354,7 +1298,7 @@ func localBrokerCloudIdentityFromConfig(ctx context.Context, scheme, profile str return "", fmt.Errorf("unsupported local broker cloud scheme %q", scheme) } -func localBrokerImportCloudIdentity(ctx context.Context, scheme, profile string, global globalConfig, path string) (string, error) { +func localBrokerImportCloudIdentity(ctx context.Context, scheme, profile string, global internalconfig.Global, path string) (string, error) { profile = firstNonEmpty(strings.TrimSpace(profile), "default") if global.Version == 0 { global.Version = globalConfigVersion @@ -1365,11 +1309,11 @@ func localBrokerImportCloudIdentity(ctx context.Context, scheme, profile string, if accountID == "" { return "", fmt.Errorf("AWS profile %q has no cached account id; configure or refresh it with `bgit setup profile create --provider aws %s`, or verify `aws sts get-caller-identity --profile %s`", profile, profile, profile) } - global = upsertGlobalAWSProfile(global, globalAWSProfile{ + global = upsertGlobalAWSProfile(global, internalconfig.AWSProfile{ Name: profile, AccountID: accountID, ARN: arn, - Regions: []globalProfileRegion{{Name: firstNonEmpty(configuredAWSProfileRegion(profile), "us-east-1")}}, + Regions: []internalconfig.ProfileRegion{{Name: firstNonEmpty(configuredAWSProfileRegion(profile), "us-east-1")}}, }) if err := writeGlobalConfig(path, global); err != nil { return "", err @@ -1383,11 +1327,11 @@ func localBrokerImportCloudIdentity(ctx context.Context, scheme, profile string, if projectID == "" { return "", fmt.Errorf("GCP profile %q has no cached project id; configure or refresh it with `bgit setup profile create --provider gcp %s`, run `gcloud config set project PROJECT --configuration %s`, or provide default ADC project credentials", profile, profile, profile) } - global = upsertGlobalGCPProfile(global, globalGCPProfile{ + global = upsertGlobalGCPProfile(global, internalconfig.GCPProfile{ Name: profile, ProjectID: projectID, Account: gcloudConfigValue(ctx, profile, "account"), - Regions: []globalProfileRegion{{Name: firstNonEmpty(gcloudConfigValue(ctx, profile, "run/region"), gcloudConfigValue(ctx, profile, "functions/region"), "us-central1")}}, + Regions: []internalconfig.ProfileRegion{{Name: firstNonEmpty(gcloudConfigValue(ctx, profile, "run/region"), gcloudConfigValue(ctx, profile, "functions/region"), "us-central1")}}, }) if err := writeGlobalConfig(path, global); err != nil { return "", err @@ -1447,7 +1391,7 @@ func mapLocalStorageSchemeToProvider(scheme string) string { } } -func localBrokerStorageCompatible(requested, existing brokerRepo) bool { +func localBrokerStorageCompatible(requested, existing protocol.Repository) bool { requested = normalizeLocalBrokerStorageRepo(requested) existing = normalizeLocalBrokerStorageRepo(existing) if requested.Provider == "" || existing.Provider == "" { @@ -1470,7 +1414,7 @@ func localBrokerStorageCompatible(requested, existing brokerRepo) bool { } } -func normalizeLocalBrokerStorageRepo(repo brokerRepo) brokerRepo { +func normalizeLocalBrokerStorageRepo(repo protocol.Repository) protocol.Repository { if cfg, ok, _ := localBrokerCloudConfig(repo.Bucket); ok { repo.Provider = cfg.provider repo.Bucket = cfg.bucket @@ -1493,7 +1437,7 @@ func localBrokerDefaultRegion(provider, region string) string { return "us-east-1" } -func localBrokerStorageDescription(repo brokerRepo) string { +func localBrokerStorageDescription(repo protocol.Repository) string { repo = normalizeLocalBrokerStorageRepo(repo) switch repo.Provider { case "s3": @@ -1556,11 +1500,11 @@ func brokerCloneWithLocalProfile(opts brokerInitOptions, repoName string, profil localRegion.BrokerURL = managed.URL localRegion.BrokerVersion = brokerVersion() localRegion.LastSetupAt = time.Now().UTC().Format(time.RFC3339) - global = upsertGlobalLocalProfile(global, globalLocalProfile{ + global = upsertGlobalLocalProfile(global, internalconfig.LocalProfile{ Name: localProfile.Name, Root: localProfile.Root, Autostart: true, - Regions: []globalProfileRegion{localRegion}, + Regions: []internalconfig.ProfileRegion{localRegion}, }) if err := writeGlobalConfig(path, global); err != nil { return err @@ -1569,11 +1513,13 @@ func brokerCloneWithLocalProfile(opts brokerInitOptions, repoName string, profil if err != nil { return err } - var resp struct { - Repo brokerRepo `json:"repo"` + req := protocol.RepositoryRequest{Repo: protocol.Repository{Logical: logical, TeamID: firstNonEmpty(profile.TeamID, coreTeamID)}} + endpoints, err := brokerEndpointClient(managed.URL) + if err != nil { + return err } - req := brokerRepoRequest{Repo: brokerRepo{Logical: logical, TeamID: firstNonEmpty(profile.TeamID, coreTeamID)}} - if err := brokerPost(managed.URL, "/repos/get", req, &resp); err != nil { + resp, err := endpoints.GetRepository(context.Background(), req) + if err != nil { return err } if strings.TrimSpace(resp.Repo.Logical) == "" { @@ -1754,29 +1700,12 @@ func configForBrokerCommand(base config) (config, error) { return cfg, nil } -type brokerOwnerTransferRequest struct { - Repo brokerRepo `json:"repo"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - BrokerURL string `json:"broker_url,omitempty"` - Token string `json:"token,omitempty"` -} - -type brokerOwnerTransferResponse struct { - Code string `json:"code"` - AcceptCommand string `json:"accept_command"` - CancelCommand string `json:"cancel_command"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` -} - type ownerTransferCodePayload struct { - BrokerURL string `json:"broker_url"` - Repo brokerRepo `json:"repo"` - Token string `json:"token"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` + BrokerURL string `json:"broker_url"` + Repo protocol.Repository `json:"repo"` + Token string `json:"token"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` } func brokerOwnerCommand(cfg config, args []string, stdout io.Writer) error { @@ -1806,9 +1735,13 @@ func brokerConfirmOwnershipTransferCommand(cfg config, args []string, stdout io. if err != nil { return err } - repo := brokerRepo{Provider: "gcs", Logical: logical, Origin: "git@" + defaultSSHHost + ":" + logical} - var resp brokerOwnerTransferResponse - if err := brokerPost(brokerURL, "/owners/transfer/confirm", brokerOwnerTransferRequest{Repo: repo, BrokerURL: brokerURL}, &resp); err != nil { + repo := protocol.Repository{Provider: "gcs", Logical: logical, Origin: "git@" + defaultSSHHost + ":" + logical} + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + resp, err := endpoints.ConfirmOwnerTransfer(context.Background(), protocol.OwnerTransferRequest{Repo: repo, BrokerURL: brokerURL}) + if err != nil { return err } fmt.Fprintf(stdout, "ownership transfer pending for %s\n\nCode:\n %s\n\nGive this command to the new owner:\n %s\n\nCancel with:\n %s\n", repo.Logical, resp.Code, resp.AcceptCommand, resp.CancelCommand) @@ -1823,8 +1756,12 @@ func brokerAcceptOwnershipTransferCommand(args []string, stdout io.Writer) error if err != nil { return err } - var resp brokerOwnerTransferResponse - if err := brokerPost(payload.BrokerURL, "/owners/transfer/accept", brokerOwnerTransferRequest{Repo: payload.Repo, Token: payload.Token, User: "owner"}, &resp); err != nil { + endpoints, err := brokerEndpointClient(payload.BrokerURL) + if err != nil { + return err + } + resp, err := endpoints.AcceptOwnerTransfer(context.Background(), protocol.OwnerTransferRequest{Repo: payload.Repo, Token: payload.Token, User: "owner"}) + if err != nil { return err } fmt.Fprintf(stdout, "accepted ownership for %s with key %s\n", payload.Repo.Logical, resp.Fingerprint) @@ -1840,8 +1777,12 @@ func brokerCancelOwnershipTransferCommand(cfg config, args []string, stdout io.W if err != nil { return err } - repo := brokerRepo{Provider: "gcs", Logical: logical, Origin: "git@" + defaultSSHHost + ":" + logical} - if err := brokerPost(brokerURL, "/owners/transfer/cancel", brokerOwnerTransferRequest{Repo: repo}, nil); err != nil { + repo := protocol.Repository{Provider: "gcs", Logical: logical, Origin: "git@" + defaultSSHHost + ":" + logical} + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + if err := endpoints.CancelOwnerTransfer(context.Background(), protocol.OwnerTransferRequest{Repo: repo}); err != nil { return err } fmt.Fprintf(stdout, "cancelled pending ownership transfer for %s\n", repo.Logical) @@ -1964,8 +1905,12 @@ func brokerInviteUserCommand(cfg config, args []string, stdout io.Writer) error if err != nil { return err } - var resp brokerOwnerTransferResponse - if err := brokerPost(brokerURL, "/keys/invite/create", brokerOwnerTransferRequest{Repo: repo, BrokerURL: brokerURL, User: user, Role: role}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + resp, err := endpoints.CreateRepositoryInvite(context.Background(), protocol.OwnerTransferRequest{Repo: repo, BrokerURL: brokerURL, User: user, Role: role}) + if err != nil { return err } fmt.Fprintf(stdout, "invite pending for %s as %s on %s\n\nCode:\n %s\n\nGive this command to the user:\n %s\n", user, role, repo.Logical, resp.Code, resp.AcceptCommand) @@ -1980,8 +1925,12 @@ func brokerAcceptInviteCommand(args []string, stdout io.Writer) error { if err != nil { return err } - var resp brokerOwnerTransferResponse - if err := brokerPost(payload.BrokerURL, "/keys/invite/accept", brokerOwnerTransferRequest{Repo: payload.Repo, Token: payload.Token}, &resp); err != nil { + endpoints, err := brokerEndpointClient(payload.BrokerURL) + if err != nil { + return err + } + resp, err := endpoints.AcceptRepositoryInvite(context.Background(), protocol.OwnerTransferRequest{Repo: payload.Repo, Token: payload.Token}) + if err != nil { return err } fmt.Fprintf(stdout, "accepted invite for %s as %s with key %s\n", resp.User, resp.Role, resp.Fingerprint) @@ -1997,7 +1946,11 @@ func brokerCancelInviteCommand(cfg config, args []string, stdout io.Writer) erro if err != nil { return err } - if err := brokerPost(brokerURL, "/keys/invite/cancel", brokerOwnerTransferRequest{Repo: repo, User: user}, nil); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + if err := endpoints.CancelRepositoryInvite(context.Background(), protocol.OwnerTransferRequest{Repo: repo, User: user}); err != nil { return err } fmt.Fprintf(stdout, "cancelled invite for %s on %s\n", user, repo.Logical) @@ -2054,8 +2007,12 @@ func brokerInviteBrokerUserCommand(cfg config, args []string, stdout io.Writer) if !validBrokerUserRole(role) { return fmt.Errorf("invalid broker role %q", role) } - var resp brokerOwnerTransferResponse - if err := brokerPost(brokerURL, "/broker/users/invite/create", brokerRepoAdminRequest{User: user, BrokerRole: role, BrokerURL: brokerURL}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + resp, err := endpoints.CreateBrokerUserInvite(context.Background(), protocol.RepositoryAdminRequest{User: user, BrokerRole: role, BrokerURL: brokerURL}) + if err != nil { return err } fmt.Fprintf(stdout, "broker user invite pending for %s as %s\n\nCode:\n %s\n\nGive this command to the user:\n %s\n", user, role, resp.Code, resp.AcceptCommand) @@ -2070,8 +2027,12 @@ func brokerAcceptBrokerInviteCommand(args []string, stdout io.Writer) error { if err != nil { return err } - var resp brokerOwnerTransferResponse - if err := brokerPost(payload.BrokerURL, "/broker/users/invite/accept", brokerRepoAdminRequest{User: payload.User, Token: payload.Token}, &resp); err != nil { + endpoints, err := brokerEndpointClient(payload.BrokerURL) + if err != nil { + return err + } + resp, err := endpoints.AcceptBrokerUserInvite(context.Background(), protocol.RepositoryAdminRequest{User: payload.User, Token: payload.Token}) + if err != nil { if brokerInviteAcceptNeedsSSHIdentity(err) { return errors.New("accepting a broker invite requires an SSH signature; load a key with `ssh-add`, pass `--identity PATH`, or set BGIT_SSH_KEY") } @@ -2128,7 +2089,11 @@ func brokerCancelBrokerInviteCommand(cfg config, args []string, stdout io.Writer if user == "" { return errors.New("usage: bgit admin cancel-broker-invite --broker URL --user USER") } - if err := brokerPost(brokerURL, "/broker/users/invite/cancel", brokerRepoAdminRequest{User: user}, nil); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + if err := endpoints.CancelBrokerUserInvite(context.Background(), protocol.RepositoryAdminRequest{User: user}); err != nil { return err } fmt.Fprintf(stdout, "cancelled broker invite for %s\n", user) @@ -2189,13 +2154,13 @@ func parseCancelInviteTarget(cfg config, args []string) (string, string, string, return brokerURL, repoName, teamID, user, nil } -func brokerRepoForAdminTarget(cfg config, repoName, teamID string) (brokerRepo, error) { +func brokerRepoForAdminTarget(cfg config, repoName, teamID string) (protocol.Repository, error) { if repo, ok, err := brokerRepoForStorageTarget(cfg, repoName, teamID); ok || err != nil { return repo, err } logical, err := normalizeLogicalRepoName(repoName) if err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } local := cfg local.logicalRepo = logical @@ -2208,7 +2173,7 @@ func brokerRepoForAdminTarget(cfg config, repoName, teamID string) (brokerRepo, return repoForBroker(local), nil } -func brokerRepoForStorageTarget(cfg config, repoName, teamID string) (brokerRepo, bool, error) { +func brokerRepoForStorageTarget(cfg config, repoName, teamID string) (protocol.Repository, bool, error) { raw := strings.TrimSpace(repoName) scheme := "" switch { @@ -2219,11 +2184,11 @@ func brokerRepoForStorageTarget(cfg config, repoName, teamID string) (brokerRepo case strings.HasPrefix(raw, "file://"): scheme = "file" default: - return brokerRepo{}, false, nil + return protocol.Repository{}, false, nil } logical, err := logicalRepoFromStorageTarget(raw) if err != nil { - return brokerRepo{}, true, err + return protocol.Repository{}, true, err } local := cfg local.logicalRepo = logical @@ -2383,13 +2348,6 @@ func parseBrokerUserInviteCode(code string) (ownerTransferCodePayload, error) { return payload, nil } -type brokerProtectionRequest struct { - Repo brokerRepo `json:"repo"` - Ref string `json:"ref"` - RequirePR bool `json:"require_pr"` - AllowOverrides bool `json:"allow_overrides"` -} - func brokerProtectionCommand(cfg config, args []string, stdout io.Writer) error { if len(args) == 0 { return errors.New("usage: bgit admin protect add|list|remove [ref]") @@ -2398,15 +2356,17 @@ func brokerProtectionCommand(cfg config, args []string, stdout io.Writer) error if err != nil { return err } + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } switch args[0] { case "list": - var resp struct { - Protections []brokerProtectionRequest `json:"protections"` - } - if err := brokerPost(cfg.brokerURL, "/protection/list", brokerProtectionRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { + protections, err := endpoints.ListProtections(context.Background(), protocol.Protection{Repo: repoForBroker(cfg)}) + if err != nil { return err } - for _, protection := range resp.Protections { + for _, protection := range protections { mode := "pr-required" if protection.AllowOverrides { mode += ",owner-admin-override" @@ -2428,8 +2388,8 @@ func brokerProtectionCommand(cfg config, args []string, stdout io.Writer) error ref = normalizeDestinationRef(arg) } } - req := brokerProtectionRequest{Repo: repoForBroker(cfg), Ref: ref, RequirePR: true, AllowOverrides: allowOverrides} - if err := brokerPost(cfg.brokerURL, "/protection/upsert", req, nil); err != nil { + req := protocol.Protection{Repo: repoForBroker(cfg), Ref: ref, RequirePR: true, AllowOverrides: allowOverrides} + if err := endpoints.UpsertProtection(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "protected %s\n", ref) @@ -2438,8 +2398,8 @@ func brokerProtectionCommand(cfg config, args []string, stdout io.Writer) error if len(args) != 2 { return errors.New("usage: bgit admin protect remove ") } - req := brokerProtectionRequest{Repo: repoForBroker(cfg), Ref: normalizeDestinationRef(args[1])} - if err := brokerPost(cfg.brokerURL, "/protection/remove", req, nil); err != nil { + req := protocol.Protection{Repo: repoForBroker(cfg), Ref: normalizeDestinationRef(args[1])} + if err := endpoints.RemoveProtection(context.Background(), req); err != nil { return err } fmt.Fprintf(stdout, "removed protection for %s\n", req.Ref) @@ -2449,104 +2409,6 @@ func brokerProtectionCommand(cfg config, args []string, stdout io.Writer) error } } -type brokerPullRequest struct { - ID int `json:"id,omitempty"` - Title string `json:"title,omitempty"` - Body string `json:"body,omitempty"` - Source string `json:"source,omitempty"` - Target string `json:"target,omitempty"` - Status string `json:"status,omitempty"` - Author string `json:"author,omitempty"` - Version string `json:"version,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` - Approvals int `json:"approvals,omitempty"` - Checks []string `json:"checks,omitempty"` - Head string `json:"head,omitempty"` - Comments []brokerPullRequestNote `json:"comments,omitempty"` - Reviews []brokerPullRequestNote `json:"reviews,omitempty"` - MergedBy string `json:"merged_by,omitempty"` - MergedAt string `json:"merged_at,omitempty"` - ClosedBy string `json:"closed_by,omitempty"` - ClosedAt string `json:"closed_at,omitempty"` -} - -type brokerPullRequestNote struct { - ID int `json:"id,omitempty"` - User string `json:"user,omitempty"` - Body string `json:"body,omitempty"` - State string `json:"state,omitempty"` - Source string `json:"source,omitempty"` - At string `json:"at,omitempty"` - Comments []brokerPullRequestComment `json:"comments,omitempty"` - Replies []brokerPullRequestComment `json:"replies,omitempty"` - Head string `json:"head,omitempty"` -} - -type brokerPullRequestComment struct { - ID int `json:"id,omitempty"` - User string `json:"user,omitempty"` - Body string `json:"body,omitempty"` - File string `json:"file,omitempty"` - Kind string `json:"kind,omitempty"` - Side string `json:"side,omitempty"` - Hunk string `json:"hunk,omitempty"` - HunkIndex int `json:"hunk_index,omitempty"` - OldStart int `json:"old_start,omitempty"` - NewStart int `json:"new_start,omitempty"` - Offset int `json:"offset,omitempty"` - Line int `json:"line,omitempty"` - LineText string `json:"line_text,omitempty"` - LineHash string `json:"line_hash,omitempty"` - Head string `json:"head,omitempty"` - Outdated bool `json:"outdated,omitempty"` - At string `json:"at,omitempty"` - Replies []brokerPullRequestComment `json:"replies,omitempty"` -} - -type brokerPullRequestRequest struct { - Repo brokerRepo `json:"repo"` - ID int `json:"id,omitempty"` - PR brokerPullRequest `json:"pr,omitempty"` - Known map[string]string `json:"known,omitempty"` - Merge bool `json:"merge,omitempty"` - DeleteBranch bool `json:"delete_branch,omitempty"` - Comment string `json:"comment,omitempty"` - Review string `json:"review,omitempty"` - Comments []brokerPullRequestComment `json:"comments,omitempty"` - TargetNoteID int `json:"target_note_id,omitempty"` - TargetCommentID int `json:"target_comment_id,omitempty"` -} - -type brokerCIRun struct { - ID int `json:"id,omitempty"` - Provider string `json:"provider,omitempty"` - Ref string `json:"ref,omitempty"` - Commit string `json:"commit,omitempty"` - Config string `json:"config,omitempty"` - Status string `json:"status,omitempty"` - Result string `json:"result,omitempty"` - URL string `json:"url,omitempty"` - Message string `json:"message,omitempty"` - ProviderBuildID string `json:"provider_build_id,omitempty"` - ProviderBuildName string `json:"provider_build_name,omitempty"` - LogGroup string `json:"log_group,omitempty"` - LogStream string `json:"log_stream,omitempty"` - Author string `json:"author,omitempty"` - StartedAt string `json:"started_at,omitempty"` - FinishedAt string `json:"finished_at,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` -} - -type brokerCIRequest struct { - Repo brokerRepo `json:"repo"` - ID int `json:"id,omitempty"` - Provider string `json:"provider,omitempty"` - Ref string `json:"ref,omitempty"` - Commit string `json:"commit,omitempty"` - Config string `json:"config,omitempty"` -} - func brokerAdminCICommand(cfg config, args []string, stdout io.Writer) error { if len(args) == 0 { return errors.New("usage: bgit admin ci rotate-secret") @@ -2560,7 +2422,11 @@ func brokerAdminCICommand(cfg config, args []string, stdout io.Writer) error { if len(args) != 1 { return errors.New("usage: bgit admin ci rotate-secret") } - if err := brokerPost(cfg.brokerURL, "/ci/secret/rotate", brokerCIRequest{Repo: repoForBroker(cfg)}, nil); err != nil { + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } + if err := endpoints.RotateCISecret(context.Background(), protocol.CIRequest{Repo: repoForBroker(cfg)}); err != nil { return err } fmt.Fprintln(stdout, "rotated CI materializer secret") @@ -2579,15 +2445,17 @@ func ciCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } switch args[0] { case "list": - var resp struct { - Runs []brokerCIRun `json:"runs"` - } - if err := brokerPost(cfg.brokerURL, "/ci/list", brokerCIRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { + runs, err := endpoints.ListCIRuns(context.Background(), protocol.CIRequest{Repo: repoForBroker(cfg)}) + if err != nil { return err } - for _, run := range resp.Runs { + for _, run := range runs { fmt.Fprintf(stdout, "#%d\t%s\t%s\t%s\t%s\n", run.ID, firstNonEmpty(run.Status, "queued"), shortRefName(run.Ref), shortHash(run.Commit), run.Config) } return nil @@ -2599,18 +2467,16 @@ func ciCommand(args []string, stdin io.Reader, stdout io.Writer) error { if id <= 0 { return errors.New("CI run id is required") } - var resp struct { - Run brokerCIRun `json:"run"` - } - if err := brokerPost(cfg.brokerURL, "/ci/view", brokerCIRequest{Repo: repoForBroker(cfg), ID: id}, &resp); err != nil { + run, err := endpoints.GetCIRun(context.Background(), protocol.CIRequest{Repo: repoForBroker(cfg), ID: id}) + if err != nil { return err } - fmt.Fprintf(stdout, "#%d %s\nprovider: %s\nref: %s\ncommit: %s\nconfig: %s\n", resp.Run.ID, firstNonEmpty(resp.Run.Status, "queued"), resp.Run.Provider, resp.Run.Ref, resp.Run.Commit, resp.Run.Config) - if resp.Run.URL != "" { - fmt.Fprintf(stdout, "url: %s\n", resp.Run.URL) + fmt.Fprintf(stdout, "#%d %s\nprovider: %s\nref: %s\ncommit: %s\nconfig: %s\n", run.ID, firstNonEmpty(run.Status, "queued"), run.Provider, run.Ref, run.Commit, run.Config) + if run.URL != "" { + fmt.Fprintf(stdout, "url: %s\n", run.URL) } - if resp.Run.Message != "" { - fmt.Fprintf(stdout, "\n%s\n", resp.Run.Message) + if run.Message != "" { + fmt.Fprintf(stdout, "\n%s\n", run.Message) } return nil case "logs": @@ -2622,12 +2488,14 @@ func ciCommand(args []string, stdin io.Reader, stdout io.Writer) error { return errors.New("CI run id is required") } var resp struct { - Run brokerCIRun `json:"run"` - Logs string `json:"logs"` + Run protocol.CIRun `json:"run"` + Logs string `json:"logs"` } - if err := brokerPost(cfg.brokerURL, "/ci/logs", brokerCIRequest{Repo: repoForBroker(cfg), ID: id}, &resp); err != nil { + logs, err := endpoints.CILogs(context.Background(), protocol.CIRequest{Repo: repoForBroker(cfg), ID: id}) + if err != nil { return err } + resp.Run, resp.Logs = logs.Run, logs.Logs if resp.Logs != "" { fmt.Fprint(stdout, resp.Logs) if !strings.HasSuffix(resp.Logs, "\n") { @@ -2645,7 +2513,7 @@ func ciCommand(args []string, stdin io.Reader, stdout io.Writer) error { } return watchCIRun(cfg, id, stdout) case "run": - req := brokerCIRequest{Repo: repoForBroker(cfg), Provider: defaultCIProvider(cfg)} + req := protocol.CIRequest{Repo: repoForBroker(cfg), Provider: defaultCIProvider(cfg)} for i := 1; i < len(args); i++ { arg := args[i] name, value, hasValue := strings.Cut(arg, "=") @@ -2689,15 +2557,13 @@ func ciCommand(args []string, stdin io.Reader, stdout io.Writer) error { return err } } - var resp struct { - Run brokerCIRun `json:"run"` - } - if err := brokerPost(cfg.brokerURL, "/ci/run", req, &resp); err != nil { + run, err := endpoints.RunCI(context.Background(), req) + if err != nil { return err } - fmt.Fprintf(stdout, "queued CI run #%d for %s at %s\n", resp.Run.ID, shortRefName(resp.Run.Ref), shortHash(resp.Run.Commit)) - if resp.Run.Message != "" { - fmt.Fprintf(stdout, "%s\n", resp.Run.Message) + fmt.Fprintf(stdout, "queued CI run #%d for %s at %s\n", run.ID, shortRefName(run.Ref), shortHash(run.Commit)) + if run.Message != "" { + fmt.Fprintf(stdout, "%s\n", run.Message) } return nil default: @@ -2706,16 +2572,22 @@ func ciCommand(args []string, stdin io.Reader, stdout io.Writer) error { } func watchCIRun(cfg config, id int, stdout io.Writer) error { + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } printed := 0 lastStatus := "" for { var resp struct { - Run brokerCIRun `json:"run"` - Logs string `json:"logs"` + Run protocol.CIRun `json:"run"` + Logs string `json:"logs"` } - if err := brokerPost(cfg.brokerURL, "/ci/logs", brokerCIRequest{Repo: repoForBroker(cfg), ID: id}, &resp); err != nil { + logs, err := endpoints.CILogs(context.Background(), protocol.CIRequest{Repo: repoForBroker(cfg), ID: id}) + if err != nil { return err } + resp.Run, resp.Logs = logs.Run, logs.Logs status := firstNonEmpty(resp.Run.Status, "queued") if status != lastStatus { fmt.Fprintf(stdout, "#%d %s\n", resp.Run.ID, status) @@ -2786,15 +2658,17 @@ func issueCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } switch args[0] { case "list": - var resp struct { - Issues []brokerIssue `json:"issues"` - } - if err := brokerPost(cfg.brokerURL, "/issues/list", brokerIssueRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { + issues, err := endpoints.ListIssues(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg)}) + if err != nil { return err } - for _, issue := range resp.Issues { + for _, issue := range issues { fmt.Fprintf(stdout, "#%d\t%s\t%s\n", issue.ID, firstNonEmpty(issue.Status, "open"), issue.Title) } return nil @@ -2824,27 +2698,23 @@ func issueCommand(args []string, stdin io.Reader, stdout io.Writer) error { if strings.TrimSpace(title) == "" { return errors.New("usage: bgit issue create TITLE [--body BODY]") } - var resp struct { - Issue brokerIssue `json:"issue"` - } - if err := brokerPost(cfg.brokerURL, "/issues/create", brokerIssueRequest{Repo: repoForBroker(cfg), Title: title, Body: body}, &resp); err != nil { + issue, err := endpoints.CreateIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), Title: title, Body: body}) + if err != nil { return err } - fmt.Fprintf(stdout, "created issue #%d\n", resp.Issue.ID) + fmt.Fprintf(stdout, "created issue #%d\n", issue.ID) return nil case "view": id, err := parseIssueIDArg(args) if err != nil { return err } - var resp struct { - Issue brokerIssue `json:"issue"` - } - if err := brokerPost(cfg.brokerURL, "/issues/view", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id}, &resp); err != nil { + issue, err := endpoints.GetIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id}) + if err != nil { return err } - fmt.Fprintf(stdout, "#%d %s\n%s\n\n%s\n", resp.Issue.ID, resp.Issue.Title, firstNonEmpty(resp.Issue.Status, "open"), resp.Issue.Body) - for _, comment := range resp.Issue.Comments { + fmt.Fprintf(stdout, "#%d %s\n%s\n\n%s\n", issue.ID, issue.Title, firstNonEmpty(issue.Status, "open"), issue.Body) + for _, comment := range issue.Comments { fmt.Fprintf(stdout, "\n%s commented:\n%s\n", firstNonEmpty(comment.User, "anonymous"), comment.Body) } return nil @@ -2857,7 +2727,7 @@ func issueCommand(args []string, stdin io.Reader, stdout io.Writer) error { return errors.New("issue id is required") } comment := strings.Join(args[2:], " ") - if err := brokerPost(cfg.brokerURL, "/issues/comment", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Comment: comment}, nil); err != nil { + if err := endpoints.CommentIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Comment: comment}); err != nil { return err } fmt.Fprintf(stdout, "commented on issue #%d\n", id) @@ -2867,8 +2737,15 @@ func issueCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/issues/"+args[0], brokerIssueRequest{Repo: repoForBroker(cfg), ID: id}, nil); err != nil { - return err + request := protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id} + var actionErr error + if args[0] == "reopen" { + actionErr = endpoints.ReopenIssue(context.Background(), request) + } else { + actionErr = endpoints.CloseIssue(context.Background(), request) + } + if actionErr != nil { + return actionErr } verb := "closed" if args[0] == "reopen" { @@ -2890,6 +2767,10 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } monogram := repoMonogram(cfg.logicalRepo) switch args[0] { case "list": @@ -2902,16 +2783,14 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { return fmt.Errorf("unsupported board list option %s", arg) } } - var resp struct { - Issues []brokerIssue `json:"issues"` - } - if err := brokerPost(cfg.brokerURL, "/issues/list", brokerIssueRequest{Repo: repoForBroker(cfg), Type: "story", IncludeArchived: includeArchived}, &resp); err != nil { + issues, err := endpoints.ListIssues(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), Type: "story", IncludeArchived: includeArchived}) + if err != nil { return err } - sortBoardStories(resp.Issues) + sortBoardStories(issues) if includeArchived { fmt.Fprintf(stdout, "archived\n") - for _, issue := range resp.Issues { + for _, issue := range issues { if issue.Type != "story" || !issue.Archived { continue } @@ -2925,7 +2804,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { } for _, lane := range kanbanLanes() { fmt.Fprintf(stdout, "%s\n", lane) - for _, issue := range resp.Issues { + for _, issue := range issues { if issue.Type != "story" || issue.Archived || normalizeKanbanLane(issue.Lane) != lane { continue } @@ -2962,13 +2841,11 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if strings.TrimSpace(story) == "" { return errors.New("usage: bgit board create STORY") } - var resp struct { - Issue brokerIssue `json:"issue"` - } - if err := brokerPost(cfg.brokerURL, "/issues/create", brokerIssueRequest{Repo: repoForBroker(cfg), Type: "story", Title: storySummary(story), Body: story, Lane: "backlog"}, &resp); err != nil { + issue, err := endpoints.CreateIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), Type: "story", Title: storySummary(story), Body: story, Lane: "backlog"}) + if err != nil { return err } - fmt.Fprintf(stdout, "created story %s\n", storyDisplayID(monogram, resp.Issue.ID)) + fmt.Fprintf(stdout, "created story %s\n", storyDisplayID(monogram, issue.ID)) return nil case "edit": if len(args) < 3 { @@ -2982,7 +2859,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if strings.TrimSpace(story) == "" { return errors.New("story is required") } - if err := brokerPost(cfg.brokerURL, "/issues/update", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Type: "story", Title: storySummary(story), Body: story}, nil); err != nil { + if err := endpoints.UpdateIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Type: "story", Title: storySummary(story), Body: story}); err != nil { return boardUpgradeError(err) } fmt.Fprintf(stdout, "edited story %s\n", storyDisplayID(monogram, id)) @@ -2999,7 +2876,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/issues/move", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Lane: lane}, nil); err != nil { + if err := endpoints.MoveIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Lane: lane}); err != nil { return err } fmt.Fprintf(stdout, "moved story %s to %s\n", storyDisplayID(monogram, id), lane) @@ -3009,7 +2886,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/issues/reorder", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Lane: lane, Order: order}, nil); err != nil { + if err := endpoints.ReorderIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Lane: lane, Order: order}); err != nil { return boardUpgradeError(err) } if lane == "" { @@ -3023,7 +2900,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/issues/take", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id}, nil); err != nil { + if err := endpoints.TakeIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id}); err != nil { return err } fmt.Fprintf(stdout, "took story %s\n", storyDisplayID(monogram, id)) @@ -3040,7 +2917,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { if strings.EqualFold(assignee, "unassigned") || strings.EqualFold(assignee, "none") || assignee == "-" { assignee = "" } - if err := brokerPost(cfg.brokerURL, "/issues/assign", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Assignee: assignee}, nil); err != nil { + if err := endpoints.AssignIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Assignee: assignee}); err != nil { return err } if assignee == "" { @@ -3055,7 +2932,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { return err } archived := args[0] == "archive" - if err := brokerPost(cfg.brokerURL, "/issues/archive", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Archived: archived}, nil); err != nil { + if err := endpoints.ArchiveIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Archived: archived}); err != nil { return boardUpgradeError(err) } if archived { @@ -3073,7 +2950,7 @@ func boardCommand(args []string, stdin io.Reader, stdout io.Writer) error { return err } comment := strings.Join(args[2:], " ") - if err := brokerPost(cfg.brokerURL, "/issues/comment", brokerIssueRequest{Repo: repoForBroker(cfg), ID: id, Comment: comment}, nil); err != nil { + if err := endpoints.CommentIssue(context.Background(), protocol.IssueRequest{Repo: repoForBroker(cfg), ID: id, Comment: comment}); err != nil { return err } fmt.Fprintf(stdout, "commented on story %s\n", storyDisplayID(monogram, id)) @@ -3133,7 +3010,7 @@ func boardUpgradeError(err error) error { return err } -func sortBoardStories(stories []brokerIssue) { +func sortBoardStories(stories []protocol.Issue) { sort.SliceStable(stories, func(i, j int) bool { leftLane := normalizeKanbanLane(stories[i].Lane) rightLane := normalizeKanbanLane(stories[j].Lane) @@ -3213,17 +3090,19 @@ func prCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err + } switch args[0] { case "create": return prCreateCommand(cfg, args[1:], stdout) case "list": - var resp struct { - PRs []brokerPullRequest `json:"prs"` - } - if err := brokerPost(cfg.brokerURL, "/prs/list", brokerPullRequestRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { + prs, err := endpoints.ListPullRequests(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg)}) + if err != nil { return err } - for _, pr := range resp.PRs { + for _, pr := range prs { fmt.Fprintf(stdout, "#%d\t%s\t%s -> %s\t%s\n", pr.ID, pr.Status, pr.Source, pr.Target, pr.Title) } return nil @@ -3242,7 +3121,7 @@ func prCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/prs/close", brokerPullRequestRequest{Repo: repoForBroker(cfg), ID: id}, nil); err != nil { + if _, err := endpoints.ClosePullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), ID: id}); err != nil { return err } fmt.Fprintf(stdout, "closed PR #%d\n", id) @@ -3252,7 +3131,7 @@ func prCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/prs/reopen", brokerPullRequestRequest{Repo: repoForBroker(cfg), ID: id}, nil); err != nil { + if _, err := endpoints.ReopenPullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), ID: id}); err != nil { return err } fmt.Fprintf(stdout, "reopened PR #%d\n", id) @@ -3272,7 +3151,7 @@ func prCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/prs/merge", brokerPullRequestRequest{Repo: repoForBroker(cfg), ID: id, Merge: true, DeleteBranch: deleteBranch}, nil); err != nil { + if _, err := endpoints.MergePullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), ID: id, Merge: true, DeleteBranch: deleteBranch}); err != nil { return err } fmt.Fprintf(stdout, "merged PR #%d\n", id) @@ -3282,7 +3161,7 @@ func prCommand(args []string, stdin io.Reader, stdout io.Writer) error { if err != nil { return err } - if err := brokerPost(cfg.brokerURL, "/prs/comment", brokerPullRequestRequest{Repo: repoForBroker(cfg), ID: id, Comment: comment}, nil); err != nil { + if _, err := endpoints.CommentPullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), ID: id, Comment: comment}); err != nil { return err } fmt.Fprintf(stdout, "commented on PR #%d\n", id) @@ -3298,7 +3177,7 @@ func prCommand(args []string, stdin io.Reader, stdout io.Writer) error { review = "changes_requested" verb = "requested changes on" } - if err := brokerPost(cfg.brokerURL, "/prs/review", brokerPullRequestRequest{Repo: repoForBroker(cfg), ID: id, Review: review, Comment: comment}, nil); err != nil { + if _, err := endpoints.ReviewPullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), ID: id, Review: review, Comment: comment}); err != nil { return err } fmt.Fprintf(stdout, "%s PR #%d\n", verb, id) @@ -3361,7 +3240,7 @@ func parsePRIDAndOptionalTextArg(args []string) (int, string, error) { } func prCreateCommand(cfg config, args []string, stdout io.Writer) error { - pr := brokerPullRequest{Target: "refs/heads/main"} + pr := protocol.PullRequest{Target: "refs/heads/main"} for i := 0; i < len(args); i++ { arg := args[i] name, value, hasValue := strings.Cut(arg, "=") @@ -3408,28 +3287,32 @@ func prCreateCommand(cfg config, args []string, stdout io.Writer) error { if pr.Title == "" { pr.Title = shortRefName(pr.Source) + " into " + shortRefName(pr.Target) } - var resp struct { - PR brokerPullRequest `json:"pr"` + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return err } - if err := brokerPost(cfg.brokerURL, "/prs/create", brokerPullRequestRequest{Repo: repoForBroker(cfg), PR: pr}, &resp); err != nil { + created, err := endpoints.CreatePullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), PR: pr}) + if err != nil { return err } - fmt.Fprintf(stdout, "created PR #%d %s\n", resp.PR.ID, resp.PR.Title) + fmt.Fprintf(stdout, "created PR #%d %s\n", created.ID, created.Title) return nil } -func brokerGetPullRequest(cfg config, args []string) (brokerPullRequest, error) { +func brokerGetPullRequest(cfg config, args []string) (protocol.PullRequest, error) { id, err := parsePRIDArg(args) if err != nil { - return brokerPullRequest{}, err + return protocol.PullRequest{}, err } - var resp struct { - PR brokerPullRequest `json:"pr"` + endpoints, err := brokerEndpointClient(cfg.brokerURL) + if err != nil { + return protocol.PullRequest{}, err } - if err := brokerPost(cfg.brokerURL, "/prs/view", brokerPullRequestRequest{Repo: repoForBroker(cfg), ID: id}, &resp); err != nil { - return brokerPullRequest{}, err + pr, err := endpoints.GetPullRequest(context.Background(), protocol.PullRequestRequest{Repo: repoForBroker(cfg), ID: id}) + if err != nil { + return protocol.PullRequest{}, err } - return resp.PR, nil + return pr, nil } func parsePRIDArg(args []string) (int, error) { @@ -3623,11 +3506,6 @@ func discoverBrokerCloneURL(raw string) (string, string, string, bool, error) { return "", "", "", false, nil } -func brokerURLFromTXTRecord(record string) string { - broker, _ := brokerDiscoveryFromTXTRecord(record, "") - return broker -} - func brokerDiscoveryFromTXTRecord(record, teamName string) (string, string) { fields := strings.Fields(strings.TrimSpace(record)) if len(fields) == 0 || fields[0] != "v=bgit1" { @@ -3664,16 +3542,18 @@ func resolveBrokerTeamName(brokerURL, teamName string) (string, error) { if teamName == "" || teamName == coreTeamName || teamName == coreTeamID { return coreTeamID, nil } - var resp struct { - Team brokerTeamInfo `json:"team"` + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return "", err } - if err := brokerPost(brokerURL, "/teams/resolve", brokerRepoAdminRequest{Name: teamName}, &resp); err != nil { + team, err := endpoints.ResolveTeam(context.Background(), protocol.RepositoryAdminRequest{Name: teamName}) + if err != nil { return "", err } - if strings.TrimSpace(resp.Team.ID) == "" { + if strings.TrimSpace(team.ID) == "" { return "", fmt.Errorf("team %q not found", teamName) } - return resp.Team.ID, nil + return team.ID, nil } func isDirectBrokerHost(host string) bool { @@ -3707,25 +3587,25 @@ func brokerProfileForCloneURL(brokerURL string) (brokerProfile, error) { }, nil } -func loadGlobalConfigForInit(path string) (globalConfig, string, error) { +func loadGlobalConfigForInit(path string) (internalconfig.Global, string, error) { var err error if path == "" { path, err = defaultGlobalConfigPath() if err != nil { - return globalConfig{}, "", err + return internalconfig.Global{}, "", err } } cfg, err := readGlobalConfig(path) if err != nil { if errors.Is(err, os.ErrNotExist) { - return globalConfig{Version: globalConfigVersion}, path, nil + return internalconfig.Global{Version: globalConfigVersion}, path, nil } - return globalConfig{}, path, err + return internalconfig.Global{}, path, err } return cfg, path, nil } -func brokerProfilesFromGlobalConfig(cfg globalConfig) []brokerProfile { +func brokerProfilesFromGlobalConfig(cfg internalconfig.Global) []brokerProfile { var profiles []brokerProfile for _, profile := range cfg.GCPProfiles { for _, region := range profile.Regions { @@ -3757,10 +3637,6 @@ func brokerProfilesFromGlobalConfig(cfg globalConfig) []brokerProfile { return profiles } -func selectBrokerProfile(profiles []brokerProfile, name string) (brokerProfile, error) { - return selectBrokerProfileForCommand(profiles, name, "", "bgit") -} - func selectBrokerProfileForCommand(profiles []brokerProfile, name, region, command string) (brokerProfile, error) { if strings.TrimSpace(name) == "" { if len(profiles) == 1 { @@ -3880,11 +3756,11 @@ func brokerInitProfilesWithLocal(profiles []brokerProfile) []brokerProfile { return out } } - out = append(out, brokerInitLocalProfile(globalConfig{})) + out = append(out, brokerInitLocalProfile(internalconfig.Global{})) return out } -func brokerInitLocalProfile(global globalConfig) brokerProfile { +func brokerInitLocalProfile(global internalconfig.Global) brokerProfile { profile := ensureDefaultLocalProfile(global) region := ensureDefaultLocalRegion(profile) return brokerProfile{ @@ -4011,8 +3887,12 @@ func brokerInitSelectTeam(reader *bufio.Reader, rawInput io.Reader, stdout io.Wr } func brokerInitTeamChoices(profile brokerProfile) ([]setupChoice, error) { - var resp brokerTeamsResponse - if err := brokerPost(profile.BrokerURL, "/teams/list", brokerRepoAdminRequest{}, &resp); err != nil { + endpoints, endpointErr := brokerEndpointClient(profile.BrokerURL) + if endpointErr != nil { + return nil, endpointErr + } + teamsResponse, err := endpoints.ListTeams(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { repos, repoErr := brokerReposMineAllKeys(context.Background(), profile.BrokerURL) if repoErr != nil { return nil, err @@ -4033,8 +3913,8 @@ func brokerInitTeamChoices(profile brokerProfile) ([]setupChoice, error) { sort.Slice(choices, func(i, j int) bool { return choices[i].Label < choices[j].Label }) return choices, nil } - choices := make([]setupChoice, 0, len(resp.Teams)) - for _, team := range resp.Teams { + choices := make([]setupChoice, 0, len(teamsResponse)) + for _, team := range teamsResponse { label := firstNonEmpty(strings.TrimSpace(team.Name), strings.TrimSpace(team.ID)) value := strings.TrimSpace(team.ID) if value == "" { @@ -4223,34 +4103,15 @@ func runInitDialogWithRaw(reader *bufio.Reader, rawInput io.Reader, stdout io.Wr } } -func logicalRepoWithGit(name string) string { - logical, err := normalizeLogicalRepoName(name) - if err != nil { - return "repo.git" - } - return logical -} - func normalizeLogicalRepoName(name string) (string, error) { - name = strings.TrimSpace(name) - name = strings.TrimSuffix(name, ".git") - if name == "" { - return "", errors.New("logical repo name is required") - } - if strings.ContainsAny(name, `/\`) { - return "", fmt.Errorf("logical repo names must be flat; use %q instead of a path", filepath.Base(name)) - } - if name == "." || name == ".." { - return "", errors.New("logical repo name is invalid") - } - return name + ".git", nil + return internalconfig.NormalizeLogicalRepository(name) } func logicalRepoDisplayName(name string) string { return strings.TrimSuffix(strings.Trim(strings.TrimSpace(name), "/"), ".git") } -func initDialogInitialState(target string, global globalConfig, repoName, profileName string) initDialogConfig { +func initDialogInitialState(target string, global internalconfig.Global, repoName, profileName string) initDialogConfig { initial := initDialogConfig{ RepoName: strings.TrimSpace(repoName), ProfileName: strings.TrimSpace(profileName), @@ -4569,11 +4430,11 @@ func initBrokerWorktree(target, repoName string, profile brokerProfile, identity if err != nil { return err } - repo := brokerRepo{Provider: profile.Provider, Logical: repoName, Prefix: repoName, Origin: fmt.Sprintf("git@%s:%s", defaultSSHHost, repoName), TeamID: profile.TeamID} + repo := protocol.Repository{Provider: profile.Provider, Logical: repoName, Prefix: repoName, Origin: fmt.Sprintf("git@%s:%s", defaultSSHHost, repoName), TeamID: profile.TeamID} return initBrokerWorktreeWithLocalRepo(target, repo, profile, identityName, identityEmail, stdout) } -func initBrokerWorktreeWithLocalRepo(target string, repo brokerRepo, profile brokerProfile, identityName, identityEmail string, stdout io.Writer) error { +func initBrokerWorktreeWithLocalRepo(target string, repo protocol.Repository, profile brokerProfile, identityName, identityEmail string, stdout io.Writer) error { absTarget, err := filepath.Abs(target) if err != nil { return err @@ -4661,7 +4522,12 @@ func brokerRequireExistingLogicalRepo(brokerURL, provider, logicalRepo, teamID s origin: fmt.Sprintf("git@%s:%s", defaultSSHHost, logical), teamID: strings.TrimSpace(teamID), } - return brokerPost(brokerURL, "/repos/get", brokerRepoRequest{Repo: repoForBroker(cfg)}, nil) + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + _, err = endpoints.GetRepository(context.Background(), protocol.RepositoryRequest{Repo: repoForBroker(cfg)}) + return err } func gitSSHCommandForExecutable() string { diff --git a/broker_commands_test.go b/internal/app/broker_commands_test.go similarity index 94% rename from broker_commands_test.go rename to internal/app/broker_commands_test.go index cc807e8..53c5e48 100644 --- a/broker_commands_test.go +++ b/internal/app/broker_commands_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -13,6 +13,9 @@ import ( "path/filepath" "strings" "testing" + + internalconfig "github.com/bucketgit/bgit/internal/config" + "github.com/bucketgit/bgit/protocol" ) func TestBrokerInitWritesBrokerGitConfig(t *testing.T) { @@ -21,7 +24,7 @@ func TestBrokerInitWritesBrokerGitConfig(t *testing.T) { if r.URL.Path != "/repos/get" { t.Fatalf("unexpected path %s", r.URL.Path) } - var req brokerRepoRequest + var req protocol.RepositoryRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -36,12 +39,12 @@ func TestBrokerInitWritesBrokerGitConfig(t *testing.T) { })) defer server.Close() configPath := filepath.Join(root, ".bgit", "config") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", ProjectID: "project-id", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: server.URL, }}, @@ -141,12 +144,12 @@ func TestLocalBrokerCloudIdentityUsesGlobalConfig(t *testing.T) { if err != nil { t.Fatal(err) } - global := globalConfig{ - AWSProfiles: []globalAWSProfile{{ + global := internalconfig.Global{ + AWSProfiles: []internalconfig.AWSProfile{{ Name: "work", AccountID: "123456789012", }}, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "prod", ProjectID: "example-project", }}, @@ -307,8 +310,8 @@ func TestGCPADCProjectIDFromFileUsesQuotaProject(t *testing.T) { } func TestLocalBrokerStorageCompatibleRequiresSameCloudBucket(t *testing.T) { - requested := brokerRepo{Provider: "s3", Bucket: "123456789012-demo", Profile: "default", Region: "us-east-1"} - existing := brokerRepo{Provider: "s3", Bucket: "123456789012-demo", Profile: "default", Region: "us-east-1"} + requested := protocol.Repository{Provider: "s3", Bucket: "123456789012-demo", Profile: "default", Region: "us-east-1"} + existing := protocol.Repository{Provider: "s3", Bucket: "123456789012-demo", Profile: "default", Region: "us-east-1"} if !localBrokerStorageCompatible(requested, existing) { t.Fatal("same deterministic s3 repo should be compatible") } @@ -321,7 +324,7 @@ func TestLocalBrokerStorageCompatibleRequiresSameCloudBucket(t *testing.T) { if localBrokerStorageCompatible(requested, existing) { t.Fatal("different s3 regions should not be compatible") } - fileRepo := brokerRepo{Provider: "file", Bucket: "file://demo.git"} + fileRepo := protocol.Repository{Provider: "file", Bucket: "file://demo.git"} if localBrokerStorageCompatible(requested, fileRepo) { t.Fatal("s3 and file local broker repos should not be compatible") } @@ -399,8 +402,8 @@ func TestInitBrokerWorktreeOmitsIdentityWhenUnset(t *testing.T) { } func TestBoardCommandCreatesAndMovesStories(t *testing.T) { - var createReq brokerIssueRequest - var moveReq brokerIssueRequest + var createReq protocol.IssueRequest + var moveReq protocol.IssueRequest target, server, requests := setupBrokerCommandTestRepo(t, func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/issues/create": @@ -481,8 +484,8 @@ func TestBoardCommandListsStoriesByLane(t *testing.T) { } func TestBoardCommandEditsAndArchivesStories(t *testing.T) { - var updateReq brokerIssueRequest - var archiveReq brokerIssueRequest + var updateReq protocol.IssueRequest + var archiveReq protocol.IssueRequest target, server, requests := setupBrokerCommandTestRepo(t, func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/issues/update": @@ -528,7 +531,7 @@ func TestBoardCommandEditsAndArchivesStories(t *testing.T) { } func TestBoardCommandPrioritizesStory(t *testing.T) { - var reorderReq brokerIssueRequest + var reorderReq protocol.IssueRequest target, server, requests := setupBrokerCommandTestRepo(t, func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/issues/reorder" { t.Fatalf("unexpected path %s", r.URL.Path) @@ -563,7 +566,7 @@ func TestBoardCommandPrioritizesStory(t *testing.T) { } func TestBoardCommandListsArchivedStoriesSeparately(t *testing.T) { - var listReq brokerIssueRequest + var listReq protocol.IssueRequest target, server, _ := setupBrokerCommandTestRepo(t, func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/issues/list" { t.Fatalf("unexpected path %s", r.URL.Path) @@ -689,9 +692,9 @@ func TestParseBoardStoryIDArgRequiresRepoPrefix(t *testing.T) { } func TestBrokerUpgradeTargetUsesCurrentRepoProfile(t *testing.T) { - global := globalConfig{GCPProfiles: []globalGCPProfile{{ + global := internalconfig.Global{GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: "https://broker.example.test", }}, @@ -710,9 +713,9 @@ func TestBrokerUpgradeTargetUsesCurrentRepoProfile(t *testing.T) { } func TestBrokerUpgradeTargetFallsBackToBrokerURL(t *testing.T) { - global := globalConfig{AWSProfiles: []globalAWSProfile{{ + global := internalconfig.Global{AWSProfiles: []internalconfig.AWSProfile{{ Name: "prod", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "us-east-1", BrokerURL: "https://abc.lambda-url.us-east-1.on.aws/", }}, @@ -746,7 +749,7 @@ func TestAdminKeysListUsesLogicalBrokerRepo(t *testing.T) { if r.URL.Path != "/keys/list" { t.Fatalf("unexpected path %s", r.URL.Path) } - var req brokerRepoRequest + var req protocol.RepositoryRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -776,7 +779,7 @@ func TestAdminKeysListUsesLogicalBrokerRepo(t *testing.T) { } func TestInviteUserPreservesTeamScopedRepo(t *testing.T) { - var got brokerOwnerTransferRequest + var got protocol.OwnerTransferRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/keys/invite/create" { t.Fatalf("unexpected path %s", r.URL.Path) @@ -802,11 +805,11 @@ func TestInviteUserPreservesTeamScopedRepo(t *testing.T) { func TestPrintBrokerUsersUsesReadableColumns(t *testing.T) { var stdout bytes.Buffer - printBrokerUsers(&stdout, []brokerUserInfo{{ + printBrokerUsers(&stdout, []protocol.UserInfo{{ ID: "u_owner", Username: "owner", BrokerRole: "owner", - Keys: []brokerKey{{PublicKey: "ssh-ed25519 AAAA owner"}}, + Keys: []protocol.Key{{PublicKey: "ssh-ed25519 AAAA owner"}}, }, { ID: "u_pending", Username: "pending", @@ -834,20 +837,20 @@ func TestTopLevelBrokerInitForwardsGlobalProfile(t *testing.T) { })) defer server.Close() configPath := filepath.Join(root, ".bgit", "config") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", ProjectID: "project-id", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: server.URL, }}, }}, - AWSProfiles: []globalAWSProfile{{ + AWSProfiles: []internalconfig.AWSProfile{{ Name: "prod", AccountID: "123456789012", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "eu-west-1", BrokerURL: "https://aws-broker.example.test", }}, @@ -922,11 +925,11 @@ func TestExplicitBrokerProfileSelectionUsesRegionForDataPlaneCommand(t *testing. home := t.TempDir() setTestHome(t, home) configPath := filepath.Join(home, ".bgit", "config.yaml") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "us-central1", BrokerURL: "https://us.example.test", }, { @@ -950,11 +953,11 @@ func TestExplicitBrokerProfileSelectionRejectsAmbiguousDataPlaneProfile(t *testi home := t.TempDir() setTestHome(t, home) configPath := filepath.Join(home, ".bgit", "config.yaml") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "us-central1", BrokerURL: "https://us.example.test", }, { @@ -1093,12 +1096,12 @@ func TestBrokerInitInteractivePromptsForRepoAndProfile(t *testing.T) { })) defer server.Close() configPath := filepath.Join(root, ".bgit", "config") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - AWSProfiles: []globalAWSProfile{{ + AWSProfiles: []internalconfig.AWSProfile{{ Name: "prod", AccountID: "123456789012", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "eu-west-1", BrokerURL: server.URL, }}, @@ -1124,12 +1127,12 @@ func TestBrokerInitInteractivePromptsForRepoAndProfile(t *testing.T) { func TestBrokerInitInteractiveIdentityOnlyUpdatesRepoConfig(t *testing.T) { root := t.TempDir() configPath := filepath.Join(root, ".bgit", "config") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - Identity: globalIdentityConfig{Name: "Global User", Email: "global@example.com"}, - GCPProfiles: []globalGCPProfile{{ + Identity: internalconfig.Identity{Name: "Global User", Email: "global@example.com"}, + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: "https://broker.example.test", }}, @@ -1181,7 +1184,7 @@ func TestBrokerInitInteractiveSelectsTeamThenRepository(t *testing.T) { {"repo":{"provider":"gcs","logical":"other.git","team_id":"t_other"},"logical":"other.git","teams":[{"id":"t_other","role":"developer"}]} ]}`)) case "/repos/get": - var req brokerRepoAdminRequest + var req protocol.RepositoryAdminRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -1195,11 +1198,11 @@ func TestBrokerInitInteractiveSelectsTeamThenRepository(t *testing.T) { })) defer server.Close() configPath := filepath.Join(root, ".bgit", "config") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: server.URL, }}, @@ -1358,12 +1361,12 @@ func TestInitDialogInitialStateUsesRepoThenGlobalIdentity(t *testing.T) { t.Fatal(err) } } - initial := initDialogInitialState(target, globalConfig{Identity: globalIdentityConfig{Name: "Global User", Email: "global@example.com"}}, "", "") + initial := initDialogInitialState(target, internalconfig.Global{Identity: internalconfig.Identity{Name: "Global User", Email: "global@example.com"}}, "", "") if !initial.Existing || initial.RepoName != "app.git" || initial.ProfileName != "gcp:work/europe-west1" || initial.IdentityName != "Repo User" || initial.IdentityEmail != "repo@example.com" { t.Fatalf("initial = %#v", initial) } - fresh := initDialogInitialState(filepath.Join(root, "fresh"), globalConfig{Identity: globalIdentityConfig{Name: "Global User", Email: "global@example.com"}}, "new.git", "") + fresh := initDialogInitialState(filepath.Join(root, "fresh"), internalconfig.Global{Identity: internalconfig.Identity{Name: "Global User", Email: "global@example.com"}}, "new.git", "") if fresh.Existing || fresh.IdentityName != "Global User" || fresh.IdentityEmail != "global@example.com" { t.Fatalf("fresh = %#v", fresh) } @@ -1433,7 +1436,7 @@ func TestAdminCloudIAMMovedToDirect(t *testing.T) { } func TestAdminRepoCreateUsesCreateEndpointAndTeam(t *testing.T) { - var got brokerRepoAdminRequest + var got protocol.RepositoryAdminRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") switch r.URL.Path { @@ -1463,7 +1466,7 @@ func TestAdminRepoCreateUsesCreateEndpointAndTeam(t *testing.T) { } func TestAdminRepoCreateAllowsOwnerTeamGrant(t *testing.T) { - var got brokerRepoAdminRequest + var got protocol.RepositoryAdminRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") switch r.URL.Path { @@ -1488,7 +1491,7 @@ func TestAdminRepoCreateAllowsOwnerTeamGrant(t *testing.T) { } func TestAdminTeamsRepoAddAllowsOwnerRole(t *testing.T) { - var got brokerRepoAdminRequest + var got protocol.RepositoryAdminRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") if r.URL.Path != "/repo/teams/upsert" { @@ -1518,7 +1521,7 @@ func TestBrokerAdminProtectAndPRCommandsUseBroker(t *testing.T) { case "/protection/list": _, _ = w.Write([]byte(`{"protections":[{"ref":"refs/heads/main","require_pr":true,"allow_overrides":true}]}`)) case "/prs/create": - var req brokerPullRequestRequest + var req protocol.PullRequestRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -1576,7 +1579,7 @@ func TestBrokerAdminProtectAndPRCommandsUseBroker(t *testing.T) { } func TestImportGitHubKeysConfirmsAndStoresSource(t *testing.T) { - var addReq brokerKeyRequest + var addReq protocol.KeyRequest target, server, _ := setupBrokerCommandTestRepo(t, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") if r.URL.Path != "/keys/add" { diff --git a/internal/app/broker_data_path.go b/internal/app/broker_data_path.go new file mode 100644 index 0000000..f1b0a48 --- /dev/null +++ b/internal/app/broker_data_path.go @@ -0,0 +1,102 @@ +package app + +import ( + "context" + "errors" + "net/http" + "strings" + + "github.com/bucketgit/bgit/broker/capability" + "github.com/bucketgit/bgit/protocol" +) + +type brokerGitStore struct { + brokerURL string + cfg config +} + +type brokerGitControlPlane struct { + url string +} + +func (b brokerGitControlPlane) PostJSON(ctx context.Context, endpoint string, request, response any, headers http.Header) error { + extraHeaders := make(map[string]string, len(headers)) + for key := range headers { + extraHeaders[key] = headers.Get(key) + } + return brokerPostJSONContextWithHeaders(ctx, b.url, endpoint, request, response, extraHeaders) +} + +type brokerGitLocalCapabilities struct{} + +func (brokerGitLocalCapabilities) Read(ctx context.Context, issued protocol.ObjectCapabilityResponse) ([]byte, error) { + return localBrokerCapabilityRead(ctx, issued) +} + +func (brokerGitLocalCapabilities) Write(ctx context.Context, issued protocol.ObjectCapabilityResponse, data []byte) error { + return localBrokerCapabilityWrite(ctx, issued, data) +} + +func (brokerGitLocalCapabilities) Delete(ctx context.Context, issued protocol.ObjectCapabilityResponse) error { + return localBrokerCapabilityDelete(ctx, issued) +} + +func (s *brokerGitStore) capabilityStore() (*capability.Store, error) { + return capability.New(brokerGitControlPlane{url: s.brokerURL}, repoForBroker(s.cfg), capability.Options{ + HTTPClient: http.DefaultClient, + Local: brokerGitLocalCapabilities{}, + IsNotFound: isBrokerNotFoundError, + IsUnsupported: isBrokerCapabilityUnsupported, + }) +} + +func (s *brokerGitStore) read(ctx context.Context, objectPath string) ([]byte, error) { + backend, err := s.capabilityStore() + if err != nil { + return nil, err + } + return backend.Read(ctx, strings.TrimPrefix(objectPath, "/")) +} + +func (s *brokerGitStore) list(ctx context.Context, prefix string) ([]string, error) { + backend, err := s.capabilityStore() + if err != nil { + return nil, err + } + return backend.List(ctx, strings.TrimPrefix(prefix, "/")) +} + +func (s *brokerGitStore) write(ctx context.Context, objectPath string, data []byte) error { + backend, err := s.capabilityStore() + if err != nil { + return err + } + return backend.Write(ctx, strings.TrimPrefix(objectPath, "/"), data) +} + +func (s *brokerGitStore) delete(ctx context.Context, objectPath string) error { + backend, err := s.capabilityStore() + if err != nil { + return err + } + return backend.Delete(ctx, strings.TrimPrefix(objectPath, "/")) +} + +func (s *brokerGitStore) listRefs(ctx context.Context) (map[string]string, error) { + backend, err := s.capabilityStore() + if err != nil { + return nil, err + } + return backend.ListRefs(ctx) +} + +func isBrokerCapabilityUnsupported(err error) bool { + if err == nil { + return false + } + if errors.Is(err, protocol.ErrUnsupported) { + return true + } + message := err.Error() + return strings.Contains(message, "unknown broker endpoint") || strings.Contains(message, "404") +} diff --git a/broker_lifecycle.go b/internal/app/broker_lifecycle.go similarity index 95% rename from broker_lifecycle.go rename to internal/app/broker_lifecycle.go index 0d69dd1..aa977c5 100644 --- a/broker_lifecycle.go +++ b/internal/app/broker_lifecycle.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -8,6 +8,8 @@ import ( "io" "os" "strings" + + internalconfig "github.com/bucketgit/bgit/internal/config" ) type brokerDeleteOptions struct { @@ -38,7 +40,7 @@ type managedLocalBroker struct { BootstrapToken string } -func ensureManagedLocalBroker(ctx context.Context, profile globalLocalProfile, region globalProfileRegion) (*managedLocalBroker, error) { +func ensureManagedLocalBroker(ctx context.Context, profile internalconfig.LocalProfile, region internalconfig.ProfileRegion) (*managedLocalBroker, error) { _ = ctx root := expandHome(profile.Root) if strings.TrimSpace(root) == "" { @@ -254,12 +256,12 @@ func removeDeletedBrokerFromGlobalConfig(opts brokerDeleteOptions, provider, pro return writeGlobalConfig(path, global) } -func clearGlobalProfileRegion(regions []globalProfileRegion, region string) []globalProfileRegion { +func clearGlobalProfileRegion(regions []internalconfig.ProfileRegion, region string) []internalconfig.ProfileRegion { region = strings.TrimSpace(region) if region == "" { return nil } - var out []globalProfileRegion + var out []internalconfig.ProfileRegion for _, entry := range regions { if entry.Name != region { out = append(out, entry) diff --git a/cloud_identity.go b/internal/app/cloud_identity.go similarity index 99% rename from cloud_identity.go rename to internal/app/cloud_identity.go index 679305c..d5ad3f2 100644 --- a/cloud_identity.go +++ b/internal/app/cloud_identity.go @@ -1,4 +1,4 @@ -package main +package app import ( "context" diff --git a/git_pack.go b/internal/app/git_pack.go similarity index 87% rename from git_pack.go rename to internal/app/git_pack.go index 0ea6ea6..5393801 100644 --- a/git_pack.go +++ b/internal/app/git_pack.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -13,6 +13,8 @@ import ( "io/fs" "sort" "strings" + + transportpkg "github.com/bucketgit/bgit/transport" ) type uploadPackRequest struct { @@ -25,41 +27,7 @@ type uploadPackRequest struct { } func serveUploadPack(ctx context.Context, repo *nativeGitRepo, stdin io.Reader, stdout io.Writer) error { - req, err := readUploadPackRequestWithResponses(stdin, stdout) - if err != nil { - if errors.Is(err, io.EOF) { - return nil - } - return err - } - if len(req.wants) == 0 { - return nil - } - if !req.responded { - if err := writePktLineString(stdout, "NAK\n"); err != nil { - return err - } - } - hashes, err := repo.objectsForUploadPack(ctx, req.wants, req.haves) - if err != nil { - return err - } - pack, err := repo.encodePack(ctx, hashes) - if err != nil { - return err - } - if req.sideband || req.sideband64 { - maxPayload := 1000 - if req.sideband64 { - maxPayload = 65516 - } - if err := writeSideband(stdout, 1, pack, maxPayload); err != nil { - return err - } - return writePktFlush(stdout) - } - _, err = stdout.Write(pack) - return err + return transportpkg.ServeUploadPack(ctx, repo.engine, stdin, stdout) } func readUploadPackRequest(stdin io.Reader) (uploadPackRequest, error) { diff --git a/git_pack_test.go b/internal/app/git_pack_test.go similarity index 99% rename from git_pack_test.go rename to internal/app/git_pack_test.go index 176410b..54ec5bb 100644 --- a/git_pack_test.go +++ b/internal/app/git_pack_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" diff --git a/git_protocol.go b/internal/app/git_protocol.go similarity index 62% rename from git_protocol.go rename to internal/app/git_protocol.go index f6684fb..7519ef5 100644 --- a/git_protocol.go +++ b/internal/app/git_protocol.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -6,9 +6,10 @@ import ( "errors" "fmt" "io" - "sort" "strconv" "strings" + + transportpkg "github.com/bucketgit/bgit/transport" ) const ( @@ -16,8 +17,8 @@ const ( pktDelim = "0001" pktResponseEnd = "0002" - gitUploadPackService = "git-upload-pack" - gitReceivePackService = "git-receive-pack" + gitUploadPackService = transportpkg.UploadPackService + gitReceivePackService = transportpkg.ReceivePackService ) type pktLineKind int @@ -34,31 +35,20 @@ type pktLine struct { data []byte } -func writePktLine(w io.Writer, data []byte) error { - if len(data) > 65516 { - return errors.New("pkt-line payload too large") - } - _, err := fmt.Fprintf(w, "%04x%s", len(data)+4, data) - return err -} - func writePktLineString(w io.Writer, data string) error { - return writePktLine(w, []byte(data)) + return transportpkg.WriteString(w, data) } func writePktFlush(w io.Writer) error { - _, err := io.WriteString(w, pktFlush) - return err + return transportpkg.WriteFlush(w) } func writePktDelim(w io.Writer) error { - _, err := io.WriteString(w, pktDelim) - return err + return transportpkg.WriteDelimiter(w) } func writePktResponseEnd(w io.Writer) error { - _, err := io.WriteString(w, pktResponseEnd) - return err + return transportpkg.WriteResponseEnd(w) } func readPktLine(r *bufio.Reader) (pktLine, error) { @@ -88,33 +78,8 @@ func readPktLine(r *bufio.Reader) (pktLine, error) { return pktLine{kind: pktLineData, data: data}, nil } -func pktLineDataString(line pktLine) (string, bool) { - if line.kind != pktLineData { - return "", false - } - return string(line.data), true -} - func writeSideband(w io.Writer, channel byte, data []byte, maxPayload int) error { - if channel < 1 || channel > 3 { - return fmt.Errorf("invalid sideband channel %d", channel) - } - if maxPayload <= 1 || maxPayload > 65516 { - maxPayload = 65516 - } - chunkSize := maxPayload - 1 - for len(data) > 0 { - n := chunkSize - if len(data) < n { - n = len(data) - } - payload := append([]byte{channel}, data[:n]...) - if err := writePktLine(w, payload); err != nil { - return err - } - data = data[n:] - } - return nil + return transportpkg.WriteSideband(w, channel, data, maxPayload) } type gitCapabilities []string @@ -124,37 +89,11 @@ func (c gitCapabilities) String() string { } func uploadPackCapabilities() gitCapabilities { - return gitCapabilities{ - "multi_ack", - "multi_ack_detailed", - "no-done", - "thin-pack", - "side-band", - "side-band-64k", - "ofs-delta", - "no-progress", - "include-tag", - "allow-tip-sha1-in-want", - "allow-reachable-sha1-in-want", - "symref=HEAD:refs/heads/main", - "object-format=sha1", - "agent=bgit", - } + return gitCapabilities(transportpkg.UploadPackCapabilities()) } func receivePackCapabilities() gitCapabilities { - return gitCapabilities{ - "report-status", - "report-status-v2", - "delete-refs", - "side-band-64k", - "quiet", - "atomic", - "ofs-delta", - "push-options", - "object-format=sha1", - "agent=bgit", - } + return gitCapabilities(transportpkg.ReceivePackCapabilities()) } type gitCommandInvocation struct { @@ -228,48 +167,7 @@ func cleanGitServiceRepo(repo string) string { } func writeAdvertisedRefs(w io.Writer, service string, refs map[string]string, caps gitCapabilities) error { - if service != gitUploadPackService && service != gitReceivePackService { - return fmt.Errorf("unsupported service %q", service) - } - names := make([]string, 0, len(refs)) - for name := range refs { - if name == "HEAD" || strings.HasPrefix(name, "refs/") { - names = append(names, name) - } - } - sort.Strings(names) - if _, ok := refs["HEAD"]; ok { - names = append([]string{"HEAD"}, removeString(names, "HEAD")...) - } - if len(names) == 0 { - return writePktFlush(w) - } - first := true - for _, name := range names { - hash := strings.TrimSpace(refs[name]) - if !isHexHash(hash) { - continue - } - line := hash + " " + name - if first { - line += "\x00" + caps.String() - first = false - } - if err := writePktLineString(w, line+"\n"); err != nil { - return err - } - } - return writePktFlush(w) -} - -func removeString(values []string, remove string) []string { - out := values[:0] - for _, value := range values { - if value != remove { - out = append(out, value) - } - } - return out + return transportpkg.WriteAdvertisedRefs(w, service, refs, transportpkg.Capabilities(caps)) } func pktLinesForTest(buf []byte) ([]pktLine, error) { diff --git a/git_protocol_test.go b/internal/app/git_protocol_test.go similarity index 99% rename from git_protocol_test.go rename to internal/app/git_protocol_test.go index 6566867..44ef254 100644 --- a/git_protocol_test.go +++ b/internal/app/git_protocol_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" diff --git a/internal/app/git_receive.go b/internal/app/git_receive.go new file mode 100644 index 0000000..3ea4b6d --- /dev/null +++ b/internal/app/git_receive.go @@ -0,0 +1,222 @@ +package app + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "strings" + + transportpkg "github.com/bucketgit/bgit/transport" +) + +type receivePackCommand struct { + old string + new string + ref string + caps map[string]bool +} + +type receivePackRequest struct { + commands []receivePackCommand + caps map[string]bool + pushOptions []string + pack io.Reader +} + +type nativeReceiveStore struct { + repo *nativeGitRepo + store writableGitRemoteStore +} + +func (s nativeReceiveStore) Read(ctx context.Context, path string) ([]byte, error) { + return s.store.read(ctx, path) +} +func (s nativeReceiveStore) List(ctx context.Context, prefix string) ([]string, error) { + return s.store.list(ctx, prefix) +} +func (s nativeReceiveStore) Write(ctx context.Context, path string, data []byte) error { + return s.store.write(ctx, path, data) +} +func (s nativeReceiveStore) Delete(ctx context.Context, path string) error { + return s.store.delete(ctx, path) +} +func (s nativeReceiveStore) ListRefs(ctx context.Context) (map[string]string, error) { + return s.repo.refs(ctx) +} +func (s nativeReceiveStore) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + if s.repo.cfg.origin != "" { + brokerURL, err := brokerURLForSSHService(s.repo.cfg) + if err != nil { + return err + } + if err := brokerUpdateRef(brokerURL, s.repo.cfg, ref, oldOID, firstNonEmpty(newOID, zeroObjectID())); err != nil { + return err + } + if newOID == "" && strings.HasPrefix(ref, "refs/heads/") { + _ = unsetGitBranchTracking(".", strings.TrimPrefix(ref, "refs/heads/")) + } + return nil + } + if refs, ok := s.store.(refCASGitRemoteStore); ok { + if err := refs.compareAndSwapRef(ctx, ref, oldOID, newOID); err != nil { + return err + } + if newOID == "" && strings.HasPrefix(ref, "refs/heads/") { + _ = unsetGitBranchTracking(".", strings.TrimPrefix(ref, "refs/heads/")) + } + return nil + } + refs, err := s.repo.refs(ctx) + if err != nil { + return err + } + current, exists := refs[ref] + if (oldOID == "" && exists) || (oldOID != "" && current != oldOID) { + return errors.New("stale ref") + } + if newOID == "" { + if err := s.store.delete(ctx, ref); err != nil { + return err + } + if strings.HasPrefix(ref, "refs/heads/") { + _ = unsetGitBranchTracking(".", strings.TrimPrefix(ref, "refs/heads/")) + } + return nil + } + return s.store.write(ctx, ref, []byte(newOID+"\n")) +} + +func readReceivePackRequest(stdin io.Reader) (receivePackRequest, error) { + r := bufio.NewReader(stdin) + var req receivePackRequest + for { + line, err := readPktLine(r) + if err != nil { + return req, err + } + switch line.kind { + case pktLineFlush: + if req.caps["push-options"] { + options, err := readReceivePackPushOptions(r) + if err != nil { + return req, err + } + req.pushOptions = options + } + req.pack = r + return req, nil + case pktLineData: + default: + continue + } + cmd, caps, err := parseReceivePackCommandLine(string(line.data), len(req.commands) == 0) + if err != nil { + return req, err + } + req.commands = append(req.commands, cmd) + if caps != nil { + req.caps = caps + } + } +} + +func readReceivePackPushOptions(r *bufio.Reader) ([]string, error) { + var options []string + for { + line, err := readPktLine(r) + if err != nil { + return nil, err + } + switch line.kind { + case pktLineFlush: + return options, nil + case pktLineData: + options = append(options, strings.TrimRight(string(line.data), "\n")) + default: + return nil, fmt.Errorf("invalid push-options pkt-line") + } + } +} + +func parseReceivePackCommandLine(line string, first bool) (receivePackCommand, map[string]bool, error) { + line = strings.TrimRight(line, "\n") + capText := "" + if first { + before, after, ok := strings.Cut(line, "\x00") + if ok { + line = before + capText = after + } + } + fields := strings.Fields(line) + if len(fields) != 3 { + return receivePackCommand{}, nil, fmt.Errorf("invalid receive-pack command %q", line) + } + if !isHexHash(fields[0]) || !isHexHash(fields[1]) { + return receivePackCommand{}, nil, fmt.Errorf("invalid receive-pack object id in %q", line) + } + if !strings.HasPrefix(fields[2], "refs/") { + return receivePackCommand{}, nil, fmt.Errorf("invalid receive-pack ref %q", fields[2]) + } + cmd := receivePackCommand{old: fields[0], new: fields[1], ref: fields[2]} + caps := map[string]bool{} + if capText != "" { + for _, cap := range strings.Fields(capText) { + caps[cap] = true + if name, _, ok := strings.Cut(cap, "="); ok { + caps[name] = true + } + } + cmd.caps = caps + } + return cmd, caps, nil +} + +func zeroObjectID() string { + return "0000000000000000000000000000000000000000" +} + +func (c receivePackCommand) action() string { + switch { + case c.old == zeroObjectID() && c.new != zeroObjectID(): + return "create" + case c.old != zeroObjectID() && c.new == zeroObjectID(): + return "delete" + case c.old != zeroObjectID() && c.new != zeroObjectID(): + return "update" + default: + return "noop" + } +} + +func writeReceivePackReportStatus(w io.Writer, commands []receivePackCommand, unpackErr error, commandErrs map[string]error) error { + if unpackErr != nil { + if err := writePktLineString(w, "unpack "+unpackErr.Error()+"\n"); err != nil { + return err + } + } else if err := writePktLineString(w, "unpack ok\n"); err != nil { + return err + } + for _, cmd := range commands { + if err := commandErrs[cmd.ref]; err != nil { + if err := writePktLineString(w, "ng "+cmd.ref+" "+err.Error()+"\n"); err != nil { + return err + } + continue + } + if err := writePktLineString(w, "ok "+cmd.ref+"\n"); err != nil { + return err + } + } + return writePktFlush(w) +} + +func serveReceivePack(ctx context.Context, repo *nativeGitRepo, stdin io.Reader, stdout io.Writer) error { + store, ok := repo.store.(writableGitRemoteStore) + if !ok { + return errors.New("receive-pack requires a writable store") + } + return transportpkg.ServeReceivePack(ctx, repo.engine, nativeReceiveStore{repo: repo, store: store}, stdin, stdout) +} diff --git a/git_receive_pack.go b/internal/app/git_receive_pack.go similarity index 82% rename from git_receive_pack.go rename to internal/app/git_receive_pack.go index a68b91f..18a5f09 100644 --- a/git_receive_pack.go +++ b/internal/app/git_receive_pack.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" @@ -26,25 +26,6 @@ type packedObjectRecord struct { hash string } -func ingestReceivePack(ctx context.Context, repo *nativeGitRepo, store writableGitRemoteStore, pack io.Reader) (map[string]gitObject, error) { - data, err := io.ReadAll(pack) - if err != nil { - return nil, err - } - objects, err := decodePackObjectsWithBase(ctx, repo, data) - if err != nil { - return nil, err - } - written := map[string]gitObject{} - for _, obj := range objects { - if err := writeLooseObject(ctx, store, obj.typ, obj.data); err != nil { - return nil, err - } - written[obj.hash] = gitObject{typ: obj.typ, data: obj.data} - } - return written, nil -} - func decodePackObjects(pack []byte) ([]receivedPackObject, error) { return decodePackObjectsWithBase(context.Background(), nil, pack) } @@ -207,24 +188,6 @@ func inflatePackEntry(data []byte) ([]byte, int, error) { return out.Bytes(), consumed, nil } -func writeLooseObject(ctx context.Context, store writableGitRemoteStore, typ string, data []byte) error { - hash := objectHash(typ, data) - var raw bytes.Buffer - fmt.Fprintf(&raw, "%s %d", typ, len(data)) - raw.WriteByte(0) - raw.Write(data) - var compressed bytes.Buffer - zw := zlib.NewWriter(&compressed) - if _, err := zw.Write(raw.Bytes()); err != nil { - _ = zw.Close() - return err - } - if err := zw.Close(); err != nil { - return err - } - return store.write(ctx, "objects/"+hash[:2]+"/"+hash[2:], compressed.Bytes()) -} - func objectHash(typ string, data []byte) string { var raw bytes.Buffer fmt.Fprintf(&raw, "%s %d", typ, len(data)) diff --git a/git_receive_test.go b/internal/app/git_receive_test.go similarity index 99% rename from git_receive_test.go rename to internal/app/git_receive_test.go index a31a624..18ab71c 100644 --- a/git_receive_test.go +++ b/internal/app/git_receive_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" diff --git a/gitignore_native.go b/internal/app/gitignore_native.go similarity index 99% rename from gitignore_native.go rename to internal/app/gitignore_native.go index ccfee95..1693e7a 100644 --- a/gitignore_native.go +++ b/internal/app/gitignore_native.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" diff --git a/internal/app/global_config.go b/internal/app/global_config.go new file mode 100644 index 0000000..b46b050 --- /dev/null +++ b/internal/app/global_config.go @@ -0,0 +1,15 @@ +package app + +import internalconfig "github.com/bucketgit/bgit/internal/config" + +const globalConfigVersion = internalconfig.GlobalVersion + +func defaultGlobalConfigPath() (string, error) { return internalconfig.DefaultGlobalPath() } +func defaultLocalBrokerRoot() (string, error) { return internalconfig.DefaultLocalBrokerRoot() } +func defaultBGitCacheDir() (string, error) { return internalconfig.DefaultCacheDir() } +func readGlobalConfig(path string) (internalconfig.Global, error) { + return internalconfig.ReadGlobal(path) +} +func writeGlobalConfig(path string, value internalconfig.Global) error { + return internalconfig.WriteGlobal(path, value) +} diff --git a/global_config_test.go b/internal/app/global_config_test.go similarity index 91% rename from global_config_test.go rename to internal/app/global_config_test.go index 1c560c4..a849bfc 100644 --- a/global_config_test.go +++ b/internal/app/global_config_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "os" @@ -6,51 +6,53 @@ import ( "reflect" "strings" "testing" + + internalconfig "github.com/bucketgit/bgit/internal/config" ) func TestGlobalConfigRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), ".bgit", "config.yaml") - want := globalConfig{ + want := internalconfig.Global{ Version: globalConfigVersion, - Identity: globalIdentityConfig{ + Identity: internalconfig.Identity{ Name: "Dennis Example", Email: "dennis@example.com", }, - GCPProfiles: []globalGCPProfile{{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", ProjectID: "example-test-123456", Account: "dennis@example.com", ServiceAccount: "bgit-broker@example-test-123456.iam.gserviceaccount.com", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: "https://gcp.example.test", BrokerVersion: brokerVersion(), LastSetupAt: "2026-05-16T10:00:00Z", }}, }}, - AWSProfiles: []globalAWSProfile{{ + AWSProfiles: []internalconfig.AWSProfile{{ Name: "work", AccountID: "123456789012", ARN: "arn:aws:iam::123456789012:user/dennis", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "us-east-1", BrokerURL: "https://aws.example.test", BrokerVersion: brokerVersion(), LastSetupAt: "2026-05-16T10:00:00Z", }}, }}, - LocalProfiles: []globalLocalProfile{{ + LocalProfiles: []internalconfig.LocalProfile{{ Name: "default", Root: "/tmp/bgit-local", Autostart: true, - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "default", BrokerURL: "local://default/default", BrokerVersion: brokerVersion(), LastSetupAt: "2026-05-16T10:00:00Z", }}, }}, - Repos: []globalRepoConfig{{ + Repos: []internalconfig.Repo{{ Name: "app.git", Profile: "gcp:work", BrokerURL: "https://gcp.example.test", diff --git a/identity.go b/internal/app/identity.go similarity index 79% rename from identity.go rename to internal/app/identity.go index 68d0ccb..b89e15a 100644 --- a/identity.go +++ b/internal/app/identity.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -6,13 +6,21 @@ import ( "fmt" "io" "os" - "regexp" "strings" + + internalconfig "github.com/bucketgit/bgit/internal/config" + internalidentity "github.com/bucketgit/bgit/internal/identity" ) -const defaultBucketGitIdentityName = "BucketGit Client" +const defaultBucketGitIdentityName = internalidentity.DefaultName + +type identityEmailValidator struct{} + +func (identityEmailValidator) MatchString(value string) bool { + return internalidentity.ValidEmail(value) +} -var identityEmailPattern = regexp.MustCompile(`^[^@\s]+@[^@\s]+\.[^@\s]+$`) +var identityEmailPattern identityEmailValidator type identityConfig struct { Name string @@ -22,27 +30,17 @@ type identityConfig struct { func defaultBucketGitIdentityEmail() string { username := firstNonEmpty(os.Getenv("USER"), os.Getenv("USERNAME"), "username") - username = strings.ToLower(strings.TrimSpace(username)) - var clean strings.Builder - for _, r := range username { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' { - clean.WriteRune(r) - } - } - if clean.Len() == 0 { - return "username@bucketgit.com" - } - return clean.String() + "@bucketgit.com" + return internalidentity.DefaultEmail(username) } -func readGlobalIdentity() globalIdentityConfig { +func readGlobalIdentity() internalconfig.Identity { path, err := defaultGlobalConfigPath() if err != nil { - return globalIdentityConfig{} + return internalconfig.Identity{} } cfg, err := readGlobalConfig(path) if err != nil { - return globalIdentityConfig{} + return internalconfig.Identity{} } return cfg.Identity } @@ -58,11 +56,8 @@ func effectiveRepositoryIdentity(repo *localRepository) identityConfig { defaultEmail := defaultBucketGitIdentityEmail() name = firstNonEmpty(name, defaultBucketGitIdentityName) email = firstNonEmpty(email, defaultEmail) - return identityConfig{ - Name: name, - Email: email, - UsesDefault: name == defaultBucketGitIdentityName || email == defaultEmail, - } + value := internalidentity.Effective(name, email, defaultEmail) + return identityConfig{Name: value.Name, Email: value.Email, UsesDefault: value.UsesDefault} } func (r *localRepository) identityValue(key string) string { @@ -144,7 +139,7 @@ func readIdentityFields(reader *bufio.Reader, stdout io.Writer, currentName, cur if email == "" { email = strings.TrimSpace(currentEmail) } - if email != "" && !identityEmailPattern.MatchString(email) { + if email != "" && !internalidentity.ValidEmail(email) { return "", "", fmt.Errorf("email address %q looks invalid", email) } return name, email, nil @@ -160,8 +155,8 @@ func writeGlobalIdentity(name, email string) error { if !errors.Is(err, os.ErrNotExist) { return err } - cfg = globalConfig{Version: globalConfigVersion} + cfg = internalconfig.Global{Version: globalConfigVersion} } - cfg.Identity = globalIdentityConfig{Name: strings.TrimSpace(name), Email: strings.TrimSpace(email)} + cfg.Identity = internalconfig.Identity{Name: strings.TrimSpace(name), Email: strings.TrimSpace(email)} return writeGlobalConfig(path, cfg) } diff --git a/internal/app/local_broker_native.go b/internal/app/local_broker_native.go new file mode 100644 index 0000000..d9327d1 --- /dev/null +++ b/internal/app/local_broker_native.go @@ -0,0 +1,785 @@ +package app + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "io/fs" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + localbroker "github.com/bucketgit/bgit/broker/local" + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/store" +) + +type localBrokerServer struct { + root string + objectRoot string + baseURL string + coreMu sync.Mutex + core *localbroker.Broker +} + +type localBrokerStoreAdapter struct{ store writableGitRemoteStore } + +func (a localBrokerStoreAdapter) Read(ctx context.Context, path string) ([]byte, error) { + return a.store.read(ctx, path) +} +func (a localBrokerStoreAdapter) List(ctx context.Context, prefix string) ([]string, error) { + return a.store.list(ctx, prefix) +} +func (a localBrokerStoreAdapter) Write(ctx context.Context, path string, data []byte) error { + return a.store.write(ctx, path, data) +} +func (a localBrokerStoreAdapter) Delete(ctx context.Context, path string) error { + return a.store.delete(ctx, path) +} +func (a localBrokerStoreAdapter) ListRefs(ctx context.Context) (map[string]string, error) { + refs, ok := a.store.(refListingGitRemoteStore) + if !ok { + return nil, store.ErrNotSupported + } + return refs.listRefs(ctx) +} +func (a localBrokerStoreAdapter) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + refs, ok := a.store.(refCASGitRemoteStore) + if !ok { + return store.ErrNotSupported + } + return refs.compareAndSwapRef(ctx, ref, oldOID, newOID) +} +func (a localBrokerStoreAdapter) CompareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error { + objects, ok := a.store.(objectCASGitRemoteStore) + if !ok { + return store.ErrNotSupported + } + return objects.compareAndSwap(ctx, path, expected, replacement) +} + +var _ store.Writer = localBrokerStoreAdapter{} +var _ store.RefStore = localBrokerStoreAdapter{} +var _ store.CompareAndSwapper = localBrokerStoreAdapter{} + +func (s *localBrokerServer) localCore() (*localbroker.Broker, error) { + s.coreMu.Lock() + defer s.coreMu.Unlock() + if s.core != nil { + return s.core, nil + } + core, err := localbroker.New(localbroker.Options{ + Root: s.root, + Resolver: localbroker.ResolverFunc(func(ctx context.Context, repo protocol.Repository) (store.Writer, func() error, bool, error) { + backend, closeBackend, ok, err := s.cloudStore(ctx, repo) + if !ok || err != nil { + return nil, nil, ok, err + } + closeFn := func() error { + if closeBackend != nil { + closeBackend() + } + return nil + } + return localBrokerStoreAdapter{store: backend}, closeFn, true, nil + }), + }) + if err != nil { + return nil, err + } + s.core = core + return core, nil +} + +func isLocalBrokerURL(value string) bool { + return strings.HasPrefix(strings.TrimSpace(value), "local://") +} + +func localBrokerPostContext(ctx context.Context, brokerURL, path string, req any, resp any) error { + server, err := localBrokerServerForURL(brokerURL) + if err != nil { + return err + } + out, status, err := server.localPostContext(ctx, path, req) + if err != nil { + return err + } + if status < 200 || status >= 300 { + msg := strings.TrimSpace(string(out)) + if msg == "" { + msg = http.StatusText(status) + } + return brokerHTTPError(path, msg) + } + if resp != nil && len(out) > 0 { + if err := json.Unmarshal(out, resp); err != nil { + return err + } + } + return nil +} + +func localBrokerCapabilityRead(ctx context.Context, capability protocol.ObjectCapabilityResponse) ([]byte, error) { + _ = ctx + server, repo, err := localBrokerCapabilityTarget(capability) + if err != nil { + return nil, err + } + return server.readObject(repo, capability.Object) +} + +func localBrokerCapabilityWrite(ctx context.Context, capability protocol.ObjectCapabilityResponse, data []byte) error { + _ = ctx + server, repo, err := localBrokerCapabilityTarget(capability) + if err != nil { + return err + } + return server.writeObject(repo, capability.Object, data) +} + +func localBrokerCapabilityDelete(ctx context.Context, capability protocol.ObjectCapabilityResponse) error { + _ = ctx + server, repo, err := localBrokerCapabilityTarget(capability) + if err != nil { + return err + } + return server.deleteObject(repo, capability.Object) +} + +func localBrokerCapabilityTarget(capability protocol.ObjectCapabilityResponse) (*localBrokerServer, protocol.Repository, error) { + server, err := localBrokerServerForURL(capability.URL) + if err != nil { + return nil, protocol.Repository{}, err + } + repo := protocol.Repository{Provider: capability.Provider, Bucket: capability.Bucket, Prefix: capability.Prefix, Profile: capability.Profile, Region: capability.Region} + return server, repo, nil +} + +func localBrokerServerForURL(brokerURL string) (*localBrokerServer, error) { + profileName, regionName := localProfileSelection(strings.TrimPrefix(strings.TrimSpace(brokerURL), "local://")) + path, err := defaultGlobalConfigPath() + if err != nil { + return nil, err + } + global, err := readGlobalConfig(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + profile, _, ok := globalLocalProfileForSelection(global, profileName, regionName) + if !ok { + profile = ensureDefaultLocalProfile(global) + } + root := expandHome(profile.Root) + if strings.TrimSpace(root) == "" { + root, err = defaultLocalBrokerRoot() + if err != nil { + return nil, err + } + } + server := &localBrokerServer{root: root, objectRoot: filepath.Join(root, "objects"), baseURL: localBrokerURL(profileName, regionName)} + if err := os.MkdirAll(server.objectRoot, 0o755); err != nil { + return nil, err + } + return server, nil +} + +func (s *localBrokerServer) localPost(path string, req any) ([]byte, int, error) { + return s.localPostContext(context.Background(), path, req) +} + +func (s *localBrokerServer) localPostContext(ctx context.Context, path string, req any) ([]byte, int, error) { + if err := ctx.Err(); err != nil { + return nil, 499, err + } + switch path { + case "/owners/upsert": + r := req.(protocol.OwnerRequest) + core, err := s.localCore() + if err != nil { + return nil, 500, err + } + if _, err := core.UpsertOwners(ctx, r); err != nil { + return nil, 500, err + } + return mustJSON(map[string]bool{"ok": true}), 200, nil + case "/teams/create": + return mustJSON(map[string]bool{"ok": true}), 200, nil + case "/teams/list": + return mustJSON(map[string]any{"teams": []protocol.Team{{ID: coreTeamID, Name: coreTeamName}}}), 200, nil + case "/teams/resolve": + return mustJSON(map[string]any{"team": protocol.Team{ID: coreTeamID, Name: coreTeamName}}), 200, nil + case "/repos/create": + r := req.(protocol.RepositoryAdminRequest) + repo := s.physicalRepo(r.Repo) + if err := s.ensureRepoStorage(ctx, repo); err != nil { + return nil, 500, err + } + var keys []protocol.Key + for _, key := range r.PublicKeys { + if key = normalizeKey(key); key != "" { + keys = append(keys, protocol.Key{User: firstNonEmpty(r.User, "owner"), Role: firstNonEmpty(r.Role, "developer"), PublicKey: key}) + } + } + if len(keys) == 0 { + owners, _ := s.loadOwners() + for _, key := range owners.Keys { + keys = append(keys, protocol.Key{User: key.User, Role: firstNonEmpty(r.Role, "developer"), PublicKey: key.PublicKey}) + } + } + core, err := s.localCore() + if err != nil { + return nil, 500, err + } + r.Repo = repo + state, err := core.CreateRepository(ctx, r, keys) + if err != nil { + if errors.Is(err, protocol.ErrConflict) { + return mustJSON(map[string]string{"error": "repository already exists"}), http.StatusConflict, nil + } + return nil, 500, err + } + return mustJSON(map[string]any{"ok": true, "repo": state.Repo}), 200, nil + case "/repos/get": + r := req.(protocol.RepositoryRequest) + core, err := s.localCore() + if err != nil { + return nil, 500, err + } + state, err := core.GetRepository(ctx, s.physicalRepo(r.Repo)) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + return mustJSON(map[string]any{"ok": true, "repo": state.Repo, "teams": state.Teams}), 200, nil + case "/repos/list": + core, err := s.localCore() + if err != nil { + return nil, 500, err + } + repos, err := core.ListRepositories(ctx) + if err != nil { + return nil, 500, err + } + return mustJSON(map[string]any{"repos": repos}), 200, nil + case "/auth/status": + r := req.(protocol.AuthStatusRequest) + state, _ := s.loadRepoForRequest(r.Repo) + return mustJSON(protocol.AuthStatus{BrokerURL: s.baseURL, BrokerVersion: brokerVersion(), Repo: state.Repo, User: "owner", Role: "owner", Capabilities: roleCapabilitiesForLocal("owner"), ResolvedAt: time.Now().UTC().Format(time.RFC3339)}), 200, nil + case "/auth/check": + return mustJSON(protocol.AuthResponse{Allowed: true, User: "owner", Role: "owner"}), 200, nil + case "/objects/capability": + r := req.(protocol.ObjectCapabilityRequest) + state, err := s.loadRepoForRequest(r.Repo) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + objectPath, err := validateLocalBrokerCapabilityPath(r.Operation, r.Path) + if err != nil { + return mustJSON(map[string]string{"error": err.Error()}), http.StatusForbidden, nil + } + object := localBrokerObjectName(state.Repo, objectPath) + return mustJSON(protocol.ObjectCapabilityResponse{Provider: state.Repo.Provider, Mode: "local", URL: s.baseURL, Bucket: state.Repo.Bucket, Prefix: state.Repo.Prefix, Object: object, Profile: state.Repo.Profile, Region: state.Repo.Region}), 200, nil + case "/objects/read": + r := req.(protocol.ObjectRequest) + state, err := s.loadRepoForRequest(r.Repo) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + data, err := s.readObject(state.Repo, r.Path) + if err != nil { + return mustJSON(map[string]string{"error": err.Error()}), http.StatusNotFound, nil + } + return mustJSON(protocol.ObjectResponse{Data: base64.StdEncoding.EncodeToString(data)}), 200, nil + case "/objects/list": + r := req.(protocol.ObjectRequest) + state, err := s.loadRepoForRequest(r.Repo) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + paths, err := s.listObjects(state.Repo, r.Prefix) + if err != nil { + return nil, 500, err + } + return mustJSON(protocol.ObjectResponse{Paths: paths}), 200, nil + case "/issues/list", "/issues/view", "/issues/create", "/issues/update", "/issues/comment", "/issues/close", "/issues/reopen", "/issues/move", "/issues/take", "/issues/assign", "/issues/archive", "/issues/reorder": + r := req.(protocol.IssueRequest) + state, err := s.loadRepoForRequest(r.Repo) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + return s.localIssueServiceEndpoint(ctx, path, r, state, "owner") + case "/refs/update": + r := req.(protocol.RefUpdateRequest) + state, err := s.loadRepoForRequest(r.Repo) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + r.Repo = state.Repo + core, err := s.localCore() + if err != nil { + return nil, 500, err + } + if err := core.UpdateRef(ctx, r, "owner"); err != nil { + status := 500 + if errors.Is(err, store.ErrConflict) { + status = http.StatusConflict + } + return mustJSON(map[string]string{"error": err.Error()}), status, nil + } + return mustJSON(map[string]bool{"ok": true}), 200, nil + case "/refs/list": + r := req.(protocol.RefsRequest) + state, err := s.loadRepoForRequest(r.Repo) + if err != nil { + return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil + } + core, err := s.localCore() + if err != nil { + return nil, 500, err + } + reconciled, err := core.ReconcileRepository(ctx, state.Repo, "local-sync") + if err != nil { + return nil, 500, err + } + return mustJSON(protocol.RefsResponse{Refs: reconciled.Refs}), 200, nil + default: + return mustJSON(map[string]string{"error": "unknown broker endpoint"}), http.StatusNotFound, nil + } +} + +func (s *localBrokerServer) localIssueServiceEndpoint(ctx context.Context, path string, req protocol.IssueRequest, state localbroker.RepositoryState, user string) ([]byte, int, error) { + core, err := s.localCore() + if err != nil { + return nil, http.StatusInternalServerError, err + } + switch path { + case "/issues/list": + issues, err := core.ListIssues(ctx, state.Repo, localbroker.IssueFilter{Type: req.Type, IncludeArchived: req.IncludeArchived}) + if err != nil { + return nil, localIssueErrorStatus(err), err + } + return mustJSON(map[string]any{"issues": issues}), http.StatusOK, nil + case "/issues/view": + issue, err := core.GetIssue(ctx, state.Repo, req.ID) + if err != nil { + return nil, localIssueErrorStatus(err), err + } + return mustJSON(map[string]any{"issue": issue}), http.StatusOK, nil + case "/issues/create": + issue, err := core.CreateIssue(ctx, state.Repo, req, user) + if err != nil { + return nil, localIssueErrorStatus(err), err + } + return mustJSON(map[string]any{"issue": issue}), http.StatusOK, nil + case "/issues/update": + err = core.UpdateIssue(ctx, state.Repo, req, user) + case "/issues/comment": + err = core.CommentIssue(ctx, state.Repo, req.ID, req.Comment, user) + case "/issues/close": + err = core.SetIssueClosed(ctx, state.Repo, req.ID, true, user) + case "/issues/reopen": + err = core.SetIssueClosed(ctx, state.Repo, req.ID, false, user) + case "/issues/reorder": + err = core.MoveStory(ctx, state.Repo, req.ID, localbroker.StoryMove{Lane: req.Lane, AfterID: req.AfterID, Order: req.Order}, user) + case "/issues/move": + if req.AfterID != nil { + err = core.MoveStory(ctx, state.Repo, req.ID, localbroker.StoryMove{Lane: req.Lane, AfterID: req.AfterID}, user) + } else { + err = core.MoveStoryToLaneEnd(ctx, state.Repo, req.ID, req.Lane, user) + } + case "/issues/take": + err = core.TakeStory(ctx, state.Repo, req.ID, user) + case "/issues/assign": + err = core.AssignIssue(ctx, state.Repo, req.ID, req.Assignee, user) + case "/issues/archive": + err = core.ArchiveIssue(ctx, state.Repo, req.ID, req.Archived, user) + default: + return mustJSON(map[string]string{"error": "unknown broker endpoint"}), http.StatusNotFound, nil + } + if err != nil { + return nil, localIssueErrorStatus(err), err + } + return mustJSON(map[string]bool{"ok": true}), http.StatusOK, nil +} + +func localIssueErrorStatus(err error) int { + message := strings.ToLower(err.Error()) + if strings.Contains(message, "not found") { + return http.StatusNotFound + } + if strings.Contains(message, "required") || strings.Contains(message, "unknown board lane") { + return http.StatusBadRequest + } + return http.StatusInternalServerError +} + +func (s *localBrokerServer) physicalRepo(repo protocol.Repository) protocol.Repository { + if repo.Logical == "" { + repo.Logical = firstNonEmpty(repo.Prefix, "repo.git") + } + if logical, err := normalizeLogicalRepoName(repo.Logical); err == nil { + repo.Logical = logical + } + repo.TeamID = firstNonEmpty(repo.TeamID, coreTeamID) + repo.TeamName = firstNonEmpty(repo.TeamName, coreTeamName) + if repo.Bucket == "" { + repo.Bucket = repo.Logical + repo.Provider = "file" + repo.Prefix = "" + return repo + } + if strings.HasPrefix(repo.Bucket, "s3://") || strings.HasPrefix(repo.Bucket, "gs://") { + if cfg, ok, err := localBrokerCloudConfig(repo.Bucket); err == nil && ok { + repo.Bucket = cfg.bucket + repo.Provider = cfg.provider + repo.Profile = cfg.gcloudConfiguration + repo.Region = cfg.region + } + repo.Prefix = "" + return repo + } + if strings.HasPrefix(repo.Bucket, "file://") { + repo.Provider = "file" + repo.Prefix = "" + return repo + } + if repo.Provider == "s3" || repo.Provider == "gcs" { + repo.Prefix = "" + return repo + } + if repo.Prefix == "" && repo.Provider == "" { + repo.Provider = "file" + } + return repo +} + +func (s *localBrokerServer) loadRepo(repo protocol.Repository) (localbroker.RepositoryState, error) { + core, err := s.localCore() + if err != nil { + return localbroker.RepositoryState{}, err + } + state, err := core.LoadRepositoryState(context.Background(), repo) + if err == nil { + state.Repo = s.physicalRepo(state.Repo) + } + return state, err +} + +func (s *localBrokerServer) loadRepoForRequest(repo protocol.Repository) (localbroker.RepositoryState, error) { + physical := s.physicalRepo(repo) + state, err := s.loadRepo(physical) + if err == nil { + return state, nil + } + logical := strings.TrimSpace(physical.Logical) + if logical == "" { + return localbroker.RepositoryState{}, err + } + if indexed, ok := s.indexedRepo(logical); ok { + state, indexErr := s.loadRepo(indexed) + if indexErr == nil && strings.EqualFold(strings.TrimSpace(state.Repo.Logical), logical) { + return state, nil + } + } + var found localbroker.RepositoryState + walkErr := filepath.WalkDir(s.objectRoot, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil || d.IsDir() || filepath.Base(path) != "repo.json" { + return nil + } + var candidate localbroker.RepositoryState + if readErr := s.readJSON(path, &candidate); readErr != nil { + return nil + } + if strings.EqualFold(strings.TrimSpace(candidate.Repo.Logical), logical) { + found = candidate + return fs.SkipAll + } + return nil + }) + if found.Repo.Logical != "" { + return found, nil + } + if walkErr != nil { + return localbroker.RepositoryState{}, walkErr + } + return localbroker.RepositoryState{}, err +} + +func (s *localBrokerServer) saveRepo(state localbroker.RepositoryState) error { + core, err := s.localCore() + if err != nil { + return err + } + return core.SaveRepositoryState(context.Background(), state) +} + +func (s *localBrokerServer) indexedRepo(logical string) (protocol.Repository, bool) { + logical = strings.TrimSpace(logical) + if logical == "" { + return protocol.Repository{}, false + } + core, err := s.localCore() + if err != nil { + return protocol.Repository{}, false + } + repo, ok, _ := core.FindRepository(context.Background(), logical) + return repo, ok +} + +func (s *localBrokerServer) bucketDir(bucket string) string { + core, err := s.localCore() + if err != nil { + return filepath.Join(s.objectRoot, base64.RawURLEncoding.EncodeToString([]byte(bucket))) + } + return core.BucketDir(bucket) +} + +func (s *localBrokerServer) ensureRepoStorage(ctx context.Context, repo protocol.Repository) error { + cfg, ok, err := localBrokerRepoCloudConfig(repo) + if err != nil || !ok { + return err + } + return ensureBucket(ctx, cfg) +} + +func (s *localBrokerServer) cloudStore(ctx context.Context, repo protocol.Repository) (writableGitRemoteStore, func(), bool, error) { + cfg, ok, err := localBrokerRepoCloudConfig(repo) + if err != nil || !ok { + return nil, nil, ok, err + } + switch cfg.provider { + case "s3": + client, err := newS3Client(ctx, cfg, false) + if err != nil { + return nil, nil, true, err + } + return &s3GitStore{client: client, bucket: cfg.bucket, prefix: cfg.prefix}, nil, true, nil + case "gcs": + client, err := newStorageClient(ctx, cfg) + if err != nil { + return nil, nil, true, err + } + return &gcsGitStore{client: client, bucket: cfg.bucket, prefix: cfg.prefix}, func() { _ = client.Close() }, true, nil + default: + return nil, nil, false, nil + } +} + +func localBrokerRepoCloudConfig(repo protocol.Repository) (config, bool, error) { + if cfg, ok, err := localBrokerCloudConfig(repo.Bucket); ok || err != nil { + return cfg, ok, err + } + provider := strings.TrimSpace(repo.Provider) + if provider != "s3" && provider != "gcs" { + return config{}, false, nil + } + region := strings.TrimSpace(repo.Region) + profile := strings.TrimSpace(repo.Profile) + if provider == "s3" { + region = firstNonEmpty(region, "us-east-1") + } else { + region = firstNonEmpty(region, "us-central1") + } + profile = firstNonEmpty(profile, "default") + return config{ + provider: provider, + bucket: strings.TrimSpace(repo.Bucket), + prefix: "", + region: region, + auth: defaultStorageAuthMode(), + gcloudConfiguration: profile, + }, true, nil +} + +func localBrokerCloudConfig(bucketURI string) (config, bool, error) { + scheme := "" + rest := "" + switch { + case strings.HasPrefix(bucketURI, "s3://"): + scheme = "s3" + rest = strings.TrimPrefix(bucketURI, "s3://") + case strings.HasPrefix(bucketURI, "gs://"): + scheme = "gs" + rest = strings.TrimPrefix(bucketURI, "gs://") + default: + return config{}, false, nil + } + host := strings.Split(rest, "/")[0] + if strings.TrimSpace(host) == "" { + return config{}, true, errors.New("local broker cloud repo URI must include a bucket name") + } + labels := strings.Split(host, ".") + clean := labels[:0] + for _, label := range labels { + if strings.TrimSpace(label) != "" { + clean = append(clean, strings.TrimSpace(label)) + } + } + if len(clean) == 0 { + return config{}, true, errors.New("local broker cloud repo URI must include a bucket name") + } + profile := "default" + region := "us-east-1" + provider := "s3" + if scheme == "gs" { + provider = "gcs" + region = "us-central1" + } + bucket := strings.Join(clean, ".") + if len(clean) >= 3 && localBrokerLooksCloudRegion(scheme, clean[1]) { + profile = clean[0] + region = clean[1] + bucket = strings.Join(clean[2:], ".") + } else if len(clean) >= 2 { + profile = clean[0] + bucket = strings.Join(clean[1:], ".") + } + if strings.TrimSpace(bucket) == "" { + return config{}, true, errors.New("local broker cloud repo URI must include a bucket name") + } + return config{ + provider: provider, + bucket: bucket, + prefix: "", + region: region, + auth: defaultStorageAuthMode(), + gcloudConfiguration: profile, + }, true, nil +} + +func localBrokerLooksCloudRegion(scheme, value string) bool { + if setupAWSRegionPattern.MatchString(value) || looksLikeGCPRegion(value) { + return true + } + if scheme != "gs" { + return false + } + if !strings.Contains(value, "-") { + return false + } + for i := len(value) - 1; i >= 0; i-- { + if value[i] >= '0' && value[i] <= '9' { + return true + } + } + return false +} + +func (s *localBrokerServer) readObject(repo protocol.Repository, object string) ([]byte, error) { + core, err := s.localCore() + if err != nil { + return nil, err + } + return core.ReadObject(context.Background(), repo, object) +} + +func (s *localBrokerServer) writeObject(repo protocol.Repository, object string, data []byte) error { + core, err := s.localCore() + if err != nil { + return err + } + return core.WriteObject(context.Background(), repo, object, data) +} + +func (s *localBrokerServer) deleteObject(repo protocol.Repository, object string) error { + core, err := s.localCore() + if err != nil { + return err + } + return core.DeleteObject(context.Background(), repo, object) +} + +func (s *localBrokerServer) listObjects(repo protocol.Repository, prefix string) ([]string, error) { + core, err := s.localCore() + if err != nil { + return nil, err + } + return core.ListObjects(context.Background(), repo, prefix) +} + +func (s *localBrokerServer) loadOwners() (localbroker.Owners, error) { + core, err := s.localCore() + if err != nil { + return localbroker.Owners{}, err + } + return core.LoadOwners(context.Background()) +} + +func (s *localBrokerServer) readJSON(path string, dst any) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + return json.Unmarshal(data, dst) +} + +func roleAllowsLocal(role, operation string) bool { + role = normalizeBrokerRole(role) + if role == "owner" || role == "admin" { + return true + } + if operation == "read" { + return role == "read" || role == "triage" || role == "developer" || role == "maintainer" + } + if operation == "write" { + return role == "developer" || role == "maintainer" + } + return false +} + +func roleCapabilitiesForLocal(role string) map[string]bool { + return map[string]bool{ + "read": roleAllowsLocal(role, "read"), + "push": roleAllowsLocal(role, "write"), + "merge": role == "maintainer" || role == "admin" || role == "owner", + } +} + +func validateLocalBrokerCapabilityPath(operation, objectPath string) (string, error) { + path := strings.TrimPrefix(filepath.ToSlash(filepath.Clean("/"+objectPath)), "/") + if path == "" || strings.Contains(path, "..") { + return "", errors.New("invalid object path") + } + if operation == "write" && !(path == "objects" || strings.HasPrefix(path, "objects/")) { + return "", errors.New("write capabilities are restricted to git object paths") + } + if operation == "read" { + if path == "HEAD" || path == "packed-refs" || strings.HasPrefix(path, "refs/") || path == "objects" || strings.HasPrefix(path, "objects/") { + return path, nil + } + return "", errors.New("read capabilities are restricted to git repository paths") + } + if operation == "delete" { + return "", errors.New("delete capabilities are not supported") + } + return path, nil +} + +func localBrokerObjectName(repo protocol.Repository, objectPath string) string { + prefix := strings.Trim(repo.Prefix, "/") + if prefix == "" { + return objectPath + } + return prefix + "/" + strings.TrimPrefix(objectPath, "/") +} + +func localBrokerIssuesPath() string { + return ".bucketgit/broker-state/v1/issues.json" +} + +func mustJSON(value any) []byte { + data, _ := json.MarshalIndent(value, "", " ") + return data +} + +func normalizeKey(key string) string { + key = strings.TrimSpace(key) + parts := strings.Fields(key) + if len(parts) >= 2 { + return parts[0] + " " + parts[1] + } + return key +} diff --git a/internal/app/local_broker_segmentation_test.go b/internal/app/local_broker_segmentation_test.go new file mode 100644 index 0000000..32ce574 --- /dev/null +++ b/internal/app/local_broker_segmentation_test.go @@ -0,0 +1,19 @@ +package app + +import ( + "errors" + "testing" + + "github.com/bucketgit/bgit/store" +) + +func TestLocalBrokerStoreAdapterProvidesMetadataCAS(t *testing.T) { + adapter := localBrokerStoreAdapter{store: &localGitStore{root: t.TempDir()}} + path := ".bucketgit/broker-state/v1/issues.json" + if err := adapter.CompareAndSwap(t.Context(), path, store.ObjectState{}, []byte(`{"next_id":1}`)); err != nil { + t.Fatal(err) + } + if err := adapter.CompareAndSwap(t.Context(), path, store.ObjectState{}, []byte(`{"next_id":2}`)); !errors.Is(err, store.ErrConflict) { + t.Fatalf("stale metadata write error=%v", err) + } +} diff --git a/local_config.go b/internal/app/local_config.go similarity index 55% rename from local_config.go rename to internal/app/local_config.go index a31917f..41287c7 100644 --- a/local_config.go +++ b/internal/app/local_config.go @@ -1,4 +1,4 @@ -package main +package app import ( "errors" @@ -6,13 +6,13 @@ import ( "io" "os" "path/filepath" - "sort" "strings" + + internalconfig "github.com/bucketgit/bgit/internal/config" ) type localConfigFile struct { - sections map[string]map[string]string - order []string + value *internalconfig.RepositoryFile } func (r *localRepository) config(args []string, stdout io.Writer) error { @@ -122,7 +122,7 @@ func globalConfigCommand(args []string, stdout io.Writer) error { if !errors.Is(err, os.ErrNotExist) { return err } - cfg = globalConfig{Version: globalConfigVersion} + cfg = internalconfig.Global{Version: globalConfigVersion} } if opts.list { if cfg.Identity.Name != "" { @@ -185,142 +185,26 @@ func parseGlobalConfigArgs(args []string) (configOptions, error) { } func readLocalConfigFile(path string) (localConfigFile, error) { - cfg := localConfigFile{sections: map[string]map[string]string{}} - data, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - return cfg, nil - } - if err != nil { - return cfg, err - } - section := "" - for _, line := range strings.Split(string(data), "\n") { - trimmed := strings.TrimSpace(line) - if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";") { - continue - } - if strings.HasPrefix(trimmed, "[") && strings.HasSuffix(trimmed, "]") { - section = strings.TrimSpace(strings.Trim(trimmed, "[]")) - cfg.ensureSection(section) - continue - } - name, value, ok := strings.Cut(trimmed, "=") - if !ok || section == "" { - continue - } - cfg.sections[section][strings.TrimSpace(name)] = strings.Trim(strings.TrimSpace(value), `"`) - } - return cfg, nil -} - -func (c *localConfigFile) ensureSection(section string) { - if c.sections == nil { - c.sections = map[string]map[string]string{} - } - if _, ok := c.sections[section]; ok { - return - } - c.sections[section] = map[string]string{} - c.order = append(c.order, section) + value, err := internalconfig.ReadRepositoryFile(path) + return localConfigFile{value: value}, err } func (c *localConfigFile) get(key string) (string, bool) { - section, name := configSectionAndName(key) - values, ok := c.sections[section] - if !ok { - return "", false - } - value, ok := values[name] - return value, ok + return c.value.Get(key) } func (c *localConfigFile) set(key, value string) { - section, name := configSectionAndName(key) - c.ensureSection(section) - c.sections[section][name] = value + c.value.Set(key, value) } func (c *localConfigFile) unset(key string) { - section, name := configSectionAndName(key) - if values, ok := c.sections[section]; ok { - delete(values, name) - } + c.value.Unset(key) } func (c *localConfigFile) keys() []string { - var keys []string - for _, section := range c.orderedSections() { - for name := range c.sections[section] { - keys = append(keys, configFullKey(section, name)) - } - } - sort.Strings(keys) - return keys + return c.value.Keys() } func (c *localConfigFile) write(path string) error { - var out strings.Builder - for _, section := range c.orderedSections() { - values := c.sections[section] - if len(values) == 0 { - continue - } - fmt.Fprintf(&out, "[%s]\n", section) - var names []string - for name := range values { - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - fmt.Fprintf(&out, "\t%s = %s\n", name, values[name]) - } - out.WriteByte('\n') - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, []byte(out.String()), 0o644) -} - -func (c *localConfigFile) orderedSections() []string { - seen := map[string]struct{}{} - var sections []string - for _, section := range c.order { - if _, ok := c.sections[section]; ok { - sections = append(sections, section) - seen[section] = struct{}{} - } - } - for section := range c.sections { - if _, ok := seen[section]; !ok { - sections = append(sections, section) - } - } - sort.SliceStable(sections, func(i, j int) bool { - _, iSeen := seen[sections[i]] - _, jSeen := seen[sections[j]] - if iSeen != jSeen { - return iSeen - } - return sections[i] < sections[j] - }) - return sections -} - -func configSectionAndName(key string) (string, string) { - parts := strings.Split(key, ".") - if len(parts) <= 2 { - if len(parts) == 1 { - return parts[0], "" - } - return parts[0], parts[1] - } - return fmt.Sprintf(`%s "%s"`, parts[0], parts[1]), strings.Join(parts[2:], ".") -} - -func configFullKey(section, name string) string { - if before, after, ok := strings.Cut(section, " "); ok { - return before + "." + strings.Trim(after, `"`) + "." + name - } - return section + "." + name + return c.value.Write(path) } diff --git a/local_extra.go b/internal/app/local_extra.go similarity index 99% rename from local_extra.go rename to internal/app/local_extra.go index b0a77b1..c040d36 100644 --- a/local_extra.go +++ b/internal/app/local_extra.go @@ -1,4 +1,4 @@ -package main +package app import ( "archive/tar" @@ -631,6 +631,7 @@ func (r *localRepository) stash(args []string, stdout io.Writer) error { case "pop", "apply": hash, err := r.readRef("refs/stash") if err != nil { + //lint:ignore ST1005 Match native Git's established user-facing error text. return errors.New("No stash entries found.") } if err := r.checkoutCommit(hash); err != nil { @@ -646,6 +647,7 @@ func (r *localRepository) stash(args []string, stdout io.Writer) error { return nil case "drop": if err := r.deleteRef("refs/stash"); err != nil { + //lint:ignore ST1005 Match native Git's established user-facing error text. return errors.New("No stash entries found.") } fmt.Fprintln(stdout, "Dropped refs/stash@{0}") @@ -849,6 +851,7 @@ func (r *localRepository) describe(args []string, stdout io.Writer) error { } } if len(candidates) == 0 { + //lint:ignore ST1005 Match native Git's established user-facing error text. return errors.New("No names found, cannot describe anything.") } sort.Slice(candidates, func(i, j int) bool { return candidates[i].dist < candidates[j].dist }) diff --git a/local_native.go b/internal/app/local_native.go similarity index 99% rename from local_native.go rename to internal/app/local_native.go index 812da7d..16cb5cc 100644 --- a/local_native.go +++ b/internal/app/local_native.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" diff --git a/main.go b/internal/app/main.go similarity index 86% rename from main.go rename to internal/app/main.go index 44146d8..e98d437 100644 --- a/main.go +++ b/internal/app/main.go @@ -1,20 +1,21 @@ -package main +package app import ( "bytes" "context" - _ "embed" "errors" "flag" "fmt" "io" - "net/url" "os" "os/exec" "path/filepath" "strings" "cloud.google.com/go/storage" + internalcli "github.com/bucketgit/bgit/internal/cli" + internalconfig "github.com/bucketgit/bgit/internal/config" + internalsetup "github.com/bucketgit/bgit/internal/setup" "golang.org/x/oauth2" "google.golang.org/api/googleapi" "google.golang.org/api/option" @@ -31,11 +32,20 @@ func defaultStorageAuthMode() string { return defaultAuthMode } -//go:embed CHANGELOG.md var embeddedChangelog string var version = "" +type RuntimeOptions struct { + Version string + Changelog string +} + +func ConfigureRuntime(options RuntimeOptions) { + version = options.Version + embeddedChangelog = options.Changelog +} + func appVersion() string { if strings.TrimSpace(version) != "" { return strings.TrimSpace(version) @@ -73,21 +83,21 @@ type config struct { versionRequested bool } -func main() { - if filepath.Base(os.Args[0]) == "git-remote-bgit" { - if err := remoteHelperCommand(os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { - fmt.Fprintln(os.Stderr, "fatal:", err) - os.Exit(1) - } - return - } - if err := run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { +func Main() { + if err := RunExecutable(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { fmt.Fprintln(os.Stderr, "fatal:", err) os.Exit(1) } } -func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { +func RunExecutable(executable string, args []string, stdin io.Reader, stdout, stderr io.Writer) error { + if filepath.Base(executable) == "git-remote-bgit" { + return remoteHelperCommand(args, stdin, stdout, stderr) + } + return Run(args, stdin, stdout, stderr) +} + +func Run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { cfg, rest, err := parseGlobalFlags(args) if err != nil { return err @@ -109,12 +119,13 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { return helpCommand(rest[1:], stdout) } - cmd := rest[0] - cmdArgs := rest[1:] - if commandWantsVersion(cmdArgs) { + intent := internalcli.ParseIntent(rest) + cmd := intent.Command + cmdArgs := intent.Args + if intent.Version { return versionCommand(stdout) } - if commandWantsHelp(cmdArgs) { + if intent.Help { return commandHelp(cmd, stdout) } explicitBucket := cfg.bucket != "" || cfg.prefix != "" @@ -302,6 +313,10 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { return fmt.Errorf("unknown command %q", cmd) } +func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { + return Run(args, stdin, stdout, stderr) +} + func applyExplicitBrokerProfileSelection(cfg *config, cmd string) error { path, err := defaultGlobalConfigPath() if err != nil { @@ -361,23 +376,10 @@ func ensureLocalBrokerForCommand(ctx context.Context, cfg *config) (*managedLoca } func localProfileSelection(value string) (string, string) { - value = strings.TrimSpace(value) - value = strings.TrimPrefix(value, "local:") - if value == "" { - return "default", "default" - } - if strings.Contains(value, "/") { - parts := strings.SplitN(value, "/", 2) - return firstNonEmpty(parts[0], "default"), firstNonEmpty(parts[1], "default") - } - if strings.Contains(value, ".") { - parts := strings.SplitN(value, ".", 2) - return firstNonEmpty(parts[0], "default"), firstNonEmpty(parts[1], "default") - } - return value, "default" + return internalconfig.LocalSelection(value) } -func globalLocalProfileForSelection(global globalConfig, profileName, regionName string) (globalLocalProfile, globalProfileRegion, bool) { +func globalLocalProfileForSelection(global internalconfig.Global, profileName, regionName string) (internalconfig.LocalProfile, internalconfig.ProfileRegion, bool) { profileName = firstNonEmpty(strings.TrimSpace(profileName), "default") regionName = firstNonEmpty(strings.TrimSpace(regionName), "default") for _, profile := range global.LocalProfiles { @@ -392,13 +394,9 @@ func globalLocalProfileForSelection(global globalConfig, profileName, regionName if len(profile.Regions) > 0 { return profile, profile.Regions[0], true } - return profile, globalProfileRegion{Name: regionName}, true + return profile, internalconfig.ProfileRegion{Name: regionName}, true } - return globalLocalProfile{}, globalProfileRegion{}, false -} - -func mergeBrokerProfileArg(args []string, cfg config) []string { - return mergeBrokerSelectionArgs(args, cfg) + return internalconfig.LocalProfile{}, internalconfig.ProfileRegion{}, false } func mergeBrokerSelectionArgs(args []string, cfg config) []string { @@ -507,124 +505,28 @@ func isNativeRemoteCommand(cmd string) bool { } func parseGlobalFlags(args []string) (config, []string, error) { - cfg := config{branch: defaultBranch} envAuth := strings.TrimSpace(os.Getenv("BGIT_AUTH")) envConfiguration := strings.TrimSpace(os.Getenv("BGIT_GCLOUD_CONFIGURATION")) - cfg.auth = firstNonEmpty(envAuth, defaultAuthMode) - cfg.gcloudConfiguration = envConfiguration - rest, err := extractGlobalFlags(args, &cfg) + parsed, rest, err := internalcli.ParseGlobalOptions(args, internalcli.GlobalOptionDefaults{ + Branch: defaultBranch, + Auth: firstNonEmpty(envAuth, defaultAuthMode), + Profile: envConfiguration, + AuthExplicit: envAuth != "", + ProfileExplicit: envConfiguration != "", + }) if err != nil { - return cfg, nil, err + return config{}, nil, err } - cfg.authExplicit = envAuth != "" || cfg.authExplicit - cfg.gcloudConfigurationExplicit = envConfiguration != "" || cfg.gcloudConfigurationExplicit - cfg.prefix = strings.Trim(cfg.prefix, "/") - cfg.auth = strings.ToLower(strings.TrimSpace(cfg.auth)) - if cfg.auth == "" { - cfg.auth = defaultAuthMode - } - if cfg.auth != "gcloud" && cfg.auth != "adc" { - return cfg, nil, fmt.Errorf("unsupported auth mode %q", cfg.auth) + cfg := config{ + provider: parsed.Provider, bucket: parsed.Bucket, prefix: parsed.Prefix, + branch: parsed.Branch, region: parsed.Region, auth: parsed.Auth, + gcloudConfiguration: parsed.Profile, identity: parsed.Identity, + authExplicit: parsed.AuthExplicit, authFlagExplicit: parsed.AuthFlagExplicit, + gcloudConfigurationExplicit: parsed.ProfileExplicit, versionRequested: parsed.VersionRequested, } return cfg, rest, nil } -func extractGlobalFlags(args []string, cfg *config) ([]string, error) { - var rest []string - for i := 0; i < len(args); i++ { - arg := args[i] - name, value, hasValue := strings.Cut(arg, "=") - switch name { - case "--bucket": - if !hasValue { - i++ - if i >= len(args) { - return nil, errors.New("--bucket requires a value") - } - value = args[i] - } - applyBucketFlag(cfg, value) - case "--prefix": - if !hasValue { - i++ - if i >= len(args) { - return nil, errors.New("--prefix requires a value") - } - value = args[i] - } - cfg.prefix = value - case "--branch": - if !hasValue { - i++ - if i >= len(args) { - return nil, errors.New("--branch requires a value") - } - value = args[i] - } - cfg.branch = value - case "--region": - if !hasValue { - i++ - if i >= len(args) { - return nil, errors.New("--region requires a value") - } - value = args[i] - } - cfg.region = value - case "--auth": - if !hasValue { - i++ - if i >= len(args) { - return nil, errors.New("--auth requires a value") - } - value = args[i] - } - cfg.auth = value - cfg.authExplicit = true - cfg.authFlagExplicit = true - case "--configuration", "--profile": - if !hasValue { - i++ - if i >= len(args) { - return nil, fmt.Errorf("%s requires a value", name) - } - value = args[i] - } - cfg.gcloudConfiguration = value - cfg.gcloudConfigurationExplicit = true - case "--identity": - if !hasValue { - i++ - if i >= len(args) { - return nil, errors.New("--identity requires a value") - } - value = args[i] - } - cfg.identity = value - case "--version", "-v": - cfg.versionRequested = true - default: - rest = append(rest, arg) - } - } - return rest, nil -} - -func applyBucketFlag(cfg *config, value string) { - provider, bucket, prefix := normalizeAdminTarget(value) - if provider != "" { - cfg.provider = provider - } - if bucket != "" { - cfg.bucket = bucket - } else { - cfg.bucket = value - } - if prefix != "" && cfg.prefix == "" { - cfg.prefix = prefix - } -} - func mergeConfig(primary, fallback config) config { if primary.provider == "" { primary.provider = fallback.provider @@ -682,32 +584,11 @@ func parseGCSRepoURI(raw string) (config, string, error) { } func parseRepoURI(raw string) (config, string, error) { - parsed, err := url.Parse(raw) + parsed, name, err := internalconfig.ParseRepositoryURI(raw) if err != nil { return config{}, "", err } - provider := "" - scheme := parsed.Scheme - switch scheme { - case "gs", "gcs": - provider = "gcs" - scheme = "gs" - case "s3": - provider = "s3" - default: - return config{}, "", errors.New("repository URI must start with gs:// or s3://") - } - if parsed.Host == "" { - return config{}, "", errors.New("repository URI must include a bucket name") - } - prefix := strings.Trim(parsed.Path, "/") - if prefix == "" { - return config{}, "", errors.New("repository URI must include a repository prefix") - } - parts := strings.Split(prefix, "/") - repoName := parts[len(parts)-1] - origin := fmt.Sprintf("%s://%s/%s", scheme, parsed.Host, prefix) - return config{provider: provider, bucket: parsed.Host, prefix: prefix, branch: defaultBranch, origin: origin}, repoName, nil + return config{provider: parsed.Provider, bucket: parsed.Bucket, prefix: parsed.Prefix, branch: parsed.Branch, origin: parsed.Origin}, name, nil } func readLocalConfig(dir string) (config, error) { @@ -878,6 +759,7 @@ func defaultBranchAuth(dir string) config { } func missingOriginError() error { + //lint:ignore ST1005 Multiline CLI remediation is intentionally formatted and capitalized. return errors.New(`No configured push destination. Either specify the repository from the command-line: @@ -1028,83 +910,15 @@ func gcloudAccessToken(configuration string) (string, error) { } func gcloudCommand(configuration string, args ...string) *exec.Cmd { - cmd := exec.Command("gcloud", args...) - if strings.TrimSpace(configuration) != "" { - cmd.Env = append(os.Environ(), "CLOUDSDK_ACTIVE_CONFIG_NAME="+strings.TrimSpace(configuration)) - } - return cmd + return internalsetup.GCloudCommand(context.Background(), configuration, args...) } func usage(w io.Writer) error { - _, err := fmt.Fprint(w, `usage: bgit [args]. These are common BucketGit commands: - -start a repository - setup Connect a cloud account and deploy or update BucketGit - init Create a local Git repository backed by BucketGit - clone Clone a BucketGit repository into a new directory - -local web UI - web Browse and manage a repository locally - -work on the current change - add Add file contents to the index - mv Move or rename a file, directory, or symlink - restore Restore working tree files - rm Remove files from the working tree and index - -examine history and state - diff Show changes between commits, commit and working tree, etc - grep Print lines matching a pattern - log Show commit logs - show Show objects - status Show the working tree status - -grow, mark, and tweak history - branch List, create, or delete branches - checkout Switch branches or restore paths - commit Record changes to the repository - merge Join development histories together - reset Reset HEAD, index, or working tree state - tag Create, list, delete, or verify tags - -collaborate - fetch Download objects and refs from BucketGit - pull Fetch and integrate with the current branch - push Update remote refs and upload objects - ls-remote List remote refs - pr Create, review, merge, and close pull requests - ci Run and inspect broker CI builds - board Manage the repository task board - issue Create, comment on, close, and reopen issues - -administer - whoami Show broker identity, role, and capabilities for this repo - repos List repositories visible to local SSH keys - admin Manage broker-backed users, keys, owners, and protection - janitor Run broker maintenance and repair tasks - broker Delete or decommission deployed broker infrastructure - direct Run direct bucket recovery and administration commands - -Run "bgit help " or "bgit direct help" for details. -`) - return err + return internalcli.Usage(w) } func usageWithBanner(w io.Writer) error { - if _, err := fmt.Fprintf(w, - " _____________________________________________________________\n"+ - " | ▄▄▄▄▄▄▄ ▄▄▄▄▄▄▄ |\n"+ - " | ███▀▀███▄ ▄▄ ██ ███▀▀▀▀▀ ▀▀ ██ |\n"+ - "________| ███▄▄███▀ ██ ██ ▄████ ██ ▄█▀ ▄█▀█▄ ▀██▀▀ ███ ██ ▀██▀▀ |________\n"+ - "\\ | ███ ███▄ ██ ██ ██ ████ ██▄█▀ ██ ███ ███▀ ██ ██ | /\n"+ - " \\ | ████████▀ ▀██▀█ ▀████ ██ ▀█▄ ▀█▄▄▄ ██ ▀██████▀ ██▄ ██ | /\n"+ - " / |_____________________________________________________________| \\\n"+ - "/________) [ bucketgit.com / github.com/bucketgit / bgit %s ] (________\\\n"+ - "\n", - bannerVersion()); err != nil { - return err - } - return usage(w) + return internalcli.UsageWithBanner(w, bannerVersion()) } func helpCommand(args []string, stdout io.Writer) error { @@ -1117,14 +931,6 @@ func helpCommand(args []string, stdout io.Writer) error { return commandHelp(args[0], stdout) } -func commandWantsHelp(args []string) bool { - return len(args) == 1 && (args[0] == "help" || args[0] == "-h" || args[0] == "--help") -} - -func commandWantsVersion(args []string) bool { - return len(args) == 1 && (args[0] == "version" || args[0] == "-v" || args[0] == "--version") -} - func versionCommand(stdout io.Writer) error { _, err := fmt.Fprintf(stdout, "bgit %s\n", appVersion()) return err @@ -1499,7 +1305,7 @@ func remoteCommand(args []string, stdout io.Writer) error { if len(args) == 3 && args[0] == "set-url" && args[1] == "origin" { return originCommand([]string{args[2]}, stdout) } - return errors.New("supported remote commands: remote add origin gs://...|s3://... or remote set-url origin gs://...|s3://...") + return errors.New("supported remote commands: remote add origin gs://...|s3://... or remote set-url origin gs://...|s3://...") //lint:ignore ST1005 CLI metavariables intentionally end in ellipses. } func createGcloudProfileCommand(args []string, stdin io.Reader, stdout io.Writer) error { @@ -1633,7 +1439,7 @@ func recordCreatedCloudProfile(provider, profile, region string, stdout io.Write if !errors.Is(err, os.ErrNotExist) { return err } - global = globalConfig{Version: globalConfigVersion} + global = internalconfig.Global{Version: globalConfigVersion} } if global.Version == 0 { global.Version = globalConfigVersion @@ -1647,11 +1453,11 @@ func recordCreatedCloudProfile(provider, profile, region string, stdout io.Write } account := gcloudConfigValue(context.Background(), profile, "account") region = firstNonEmpty(strings.TrimSpace(region), gcloudConfigValue(context.Background(), profile, "run/region"), gcloudConfigValue(context.Background(), profile, "functions/region"), "us-central1") - global = upsertGlobalGCPProfile(global, globalGCPProfile{ + global = upsertGlobalGCPProfile(global, internalconfig.GCPProfile{ Name: profile, ProjectID: projectID, Account: account, - Regions: []globalProfileRegion{{Name: region}}, + Regions: []internalconfig.ProfileRegion{{Name: region}}, }) case "s3": accountID, arn := awsCallerIdentity(context.Background(), profile) @@ -1659,11 +1465,11 @@ func recordCreatedCloudProfile(provider, profile, region string, stdout io.Write return fmt.Errorf("AWS profile %s has no account identity; check credentials for that profile", profile) } region = firstNonEmpty(strings.TrimSpace(region), configuredAWSProfileRegion(profile), "us-east-1") - global = upsertGlobalAWSProfile(global, globalAWSProfile{ + global = upsertGlobalAWSProfile(global, internalconfig.AWSProfile{ Name: profile, AccountID: accountID, ARN: arn, - Regions: []globalProfileRegion{{Name: region}}, + Regions: []internalconfig.ProfileRegion{{Name: region}}, }) default: return fmt.Errorf("unsupported profile provider %q", provider) @@ -1733,6 +1539,7 @@ func isUnsupportedCommand(cmd string) bool { } func unsupportedCommand(cmd string) error { + //lint:ignore ST1005 Preserve the established CLI error shown by Git passthrough failures. return fmt.Errorf("Unsupported: '%s' is not supported by bgit", cmd) } @@ -2152,14 +1959,6 @@ func shortRefName(ref string) string { return ref } -func isMissingRef(err error) bool { - msg := err.Error() - return strings.Contains(msg, "Not a valid object name") || - strings.Contains(msg, "unknown revision") || - strings.Contains(msg, "Needed a single revision") || - strings.Contains(msg, "bad revision") -} - func isNoRefs(err error) bool { msg := err.Error() return strings.Contains(msg, "show-ref") && strings.Contains(msg, "exit status 1") diff --git a/main_test.go b/internal/app/main_test.go similarity index 98% rename from main_test.go rename to internal/app/main_test.go index 7f2c917..8878f0f 100644 --- a/main_test.go +++ b/internal/app/main_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" @@ -18,6 +18,8 @@ import ( "strings" "testing" + localbroker "github.com/bucketgit/bgit/broker/local" + "github.com/bucketgit/bgit/protocol" "google.golang.org/api/googleapi" ) @@ -901,7 +903,7 @@ func TestAuthorizeSSHGitServiceUsesBrokerRoles(t *testing.T) { if r.URL.Path != "/auth/check" { t.Fatalf("unexpected broker path %s", r.URL.Path) } - var req brokerAuthRequest + var req protocol.AuthRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -1230,6 +1232,12 @@ func TestMain(m *testing.M) { if path, ok := fakeCLIActionPath(); ok { os.Exit(runFakeCLI(path, os.Args[1:])) } + changelog, err := os.ReadFile("../../CHANGELOG.md") + if err != nil { + panic(err) + } + embeddedChangelog = string(changelog) + brokerAssets = assetFileSystem{FS: os.DirFS("../..")} os.Exit(m.Run()) } @@ -1377,21 +1385,21 @@ func TestLocalBrokerRelativeFileRepoUsesBrokerRoot(t *testing.T) { func TestLocalBrokerIssuesSupportBoardOperations(t *testing.T) { root := t.TempDir() server := &localBrokerServer{root: root, objectRoot: filepath.Join(root, "objects")} - repo := server.physicalRepo(brokerRepo{Provider: "file", Bucket: "file://demo.git", Prefix: "demo.git", Logical: "demo.git"}) - state := localBrokerRepoState{ + repo := server.physicalRepo(protocol.Repository{Provider: "file", Bucket: "file://demo.git", Prefix: "demo.git", Logical: "demo.git"}) + state := localbroker.RepositoryState{ Repo: repo, - Keys: []brokerKey{{User: "owner", Role: "owner", PublicKey: "test-key"}}, + Keys: []protocol.Key{{User: "owner", Role: "owner", PublicKey: "test-key"}}, Refs: map[string]string{}, } if err := server.saveRepo(state); err != nil { t.Fatal(err) } - out, status, err := server.localPost("/issues/list", brokerIssueRequest{Repo: repo, Type: "story"}) + out, status, err := server.localPost("/issues/list", protocol.IssueRequest{Repo: repo, Type: "story"}) if err != nil || status != 200 { t.Fatalf("initial list status=%d err=%v body=%s", status, err, out) } var listResp struct { - Issues []brokerIssue `json:"issues"` + Issues []protocol.Issue `json:"issues"` } if err := json.Unmarshal(out, &listResp); err != nil { t.Fatal(err) @@ -1400,12 +1408,12 @@ func TestLocalBrokerIssuesSupportBoardOperations(t *testing.T) { t.Fatalf("initial issues = %#v", listResp.Issues) } - out, status, err = server.localPost("/issues/create", brokerIssueRequest{Repo: repo, Type: "story", Title: "Local board works", Body: "Local board works", Lane: "backlog"}) + out, status, err = server.localPost("/issues/create", protocol.IssueRequest{Repo: repo, Type: "story", Title: "Local board works", Body: "Local board works", Lane: "backlog"}) if err != nil || status != 200 { t.Fatalf("create status=%d err=%v body=%s", status, err, out) } var createResp struct { - Issue brokerIssue `json:"issue"` + Issue protocol.Issue `json:"issue"` } if err := json.Unmarshal(out, &createResp); err != nil { t.Fatal(err) @@ -1413,17 +1421,17 @@ func TestLocalBrokerIssuesSupportBoardOperations(t *testing.T) { if createResp.Issue.ID != 1 || createResp.Issue.Lane != "backlog" || createResp.Issue.Position == 0 { t.Fatalf("created issue = %#v", createResp.Issue) } - if _, status, err = server.localPost("/issues/move", brokerIssueRequest{Repo: repo, ID: createResp.Issue.ID, Lane: "doing"}); err != nil || status != 200 { + if _, status, err = server.localPost("/issues/move", protocol.IssueRequest{Repo: repo, ID: createResp.Issue.ID, Lane: "doing"}); err != nil || status != 200 { t.Fatalf("move status=%d err=%v", status, err) } - if _, status, err = server.localPost("/issues/assign", brokerIssueRequest{Repo: repo, ID: createResp.Issue.ID, Assignee: "owner"}); err != nil || status != 200 { + if _, status, err = server.localPost("/issues/assign", protocol.IssueRequest{Repo: repo, ID: createResp.Issue.ID, Assignee: "owner"}); err != nil || status != 200 { t.Fatalf("assign status=%d err=%v", status, err) } - if _, status, err = server.localPost("/issues/archive", brokerIssueRequest{Repo: repo, ID: createResp.Issue.ID, Archived: true}); err != nil || status != 200 { + if _, status, err = server.localPost("/issues/archive", protocol.IssueRequest{Repo: repo, ID: createResp.Issue.ID, Archived: true}); err != nil || status != 200 { t.Fatalf("archive status=%d err=%v", status, err) } - out, status, err = server.localPost("/issues/list", brokerIssueRequest{Repo: repo, Type: "story"}) + out, status, err = server.localPost("/issues/list", protocol.IssueRequest{Repo: repo, Type: "story"}) if err != nil || status != 200 { t.Fatalf("list status=%d err=%v body=%s", status, err, out) } @@ -1433,7 +1441,7 @@ func TestLocalBrokerIssuesSupportBoardOperations(t *testing.T) { if len(listResp.Issues) != 0 { t.Fatalf("non-archived list = %#v", listResp.Issues) } - out, status, err = server.localPost("/issues/list", brokerIssueRequest{Repo: repo, Type: "story", IncludeArchived: true}) + out, status, err = server.localPost("/issues/list", protocol.IssueRequest{Repo: repo, Type: "story", IncludeArchived: true}) if err != nil || status != 200 { t.Fatalf("archived list status=%d err=%v body=%s", status, err, out) } @@ -1451,35 +1459,36 @@ func TestLocalBrokerIssuesSupportBoardOperations(t *testing.T) { func TestLocalBrokerReorderNormalizesDenseStoryOrder(t *testing.T) { root := t.TempDir() server := &localBrokerServer{root: root, objectRoot: filepath.Join(root, "objects")} - repo := server.physicalRepo(brokerRepo{Provider: "file", Bucket: "file://ordered.git", Prefix: "ordered.git", Logical: "ordered.git"}) - state := localBrokerRepoState{ + repo := server.physicalRepo(protocol.Repository{Provider: "file", Bucket: "file://ordered.git", Prefix: "ordered.git", Logical: "ordered.git"}) + state := localbroker.RepositoryState{ Repo: repo, - Keys: []brokerKey{{User: "owner", Role: "owner", PublicKey: "test-key"}}, + Keys: []protocol.Key{{User: "owner", Role: "owner", PublicKey: "test-key"}}, Refs: map[string]string{}, } if err := server.saveRepo(state); err != nil { t.Fatal(err) } - store := localBrokerIssueStore{NextID: 5, Issues: []brokerIssue{ - {ID: 1, Type: "story", Title: "one", Lane: "doing", Position: 10}, - {ID: 2, Type: "story", Title: "two", Lane: "doing", Position: 20}, - {ID: 3, Type: "story", Title: "three", Lane: "doing", Position: 30}, - {ID: 4, Type: "story", Title: "four", Lane: "doing", Position: 40}, - }} - if err := server.saveIssueStore(repo, store); err != nil { - t.Fatal(err) + for _, title := range []string{"one", "two", "three", "four"} { + if out, status, err := server.localPost("/issues/create", protocol.IssueRequest{Repo: repo, Type: "story", Title: title, Lane: "doing"}); err != nil || status != 200 { + t.Fatalf("create %q status=%d err=%v body=%s", title, status, err, out) + } } - out, status, err := server.localPost("/issues/reorder", brokerIssueRequest{Repo: repo, ID: 4, Lane: "doing", Order: 3}) + out, status, err := server.localPost("/issues/reorder", protocol.IssueRequest{Repo: repo, ID: 4, Lane: "doing", Order: 3}) if err != nil || status != 200 { t.Fatalf("reorder status=%d err=%v body=%s", status, err, out) } - store, err = server.loadIssueStore(repo) - if err != nil { + out, status, err = server.localPost("/issues/list", protocol.IssueRequest{Repo: repo, Type: "story"}) + if err != nil || status != 200 { + t.Fatalf("list status=%d err=%v body=%s", status, err, out) + } + var response struct { + Issues []protocol.Issue `json:"issues"` + } + if err := json.Unmarshal(out, &response); err != nil { t.Fatal(err) } - sortBoardStories(store.Issues) got := []string{} - for _, issue := range store.Issues { + for _, issue := range response.Issues { got = append(got, fmt.Sprintf("%d:%.0f", issue.ID, issue.Position)) } want := []string{"1:1", "2:2", "4:3", "3:4"} @@ -3595,7 +3604,7 @@ func TestNativeGitRepoBrokerPushDoesNotWriteRefThroughObjectStore(t *testing.T) t.Fatal(err) } - var gotUpdate brokerRefUpdateRequest + var gotUpdate protocol.RefUpdateRequest updateCalls := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -4406,7 +4415,7 @@ func TestWebPullRequestCacheRendersPRTabAndPage(t *testing.T) { nil, config{branch: "main", brokerURL: "https://broker.example.test", logicalRepo: "app.git", provider: "gcs"}, ) - if err := handler.writePullRequestCache([]brokerPullRequest{{ + if err := handler.writePullRequestCache([]protocol.PullRequest{{ ID: 7, Title: "Add docs", Source: "refs/heads/docs", @@ -4497,7 +4506,7 @@ func TestBrokerGitStoreReadsAndListsThroughBroker(t *testing.T) { w.Header().Set("content-type", "application/json") switch r.URL.Path { case "/objects/capability": - var req brokerObjectCapabilityRequest + var req protocol.ObjectCapabilityRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -4508,7 +4517,7 @@ func TestBrokerGitStoreReadsAndListsThroughBroker(t *testing.T) { case "/object": _, _ = w.Write([]byte("object data")) case "/objects/list": - var req brokerObjectRequest + var req protocol.ObjectRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Fatal(err) } @@ -4561,7 +4570,7 @@ func TestBrokerGitStoreMapsMissingObjectToNotExist(t *testing.T) { } func TestBrokerGitStoreListRefsUsesBrokerEndpoint(t *testing.T) { - var got brokerRefsRequest + var got protocol.RefsRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/refs/list" { t.Fatalf("unexpected path %s", r.URL.Path) diff --git a/native_git.go b/internal/app/native_git.go similarity index 90% rename from native_git.go rename to internal/app/native_git.go index c4b08ef..447f700 100644 --- a/native_git.go +++ b/internal/app/native_git.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" @@ -20,7 +20,10 @@ import ( "sync" "cloud.google.com/go/storage" - "google.golang.org/api/iterator" + repositorypkg "github.com/bucketgit/bgit/repository" + "github.com/bucketgit/bgit/store" + fsstore "github.com/bucketgit/bgit/store/fs" + gcsstore "github.com/bucketgit/bgit/store/gcs" ) const ( @@ -50,6 +53,15 @@ type refListingGitRemoteStore interface { listRefs(ctx context.Context) (map[string]string, error) } +type refCASGitRemoteStore interface { + refListingGitRemoteStore + compareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error +} + +type objectCASGitRemoteStore interface { + compareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error +} + type fallbackGitRemoteStore struct { primary gitRemoteStore fallback gitRemoteStore @@ -58,12 +70,27 @@ type fallbackGitRemoteStore struct { type nativeGitRepo struct { cfg config store gitRemoteStore + engine *repositorypkg.Repository mu sync.Mutex cache map[string]gitObject offsetCache map[string]gitObject packs []packIndex } +type nativeStoreReader struct { + store gitRemoteStore +} + +func (s nativeStoreReader) Read(ctx context.Context, path string) ([]byte, error) { + return s.store.read(ctx, path) +} + +func (s nativeStoreReader) List(ctx context.Context, prefix string) ([]string, error) { + return s.store.list(ctx, prefix) +} + +var _ store.Reader = nativeStoreReader{} + type gcsGitStore struct { client *storage.Client bucket string @@ -74,6 +101,9 @@ type localGitStore struct { root string } +var _ objectCASGitRemoteStore = (*gcsGitStore)(nil) +var _ objectCASGitRemoteStore = (*localGitStore)(nil) + func (s *fallbackGitRemoteStore) read(ctx context.Context, path string) ([]byte, error) { data, err := s.primary.read(ctx, path) if err == nil || errors.Is(err, fs.ErrNotExist) { @@ -292,8 +322,7 @@ func (r *nativeGitRepo) initWorktree(ctx context.Context, args []string, stdout if err := setGitBranchTracking(absTarget, cfg.branch, "origin"); err != nil { return err } - cloneRepo := *r - cloneRepo.cfg = cfg + cloneRepo := openNativeGitRepo(r.store, cfg) if err := cloneRepo.fetchIntoWorktree(ctx, absTarget, true, io.Discard); err != nil { return err } @@ -950,15 +979,16 @@ func (r *nativeGitRepo) object(ctx context.Context, hash string) (gitObject, err if obj, ok := r.cache[hash]; ok { return obj, nil } - obj, err := r.looseObject(ctx, hash) + publicObject, err := r.engine.Object(ctx, repositorypkg.OID(hash)) if err == nil { + obj := gitObject{typ: string(publicObject.Type), data: publicObject.Data} r.cache[hash] = obj return obj, nil } if !errors.Is(err, fs.ErrNotExist) { return gitObject{}, err } - obj, err = r.packedObject(ctx, hash) + obj, err := r.packedObject(ctx, hash) if err != nil { return gitObject{}, err } @@ -966,34 +996,6 @@ func (r *nativeGitRepo) object(ctx context.Context, hash string) (gitObject, err return obj, nil } -func (r *nativeGitRepo) looseObject(ctx context.Context, hash string) (gitObject, error) { - if len(hash) != 40 { - return gitObject{}, fs.ErrNotExist - } - data, err := r.store.read(ctx, "objects/"+hash[:2]+"/"+hash[2:]) - if err != nil { - return gitObject{}, err - } - reader, err := zlib.NewReader(bytes.NewReader(data)) - if err != nil { - return gitObject{}, err - } - defer reader.Close() - raw, err := io.ReadAll(reader) - if err != nil { - return gitObject{}, err - } - nul := bytes.IndexByte(raw, 0) - if nul < 0 { - return gitObject{}, errors.New("invalid git object header") - } - header := strings.Split(string(raw[:nul]), " ") - if len(header) != 2 { - return gitObject{}, errors.New("invalid git object header") - } - return gitObject{typ: header[0], data: raw[nul+1:]}, nil -} - func (r *nativeGitRepo) packedObject(ctx context.Context, hash string) (gitObject, error) { if err := r.loadPackIndexes(ctx); err != nil { return gitObject{}, err @@ -1313,28 +1315,6 @@ func (r *nativeGitRepo) commit(ctx context.Context, hash string) (commitObject, return commit, nil } -func (r *nativeGitRepo) isAncestor(ctx context.Context, ancestor, descendant string) (bool, error) { - stack := []string{descendant} - seen := map[string]struct{}{} - for len(stack) > 0 { - hash := stack[len(stack)-1] - stack = stack[:len(stack)-1] - if hash == ancestor { - return true, nil - } - if _, ok := seen[hash]; ok { - continue - } - seen[hash] = struct{}{} - commit, err := r.commit(ctx, hash) - if err != nil { - return false, err - } - stack = append(stack, commit.parents...) - } - return false, nil -} - func parseCommit(hash string, data []byte) (commitObject, error) { commit := commitObject{hash: hash} text := string(data) @@ -1497,9 +1477,7 @@ func (r *nativeGitRepo) walkCommits(ctx context.Context, head string, limit, ski if err != nil { return nil, err } - for _, parent := range commit.parents { - stack = append(stack, parent) - } + stack = append(stack, commit.parents...) if pathFilter != "" { ok, err := r.commitTouchesPath(ctx, commit, pathFilter) if err != nil { @@ -1783,159 +1761,128 @@ func normalizeDestinationRef(ref string) string { } func openNativeGitRepo(store gitRemoteStore, cfg config) *nativeGitRepo { - return &nativeGitRepo{ + repo := &nativeGitRepo{ cfg: cfg, store: store, cache: map[string]gitObject{}, offsetCache: map[string]gitObject{}, } + repo.engine = repositorypkg.Open(nativeStoreReader{store: store}, nil) + return repo } func newNativeGitRepoForStore(cfg config, store gitRemoteStore) *nativeGitRepo { - return &nativeGitRepo{cfg: cfg, store: store, cache: map[string]gitObject{}, offsetCache: map[string]gitObject{}} + return openNativeGitRepo(store, cfg) } func (s *gcsGitStore) read(ctx context.Context, path string) ([]byte, error) { - name := joinObjectName(s.prefix, path) - reader, err := s.client.Bucket(s.bucket).Object(name).NewReader(ctx) + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) if err != nil { - if errors.Is(err, storage.ErrObjectNotExist) { - return nil, fs.ErrNotExist - } - return nil, gcsAccessError("read", s.bucket, name, err) + return nil, err } - defer reader.Close() - return io.ReadAll(reader) + return backend.Read(ctx, path) } func (s *gcsGitStore) list(ctx context.Context, prefix string) ([]string, error) { - queryPrefix := objectPrefix(joinObjectName(s.prefix, prefix)) - it := s.client.Bucket(s.bucket).Objects(ctx, &storage.Query{Prefix: queryPrefix}) - var paths []string - for { - attrs, err := it.Next() - if errors.Is(err, iterator.Done) { - return paths, nil - } - if err != nil { - return nil, gcsAccessError("list", s.bucket, queryPrefix, err) - } - rel := strings.TrimPrefix(attrs.Name, objectPrefix(s.prefix)) - if rel != "" && !strings.HasSuffix(rel, "/") { - paths = append(paths, rel) - } + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) + if err != nil { + return nil, err } + return backend.List(ctx, prefix) } func (s *gcsGitStore) write(ctx context.Context, path string, data []byte) error { - name := joinObjectName(s.prefix, path) - writer := s.client.Bucket(s.bucket).Object(name).NewWriter(ctx) - if _, err := writer.Write(data); err != nil { - _ = writer.Close() - return gcsAccessError("write", s.bucket, name, err) - } - if err := writer.Close(); err != nil { - return gcsAccessError("write", s.bucket, name, err) + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) + if err != nil { + return err } - return nil + return backend.Write(ctx, path, data) } func (s *gcsGitStore) delete(ctx context.Context, path string) error { - name := joinObjectName(s.prefix, path) - err := s.client.Bucket(s.bucket).Object(name).Delete(ctx) - if errors.Is(err, storage.ErrObjectNotExist) { - return nil + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) + if err != nil { + return err } - return gcsAccessError("delete", s.bucket, name, err) + return backend.Delete(ctx, path) } -func gcsAccessError(action, bucket, object string, err error) error { - if err == nil { - return nil - } - message := err.Error() - wrapped := fmt.Errorf("%s gs://%s/%s: %w", action, bucket, object, err) - if !strings.Contains(message, "invalid_rapt") && - !strings.Contains(message, "invalid_grant") && - !strings.Contains(message, "USER_PROJECT_DENIED") { - return wrapped +func (s *gcsGitStore) listRefs(ctx context.Context) (map[string]string, error) { + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) + if err != nil { + return nil, err } - return fmt.Errorf(`%w + return backend.ListRefs(ctx) +} -Google credentials need attention. If using default bgit auth, check the selected gcloud configuration: - gcloud config configurations list - gcloud auth print-access-token +func (s *gcsGitStore) compareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) + if err != nil { + return err + } + return backend.CompareAndSwapRef(ctx, ref, oldOID, newOID) +} -If using --auth adc, refresh Application Default Credentials: - gcloud auth application-default print-access-token - gcloud auth application-default set-quota-project PROJECT_ID`, wrapped) +func (s *gcsGitStore) compareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error { + backend, err := gcsstore.New(s.client, s.bucket, s.prefix) + if err != nil { + return err + } + return backend.CompareAndSwap(ctx, path, expected, replacement) } func (s *localGitStore) read(ctx context.Context, path string) ([]byte, error) { - _ = ctx - target, err := s.path(path) + backend, err := fsstore.New(s.root) if err != nil { return nil, err } - data, err := os.ReadFile(target) - if errors.Is(err, fs.ErrNotExist) { - return nil, fs.ErrNotExist - } - return data, err + return backend.Read(ctx, path) } func (s *localGitStore) list(ctx context.Context, prefix string) ([]string, error) { - _ = ctx - root, err := s.path(prefix) + backend, err := fsstore.New(s.root) if err != nil { return nil, err } - var paths []string - err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { - if walkErr != nil { - if errors.Is(walkErr, fs.ErrNotExist) { - return nil - } - return walkErr - } - if entry.IsDir() { - return nil - } - rel, err := filepath.Rel(s.root, path) - if err != nil { - return err - } - paths = append(paths, filepath.ToSlash(rel)) - return nil - }) - return paths, err + return backend.List(ctx, prefix) } func (s *localGitStore) write(ctx context.Context, path string, data []byte) error { - _ = ctx - target, err := s.path(path) + backend, err := fsstore.New(s.root) if err != nil { return err } - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return backend.Write(ctx, path, data) +} + +func (s *localGitStore) delete(ctx context.Context, path string) error { + backend, err := fsstore.New(s.root) + if err != nil { return err } - return os.WriteFile(target, data, 0o644) + return backend.Delete(ctx, path) } -func (s *localGitStore) delete(ctx context.Context, path string) error { - _ = ctx - target, pathErr := s.path(path) - if pathErr != nil { - return pathErr +func (s *localGitStore) listRefs(ctx context.Context) (map[string]string, error) { + backend, err := fsstore.New(s.root) + if err != nil { + return nil, err } - err := os.Remove(target) - if errors.Is(err, fs.ErrNotExist) { - return nil + return backend.ListRefs(ctx) +} + +func (s *localGitStore) compareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + backend, err := fsstore.New(s.root) + if err != nil { + return err } - return err + return backend.CompareAndSwapRef(ctx, ref, oldOID, newOID) } -func (s *localGitStore) path(path string) (string, error) { - return safeJoinLocalPath(s.root, path) +func (s *localGitStore) compareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error { + backend, err := fsstore.New(s.root) + if err != nil { + return err + } + return backend.CompareAndSwap(ctx, path, expected, replacement) } diff --git a/remote_helper.go b/internal/app/remote_helper.go similarity index 65% rename from remote_helper.go rename to internal/app/remote_helper.go index d6b23ed..4c7d33e 100644 --- a/remote_helper.go +++ b/internal/app/remote_helper.go @@ -1,74 +1,29 @@ -package main +package app import ( - "bufio" "context" "errors" "fmt" "io" "net/url" "strings" + + transportpkg "github.com/bucketgit/bgit/transport" ) func remoteHelperCommand(args []string, stdin io.Reader, stdout, stderr io.Writer) error { - address := remoteHelperAddress(args) - if address == "" { - return errors.New("usage: git-remote-bgit []") - } - br := bufio.NewReader(stdin) - bw := bufio.NewWriter(stdout) - for { - line, err := br.ReadString('\n') + resolver := transportpkg.ResolverFunc(func(ctx context.Context, address, service string, input io.Reader, output io.Writer) error { + cfg, err := configForRemoteHelperAddress(address) if err != nil { - if errors.Is(err, io.EOF) && strings.TrimSpace(line) == "" { - return nil - } return err } - line = strings.TrimRight(line, "\r\n") - switch { - case line == "": - return nil - case line == "capabilities": - fmt.Fprintln(bw, "connect") - fmt.Fprintln(bw) - if err := bw.Flush(); err != nil { - return err - } - case strings.HasPrefix(line, "connect "): - service := strings.TrimSpace(strings.TrimPrefix(line, "connect ")) - if service != gitUploadPackService && service != gitReceivePackService { - return fmt.Errorf("unsupported git remote helper service %q", service) - } - cfg, err := configForRemoteHelperAddress(address) - if err != nil { - return err - } - fmt.Fprintln(bw) - if err := bw.Flush(); err != nil { - return err - } - return serveGitServiceWithConfig(context.Background(), service, cfg, br, stdout) - case strings.HasPrefix(line, "option "): - fmt.Fprintln(bw, "unsupported") - if err := bw.Flush(); err != nil { - return err - } - default: - fmt.Fprintf(stderr, "unsupported git remote helper command %q\n", line) - return fmt.Errorf("unsupported git remote helper command %q", line) - } - } + return serveGitServiceWithConfig(ctx, service, cfg, input, output) + }) + return transportpkg.ServeRemoteHelper(context.Background(), resolver, args, stdin, stdout, stderr) } func remoteHelperAddress(args []string) string { - if len(args) >= 2 { - return strings.TrimSpace(args[1]) - } - if len(args) == 1 { - return strings.TrimSpace(args[0]) - } - return "" + return transportpkg.RemoteHelperAddress(args) } func configForRemoteHelperAddress(address string) (config, error) { diff --git a/remote_helper_test.go b/internal/app/remote_helper_test.go similarity index 99% rename from remote_helper_test.go rename to internal/app/remote_helper_test.go index 2dcd6d9..27c88af 100644 --- a/remote_helper_test.go +++ b/internal/app/remote_helper_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bytes" diff --git a/s3_store.go b/internal/app/s3_store.go similarity index 63% rename from s3_store.go rename to internal/app/s3_store.go index 58c332e..8bf714f 100644 --- a/s3_store.go +++ b/internal/app/s3_store.go @@ -1,12 +1,9 @@ -package main +package app import ( - "bytes" "context" "errors" "fmt" - "io" - "io/fs" "os" "strings" @@ -15,6 +12,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" + "github.com/bucketgit/bgit/store" + s3store "github.com/bucketgit/bgit/store/s3" ) type s3GitStore struct { @@ -23,6 +22,8 @@ type s3GitStore struct { prefix string } +var _ objectCASGitRemoteStore = (*s3GitStore)(nil) + func newS3Client(ctx context.Context, cfg config, anonymous bool) (*s3.Client, error) { region := awsRegion(cfg) if anonymous { @@ -65,67 +66,59 @@ func defaultAWSRegion() string { } func (s *s3GitStore) read(ctx context.Context, path string) ([]byte, error) { - key := joinObjectName(s.prefix, path) - out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - }) + backend, err := s3store.New(s.client, s.bucket, s.prefix) if err != nil { - if isS3NotFound(err) { - return nil, fs.ErrNotExist - } - return nil, s3AccessError("read", s.bucket, key, err) + return nil, err } - defer out.Body.Close() - return io.ReadAll(out.Body) + return backend.Read(ctx, path) } func (s *s3GitStore) list(ctx context.Context, prefix string) ([]string, error) { - queryPrefix := objectPrefix(joinObjectName(s.prefix, prefix)) - pager := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.bucket), - Prefix: aws.String(queryPrefix), - }) - var paths []string - for pager.HasMorePages() { - page, err := pager.NextPage(ctx) - if err != nil { - return nil, s3AccessError("list", s.bucket, queryPrefix, err) - } - for _, obj := range page.Contents { - key := aws.ToString(obj.Key) - rel := strings.TrimPrefix(key, objectPrefix(s.prefix)) - if rel != "" && !strings.HasSuffix(rel, "/") { - paths = append(paths, rel) - } - } + backend, err := s3store.New(s.client, s.bucket, s.prefix) + if err != nil { + return nil, err } - return paths, nil + return backend.List(ctx, prefix) } func (s *s3GitStore) write(ctx context.Context, path string, data []byte) error { - key := joinObjectName(s.prefix, path) - _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - Body: bytes.NewReader(data), - }) + backend, err := s3store.New(s.client, s.bucket, s.prefix) if err != nil { - return s3AccessError("write", s.bucket, key, err) + return err } - return nil + return backend.Write(ctx, path, data) } func (s *s3GitStore) delete(ctx context.Context, path string) error { - key := joinObjectName(s.prefix, path) - _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), - }) + backend, err := s3store.New(s.client, s.bucket, s.prefix) if err != nil { - return s3AccessError("delete", s.bucket, key, err) + return err } - return nil + return backend.Delete(ctx, path) +} + +func (s *s3GitStore) listRefs(ctx context.Context) (map[string]string, error) { + backend, err := s3store.New(s.client, s.bucket, s.prefix) + if err != nil { + return nil, err + } + return backend.ListRefs(ctx) +} + +func (s *s3GitStore) compareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + backend, err := s3store.New(s.client, s.bucket, s.prefix) + if err != nil { + return err + } + return backend.CompareAndSwapRef(ctx, ref, oldOID, newOID) +} + +func (s *s3GitStore) compareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error { + backend, err := s3store.New(s.client, s.bucket, s.prefix) + if err != nil { + return err + } + return backend.CompareAndSwap(ctx, path, expected, replacement) } func ensureS3Bucket(ctx context.Context, cfg config) error { @@ -163,10 +156,3 @@ func isS3NotFound(err error) bool { } return false } - -func s3AccessError(action, bucket, key string, err error) error { - if err == nil { - return nil - } - return fmt.Errorf("%s s3://%s/%s: %w", action, bucket, key, err) -} diff --git a/setup.go b/internal/app/setup.go similarity index 90% rename from setup.go rename to internal/app/setup.go index 7cb0300..1fde73e 100644 --- a/setup.go +++ b/internal/app/setup.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -19,6 +19,9 @@ import ( "strings" "time" + internalconfig "github.com/bucketgit/bgit/internal/config" + internalsetup "github.com/bucketgit/bgit/internal/setup" + "github.com/bucketgit/bgit/protocol" "golang.org/x/term" ) @@ -42,19 +45,6 @@ type setupOptions struct { action string } -type setupProfile struct { - Provider string - Name string - Active bool - Existing bool - Account string - ProjectID string - AccountID string - ARN string - Region string - ConfiguredRegions []string -} - type setupSSHKey struct { PublicKey string Source string @@ -62,7 +52,7 @@ type setupSSHKey struct { } type setupSelection struct { - Profiles []setupProfile + Profiles []internalsetup.Profile Keys []setupSSHKey IdentityName string IdentityEmail string @@ -76,12 +66,6 @@ type setupSelection struct { DefaultCreateByProvider map[string]string } -type brokerOwnerRequest struct { - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` -} - type setupConfiguredBroker struct { Provider string Profile string @@ -122,14 +106,14 @@ func setupCommand(ctx context.Context, base config, args []string, stdin io.Read if len(opts.profiles) > 0 && strings.TrimSpace(opts.profiles[0]) != "" { name = strings.TrimSpace(opts.profiles[0]) } - profiles = append(profiles, setupProfile{Provider: "local", Name: name, Region: firstNonEmpty(opts.region, "default")}) + profiles = append(profiles, internalsetup.Profile{Provider: "local", Name: name, Region: firstNonEmpty(opts.region, "default")}) } global, err := readGlobalConfig(path) if err != nil { if !errors.Is(err, os.ErrNotExist) { return err } - global = globalConfig{Version: globalConfigVersion} + global = internalconfig.Global{Version: globalConfigVersion} } if global.Version == 0 { global.Version = globalConfigVersion @@ -259,7 +243,7 @@ selectAgain: return fmt.Errorf("email address %q looks invalid", strings.TrimSpace(selection.IdentityEmail)) } if strings.TrimSpace(selection.IdentityName) != "" || strings.TrimSpace(selection.IdentityEmail) != "" { - global.Identity = globalIdentityConfig{ + global.Identity = internalconfig.Identity{ Name: strings.TrimSpace(selection.IdentityName), Email: strings.TrimSpace(selection.IdentityEmail), } @@ -448,7 +432,7 @@ func setupInteractiveCreateProfile(selection setupSelection, stdout io.Writer) e } } -func setupSelectionIdentityChanged(selection setupSelection, cfg globalConfig) bool { +func setupSelectionIdentityChanged(selection setupSelection, cfg internalconfig.Global) bool { return strings.TrimSpace(selection.IdentityName) != strings.TrimSpace(cfg.Identity.Name) || strings.TrimSpace(selection.IdentityEmail) != strings.TrimSpace(cfg.Identity.Email) } @@ -460,7 +444,7 @@ func firstSetupRequestedProfile(opts setupOptions) string { return "" } -func setupCreateProfileDefaults(profiles []setupProfile, opts setupOptions) map[string]string { +func setupCreateProfileDefaults(profiles []internalsetup.Profile, opts setupOptions) map[string]string { defaults := map[string]string{} if requested := firstSetupRequestedProfile(opts); requested != "" { if opts.provider == "" || opts.provider == "gcs" { @@ -486,7 +470,7 @@ func setupCreateProfileDefaults(profiles []setupProfile, opts setupOptions) map[ return defaults } -func configuredSetupBrokers(cfg globalConfig) []setupConfiguredBroker { +func configuredSetupBrokers(cfg internalconfig.Global) []setupConfiguredBroker { var brokers []setupConfiguredBroker for _, profile := range cfg.GCPProfiles { for _, region := range profile.Regions { @@ -527,7 +511,7 @@ func configuredSetupBrokers(cfg globalConfig) []setupConfiguredBroker { return brokers } -func configuredSetupBrokerExists(cfg globalConfig, provider, profile, region string) bool { +func configuredSetupBrokerExists(cfg internalconfig.Global, provider, profile, region string) bool { provider = normalizeSetupProvider(provider) profile = strings.TrimSpace(profile) region = strings.TrimSpace(region) @@ -539,9 +523,9 @@ func configuredSetupBrokerExists(cfg globalConfig, provider, profile, region str return false } -func profilesWithoutConfiguredExpansion(profiles []setupProfile) []setupProfile { +func profilesWithoutConfiguredExpansion(profiles []internalsetup.Profile) []internalsetup.Profile { seen := map[string]struct{}{} - var out []setupProfile + var out []internalsetup.Profile for _, profile := range profiles { key := profile.Provider + "\x00" + profile.Name if _, ok := seen[key]; ok { @@ -563,19 +547,6 @@ func setupBrokerQualifiedName(broker setupConfiguredBroker) string { return name } -func printSetupBrokerManagement(stdout io.Writer, broker setupConfiguredBroker) { - fmt.Fprintf(stdout, "Manage %s\n", setupBrokerQualifiedName(broker)) - fmt.Fprintf(stdout, "Broker: %s\n\n", broker.BrokerURL) - profileArgs := fmt.Sprintf("--profile %s --region %s", broker.Profile, broker.Region) - fmt.Fprintln(stdout, "Common broker management commands:") - fmt.Fprintf(stdout, " bgit %s admin broker-users list\n", profileArgs) - fmt.Fprintf(stdout, " bgit %s admin invite-broker-user USER --role user\n", profileArgs) - fmt.Fprintf(stdout, " bgit %s admin teams list\n", profileArgs) - fmt.Fprintf(stdout, " bgit %s admin teams create TEAM\n", profileArgs) - fmt.Fprintf(stdout, " bgit admin invite-user --broker %s --team TEAM --user USER --role developer REPO\n", broker.BrokerURL) - fmt.Fprintf(stdout, " bgit admin confirm-ownership-transfer --broker %s REPO\n", broker.BrokerURL) -} - func setupBrokerConfig(base config, broker setupConfiguredBroker) config { cfg := base cfg.provider = broker.Provider @@ -586,7 +557,7 @@ func setupBrokerConfig(base config, broker setupConfiguredBroker) config { return cfg } -func setupProvisionSelectedProfile(base config, path, now string, profile setupProfile, opts setupOptions, publicKeys []string, global *globalConfig, stdout io.Writer) error { +func setupProvisionSelectedProfile(base config, path, now string, profile internalsetup.Profile, opts setupOptions, publicKeys []string, global *internalconfig.Global, stdout io.Writer) error { _ = path if opts.action == "new" && configuredSetupBrokerExists(*global, profile.Provider, profile.Name, profile.Region) { return fmt.Errorf("%s:%s.%s already has a broker; choose Update broker to redeploy it", setupProviderLabel(profile.Provider), profile.Name, profile.Region) @@ -614,12 +585,12 @@ func setupProvisionSelectedProfile(base config, path, now string, profile setupP if strings.TrimSpace(profile.ProjectID) != "" { serviceAccount = gcpBrokerServiceAccountEmail(profile.ProjectID) } - *global = upsertGlobalGCPProfile(*global, globalGCPProfile{ + *global = upsertGlobalGCPProfile(*global, internalconfig.GCPProfile{ Name: profile.Name, ProjectID: profile.ProjectID, Account: profile.Account, ServiceAccount: serviceAccount, - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: profile.Region, BrokerURL: brokerURL, BrokerVersion: brokerVersion(), @@ -627,11 +598,11 @@ func setupProvisionSelectedProfile(base config, path, now string, profile setupP }}, }) case "s3": - *global = upsertGlobalAWSProfile(*global, globalAWSProfile{ + *global = upsertGlobalAWSProfile(*global, internalconfig.AWSProfile{ Name: profile.Name, AccountID: profile.AccountID, ARN: profile.ARN, - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: profile.Region, BrokerURL: brokerURL, BrokerVersion: brokerVersion(), @@ -647,11 +618,11 @@ func setupProvisionSelectedProfile(base config, path, now string, profile setupP return err } } - *global = upsertGlobalLocalProfile(*global, globalLocalProfile{ + *global = upsertGlobalLocalProfile(*global, internalconfig.LocalProfile{ Name: profile.Name, Root: root, Autostart: true, - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: firstNonEmpty(profile.Region, "default"), BrokerURL: brokerURL, BrokerVersion: brokerVersion(), @@ -662,88 +633,6 @@ func setupProvisionSelectedProfile(base config, path, now string, profile setupP return nil } -func offerSetupProfileBootstrap(opts setupOptions, reader *bufio.Reader, stdout io.Writer) error { - provider := opts.provider - if provider == "" { - provider = promptSetupProvider(reader, stdout) - } - switch provider { - case "gcs": - if _, err := exec.LookPath("gcloud"); err != nil { - return errors.New("gcloud is not installed") - } - fmt.Fprint(stdout, "No usable gcloud profiles found. Create one now? [y/N] ") - if !readSetupYes(reader) { - return errors.New("setup requires a cloud profile") - } - fmt.Fprint(stdout, "GCP profile name [default]: ") - name := readSetupLine(reader) - if name == "" && len(opts.profiles) > 0 { - name = opts.profiles[0] - } - if name == "" { - name = "default" - } - if err := runGcloudProfileCommand(stdout, "config", "configurations", "create", name); err != nil { - return err - } - return runGcloudProfileCommand(stdout, "auth", "login", "--configuration", name) - case "s3": - if _, err := exec.LookPath("aws"); err != nil { - return errors.New("AWS CLI is not installed") - } - fmt.Fprint(stdout, "No usable AWS profiles found. Run aws configure now? [y/N] ") - if !readSetupYes(reader) { - return errors.New("setup requires a cloud profile") - } - fmt.Fprint(stdout, "AWS profile name [default]: ") - name := readSetupLine(reader) - if name == "" && len(opts.profiles) > 0 { - name = opts.profiles[0] - } - if name == "" { - name = "default" - } - return runAWSProfileCommand(stdout, "configure", "--profile", name) - default: - return errors.New("setup requires --provider gcp or --provider aws to create a profile") - } -} - -func promptSetupProvider(reader *bufio.Reader, stdout io.Writer) string { - gcloudOK := false - awsOK := false - if _, err := exec.LookPath("gcloud"); err == nil { - gcloudOK = true - } - if _, err := exec.LookPath("aws"); err == nil { - awsOK = true - } - if gcloudOK && !awsOK { - return "gcs" - } - if awsOK && !gcloudOK { - return "s3" - } - if !gcloudOK && !awsOK { - return "" - } - fmt.Fprint(stdout, "Create profile for provider [gcp/aws]: ") - switch strings.ToLower(readSetupLine(reader)) { - case "gcp", "gcs": - return "gcs" - case "aws", "s3": - return "s3" - default: - return "" - } -} - -func readSetupYes(reader *bufio.Reader) bool { - answer := strings.ToLower(readSetupLine(reader)) - return answer == "y" || answer == "yes" -} - func readSetupLine(reader *bufio.Reader) string { line, err := reader.ReadString('\n') if err != nil && !errors.Is(err, io.EOF) { @@ -882,50 +771,8 @@ func parseSetupArgs(args []string) (setupOptions, error) { return opts, nil } -func discoverSetupProfiles(ctx context.Context) ([]setupProfile, error) { - var profiles []setupProfile - gcp, err := discoverGCPSetupProfiles(ctx) - if err != nil { - return nil, err - } - profiles = append(profiles, gcp...) - aws, err := discoverAWSSetupProfiles(ctx) - if err != nil { - return nil, err - } - profiles = append(profiles, aws...) - return profiles, nil -} - -func discoverGCPSetupProfiles(ctx context.Context) ([]setupProfile, error) { - if _, err := exec.LookPath("gcloud"); err != nil { - return nil, nil - } - probeCtx, cancel := context.WithTimeout(ctx, setupProbeTimeout) - defer cancel() - out, err := exec.CommandContext(probeCtx, "gcloud", "config", "configurations", "list", "--format=value(name,is_active)").Output() - if err != nil { - return nil, nil - } - var profiles []setupProfile - scanner := bufio.NewScanner(bytes.NewReader(out)) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) == 0 { - continue - } - name := fields[0] - active := len(fields) > 1 && strings.EqualFold(fields[1], "true") - profile := setupProfile{Provider: "gcs", Name: name, Active: active} - profile.Account = gcloudConfigValue(ctx, name, "account") - profile.ProjectID = gcloudConfigValue(ctx, name, "project") - profile.Region = firstNonEmpty(gcloudConfigValue(ctx, name, "run/region"), gcloudConfigValue(ctx, name, "functions/region"), "us-central1") - profiles = append(profiles, profile) - } - if err := scanner.Err(); err != nil { - return nil, err - } - return profiles, nil +func discoverSetupProfiles(ctx context.Context) ([]internalsetup.Profile, error) { + return internalsetup.DiscoverProfiles(ctx, internalsetup.ProfileDiscoveryOptions{ProbeTimeout: setupProbeTimeout}) } func requireSetupCLI(binary, provider string) error { @@ -936,124 +783,19 @@ func requireSetupCLI(binary, provider string) error { } func gcloudConfigValue(ctx context.Context, profile, key string) string { - probeCtx, cancel := context.WithTimeout(ctx, setupProbeTimeout) - defer cancel() - cmd := exec.CommandContext(probeCtx, "gcloud", "--configuration", profile, "config", "get-value", key, "--quiet") - out, err := cmd.Output() - if err != nil { - return "" - } - value := strings.TrimSpace(string(out)) - if value == "(unset)" { - return "" - } - return value -} - -func discoverAWSSetupProfiles(ctx context.Context) ([]setupProfile, error) { - _ = ctx - names := map[string]struct{}{} - for _, name := range awsProfilesFromFiles() { - names[name] = struct{}{} - } - var sorted []string - for name := range names { - sorted = append(sorted, name) - } - sort.Strings(sorted) - var profiles []setupProfile - for _, name := range sorted { - profile := setupProfile{Provider: "s3", Name: name, Region: configuredAWSProfileRegion(name)} - profiles = append(profiles, profile) - } - return profiles, nil -} - -func awsProfilesFromFiles() []string { - home, err := os.UserHomeDir() - if err != nil { - return nil - } - names := map[string]struct{}{} - for _, path := range []string{filepath.Join(home, ".aws", "config"), filepath.Join(home, ".aws", "credentials")} { - for _, name := range parseAWSProfileFile(path) { - names[name] = struct{}{} - } - } - var sorted []string - for name := range names { - sorted = append(sorted, name) - } - sort.Strings(sorted) - return sorted + return internalsetup.GCloudConfigValue(ctx, profile, key, internalsetup.ProfileDiscoveryOptions{ProbeTimeout: setupProbeTimeout}) } func parseAWSProfileFile(path string) []string { - data, err := os.ReadFile(path) - if err != nil { - return nil - } - names := map[string]struct{}{} - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if !strings.HasPrefix(line, "[") || !strings.HasSuffix(line, "]") { - continue - } - name := strings.TrimSuffix(strings.TrimPrefix(line, "["), "]") - name = strings.TrimSpace(strings.TrimPrefix(name, "profile ")) - if name != "" { - names[name] = struct{}{} - } - } - var sorted []string - for name := range names { - sorted = append(sorted, name) - } - sort.Strings(sorted) - return sorted + return internalsetup.ParseAWSProfileFile(path) } func configuredAWSProfileRegion(profile string) string { - home, err := os.UserHomeDir() - if err != nil { - return "" - } - for _, path := range []string{filepath.Join(home, ".aws", "config"), filepath.Join(home, ".aws", "credentials")} { - if region := awsProfileFileValue(path, profile, "region"); region != "" { - return region - } - } - return "" + return internalsetup.ConfiguredAWSProfileRegion(profile, internalsetup.ProfileDiscoveryOptions{}) } func awsProfileFileValue(path, profile, key string) string { - data, err := os.ReadFile(path) - if err != nil { - return "" - } - section := "" - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { - section = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "["), "]")) - section = strings.TrimSpace(strings.TrimPrefix(section, "profile ")) - continue - } - if section != profile { - continue - } - k, v, ok := strings.Cut(line, "=") - if !ok || strings.TrimSpace(k) != key { - continue - } - return strings.TrimSpace(v) - } - return "" + return internalsetup.AWSProfileFileValue(path, profile, key) } func awsCallerIdentity(ctx context.Context, profile string) (string, string) { @@ -1083,7 +825,7 @@ func awsCallerIdentity(ctx context.Context, profile string) (string, string) { return resp.Account, resp.ARN } -func resolveGCPSetupRegion(profile setupProfile, explicitRegion string, interactive bool, stdin io.Reader, stdout io.Writer) (string, error) { +func resolveGCPSetupRegion(profile internalsetup.Profile, explicitRegion string, interactive bool, stdin io.Reader, stdout io.Writer) (string, error) { regions, err := resolveGCPSetupRegionsWithRaw(profile, explicitRegion, interactive, stdin, stdin, stdout) if err != nil || len(regions) == 0 { return "", err @@ -1091,15 +833,7 @@ func resolveGCPSetupRegion(profile setupProfile, explicitRegion string, interact return regions[0], nil } -func resolveGCPSetupRegionWithRaw(profile setupProfile, explicitRegion string, interactive bool, stdin io.Reader, rawInput io.Reader, stdout io.Writer) (string, error) { - regions, err := resolveGCPSetupRegionsWithRaw(profile, explicitRegion, interactive, stdin, rawInput, stdout) - if err != nil || len(regions) == 0 { - return "", err - } - return regions[0], nil -} - -func resolveGCPSetupRegionsWithRaw(profile setupProfile, explicitRegion string, interactive bool, stdin io.Reader, rawInput io.Reader, stdout io.Writer) ([]string, error) { +func resolveGCPSetupRegionsWithRaw(profile internalsetup.Profile, explicitRegion string, interactive bool, stdin io.Reader, rawInput io.Reader, stdout io.Writer) ([]string, error) { if strings.TrimSpace(explicitRegion) != "" { return []string{strings.TrimSpace(explicitRegion)}, nil } @@ -1136,7 +870,7 @@ func gcpSetupRegions(defaultRegion string) []string { return regions } -func resolveAWSSetupRegion(ctx context.Context, profile setupProfile, explicitRegion string, interactive bool, stdin io.Reader, stdout io.Writer) (string, error) { +func resolveAWSSetupRegion(ctx context.Context, profile internalsetup.Profile, explicitRegion string, interactive bool, stdin io.Reader, stdout io.Writer) (string, error) { regions, err := resolveAWSSetupRegionsWithRaw(ctx, profile, explicitRegion, interactive, stdin, stdin, stdout) if err != nil || len(regions) == 0 { return "", err @@ -1144,15 +878,7 @@ func resolveAWSSetupRegion(ctx context.Context, profile setupProfile, explicitRe return regions[0], nil } -func resolveAWSSetupRegionWithRaw(ctx context.Context, profile setupProfile, explicitRegion string, interactive bool, stdin io.Reader, rawInput io.Reader, stdout io.Writer) (string, error) { - regions, err := resolveAWSSetupRegionsWithRaw(ctx, profile, explicitRegion, interactive, stdin, rawInput, stdout) - if err != nil || len(regions) == 0 { - return "", err - } - return regions[0], nil -} - -func resolveAWSSetupRegionsWithRaw(ctx context.Context, profile setupProfile, explicitRegion string, interactive bool, stdin io.Reader, rawInput io.Reader, stdout io.Writer) ([]string, error) { +func resolveAWSSetupRegionsWithRaw(ctx context.Context, profile internalsetup.Profile, explicitRegion string, interactive bool, stdin io.Reader, rawInput io.Reader, stdout io.Writer) ([]string, error) { if err := requireSetupCLI("aws", "AWS"); err != nil { return nil, err } @@ -1176,7 +902,7 @@ func resolveAWSSetupRegionsWithRaw(ctx context.Context, profile setupProfile, ex return runSetupRegionDialogWithRaw(reader, rawInput, stdout, "AWS", profile.Name, awsSetupRegions, setupRegionInitialSelection(profile)) } -func markConfiguredSetupProfiles(profiles []setupProfile, cfg globalConfig) []setupProfile { +func markConfiguredSetupProfiles(profiles []internalsetup.Profile, cfg internalconfig.Global) []internalsetup.Profile { configured := map[string][]string{} for _, profile := range cfg.GCPProfiles { if regions := configuredSetupProfileRegions(profile.Regions); len(regions) > 0 { @@ -1191,7 +917,7 @@ func markConfiguredSetupProfiles(profiles []setupProfile, cfg globalConfig) []se if len(configured) == 0 { return profiles } - var out []setupProfile + var out []internalsetup.Profile for _, profile := range profiles { regions, ok := configured[profile.Provider+":"+profile.Name] if !ok { @@ -1209,7 +935,7 @@ func markConfiguredSetupProfiles(profiles []setupProfile, cfg globalConfig) []se return out } -func configuredSetupProfileRegions(regions []globalProfileRegion) []string { +func configuredSetupProfileRegions(regions []internalconfig.ProfileRegion) []string { var out []string for _, region := range regions { if strings.TrimSpace(region.BrokerURL) == "" { @@ -1230,7 +956,7 @@ func configuredSetupProfileRegions(regions []globalProfileRegion) []string { return uniqueStrings(out) } -func setupRegionInitialSelection(profile setupProfile) []string { +func setupRegionInitialSelection(profile internalsetup.Profile) []string { if len(profile.ConfiguredRegions) > 0 { return append([]string{}, profile.ConfiguredRegions...) } @@ -1240,12 +966,12 @@ func setupRegionInitialSelection(profile setupProfile) []string { return nil } -func filterSetupProfiles(profiles []setupProfile, provider string, names []string, explicitRegion string) []setupProfile { +func filterSetupProfiles(profiles []internalsetup.Profile, provider string, names []string, explicitRegion string) []internalsetup.Profile { nameSet := map[string]struct{}{} for _, name := range names { nameSet[name] = struct{}{} } - var out []setupProfile + var out []internalsetup.Profile for _, profile := range profiles { if provider != "" && profile.Provider != provider { continue @@ -1266,7 +992,7 @@ func filterSetupProfiles(profiles []setupProfile, provider string, names []strin return out } -func setupProfileNameSelected(profile setupProfile, names map[string]struct{}) bool { +func setupProfileNameSelected(profile internalsetup.Profile, names map[string]struct{}) bool { candidates := []string{ profile.Name, profile.Name + "." + profile.Region, @@ -1282,9 +1008,9 @@ func setupProfileNameSelected(profile setupProfile, names map[string]struct{}) b return false } -func dedupeSetupProfiles(profiles []setupProfile) []setupProfile { +func dedupeSetupProfiles(profiles []internalsetup.Profile) []internalsetup.Profile { seen := map[string]struct{}{} - var out []setupProfile + var out []internalsetup.Profile for _, profile := range profiles { key := profile.Provider + ":" + profile.Name if _, ok := seen[key]; ok { @@ -1321,9 +1047,7 @@ func discoverSetupSSHKeys(opts setupSSHKeyOptions) ([]setupSSHKey, error) { if err != nil { return nil, err } - for _, key := range parseSetupSSHKeys(string(data), path) { - keys = append(keys, key) - } + keys = append(keys, parseSetupSSHKeys(string(data), path)...) } if !opts.NoAgent { agentKeys, err := sshAgentPublicKeys() @@ -1362,9 +1086,7 @@ func discoverSSHKeyFiles() ([]setupSSHKey, error) { if err != nil { continue } - for _, key := range parseSetupSSHKeys(string(data), path) { - keys = append(keys, key) - } + keys = append(keys, parseSetupSSHKeys(string(data), path)...) } } return keys, nil @@ -2459,7 +2181,7 @@ func renderSetupDialogFrame(state setupDialogState, rawMode bool) string { } type setupDialogState struct { - profiles []setupProfile + profiles []internalsetup.Profile keys []setupSSHKey createProviders []string selectedProfiles []bool @@ -3319,7 +3041,7 @@ func runSetupBrokerUserWithRaw(cfg config, broker setupConfiguredBroker, usernam } } -func setupBrokerUserActionChoices(user brokerUserInfo) []setupChoice { +func setupBrokerUserActionChoices(user protocol.UserInfo) []setupChoice { if user.BrokerRole == "owner" { return []setupChoice{ {Label: "transfer ownership", Value: "transfer-owner", Help: "create an ownership transfer command"}, @@ -3340,7 +3062,7 @@ func setupBrokerUserActionChoices(user brokerUserInfo) []setupChoice { return choices } -func runSetupBrokerUserAction(cfg config, broker setupConfiguredBroker, user brokerUserInfo, action string, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer) (string, error) { +func runSetupBrokerUserAction(cfg config, broker setupConfiguredBroker, user protocol.UserInfo, action string, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer) (string, error) { var out bytes.Buffer username := firstNonEmpty(user.Username, user.ID) role := firstNonEmpty(user.BrokerRole, "user") @@ -3698,8 +3420,12 @@ func runSetupTeamRepositoryCreateWithRaw(cfg config, teamID, teamName string, re } repo.TeamID = teamID repo.TeamName = teamName - req := brokerRepoAdminRequest{Repo: repo, Role: role} - if err := brokerPost(brokerURL, "/repos/create", req, nil); err != nil { + req := protocol.RepositoryAdminRequest{Repo: repo, Role: role} + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return "", err + } + if _, err := endpoints.CreateRepository(context.Background(), req); err != nil { return "", err } return fmt.Sprintf("created repository %s and granted %s %s access", logicalRepoDisplayName(logical), firstNonEmpty(teamName, teamID), role), nil @@ -3713,18 +3439,18 @@ func runSetupLocalBrokerRepoStorageRegionWithRaw(cfg config, repoName string, re return repoName, true, nil } -func setupRepoForTeamCreate(cfg config, value, teamID, teamName string) (brokerRepo, error) { +func setupRepoForTeamCreate(cfg config, value, teamID, teamName string) (protocol.Repository, error) { if repo, ok, err := brokerRepoForStorageTarget(cfg, value, teamID); ok || err != nil { repo.TeamName = teamName return repo, err } logical, err := normalizeLogicalRepoName(value) if err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } actionCfg, err := setupBrokerRepoConfig(cfg, logical) if err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } actionCfg.provider = firstNonEmpty(actionCfg.provider, cfg.provider, "gcs") actionCfg.brokerURL = cfg.brokerURL @@ -3801,21 +3527,25 @@ func runSetupManagedTeamRepositoryAction(cfg config, teamID, teamName, repo, act if err != nil { return "", err } - brokerRepo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) + repoInfo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) if err != nil { return "", err } - var resp brokerOwnerTransferResponse - if err := brokerPost(brokerURL, "/keys/invite/create", brokerOwnerTransferRequest{Repo: brokerRepo, BrokerURL: brokerURL, User: user, Role: role}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return "", err + } + resp, err := endpoints.CreateRepositoryInvite(context.Background(), protocol.OwnerTransferRequest{Repo: repoInfo, BrokerURL: brokerURL, User: user, Role: role}) + if err != nil { return "", err } return runSetupPlainCommandOutputWithRaw(reader, stdout, setupBreadcrumb("Manage team", teamName, "Repositories", repoName, "Invite user"), resp.AcceptCommand) case "cancel-invite": - brokerRepo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) + repoInfo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) if err != nil { return "", err } - user, ok, err := runSetupPendingRepoInviteSelect(cfg, brokerRepo, reader, rawInput, stdout, setupBreadcrumb("Manage team", teamName, "Repositories", repoName, "Pending invite")) + user, ok, err := runSetupPendingRepoInviteSelect(cfg, repoInfo, reader, rawInput, stdout, setupBreadcrumb("Manage team", teamName, "Repositories", repoName, "Pending invite")) if err != nil || !ok { return "No changes made.", err } @@ -3823,10 +3553,14 @@ func runSetupManagedTeamRepositoryAction(cfg config, teamID, teamName, repo, act if err != nil { return "", err } - if err := brokerPost(brokerURL, "/keys/invite/cancel", brokerOwnerTransferRequest{Repo: brokerRepo, User: user}, nil); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { return "", err } - fmt.Fprintf(&out, "cancelled invite for %s on %s\n", user, brokerRepo.Logical) + if err := endpoints.CancelRepositoryInvite(context.Background(), protocol.OwnerTransferRequest{Repo: repoInfo, User: user}); err != nil { + return "", err + } + fmt.Fprintf(&out, "cancelled invite for %s on %s\n", user, repoInfo.Logical) return strings.TrimSpace(out.String()), nil case "grant-team": targetTeamID, ok, err := runSetupAvailableRepoTeamSelect(cfg, repo, teamID, reader, rawInput, stdout, setupBreadcrumb("Manage team", teamName, "Repositories", repoName, "Grant team access")) @@ -3968,12 +3702,16 @@ func runSetupRepoUserAssignmentWithRaw(cfg config, teamID, teamName, repo string if err != nil { return "", err } - brokerRepo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) + repoInfo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) if err != nil { return "", err } for _, userID := range selected { - if err := brokerPost(brokerURL, "/repo/users/upsert", brokerRepoAdminRequest{Repo: brokerRepo, UserID: userID, Role: role}, nil); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return "", err + } + if err := endpoints.UpsertRepositoryUser(context.Background(), protocol.RepositoryAdminRequest{Repo: repoInfo, UserID: userID, Role: role}); err != nil { return "", err } } @@ -4031,8 +3769,12 @@ func setupAvailableRepoTeamChoices(cfg config, repo, currentTeamID string) ([]se if err != nil { return nil, err } - var teamsResp brokerTeamsResponse - if err := brokerPost(brokerURL, "/teams/list", brokerRepoAdminRequest{}, &teamsResp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return nil, err + } + teams, err := endpoints.ListTeams(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { return nil, err } repos, err := setupBrokerRepos(cfg) @@ -4070,7 +3812,7 @@ func setupAvailableRepoTeamChoices(cfg config, repo, currentTeamID string) ([]se attached[currentTeamID] = struct{}{} } var choices []setupChoice - for _, team := range teamsResp.Teams { + for _, team := range teams { id := strings.TrimSpace(team.ID) if id == "" { continue @@ -4089,18 +3831,20 @@ func setupBrokerRepoTeamIDs(cfg config, repo, teamID string) ([]string, error) { if err != nil { return nil, err } - brokerRepo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) + repoInfo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) if err != nil { return nil, err } - var resp struct { - Teams []brokerRepoTeamGrant `json:"teams"` + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return nil, err } - if err := brokerPost(brokerURL, "/repo/teams/list", brokerRepoInfoRequest{Repo: brokerRepo}, &resp); err != nil { + teams, err := endpoints.ListRepositoryTeams(context.Background(), protocol.RepositoryAdminRequest{Repo: repoInfo}) + if err != nil { return nil, err } - ids := make([]string, 0, len(resp.Teams)) - for _, grant := range resp.Teams { + ids := make([]string, 0, len(teams)) + for _, grant := range teams { id := firstNonEmpty(grant.ID, grant.TeamID) if id != "" { ids = append(ids, id) @@ -4142,8 +3886,8 @@ func setupRepoUsers(cfg config, repo, teamID string) ([]setupRepoUser, error) { grants, grantErr := setupRepoUserGrants(cfg, repo, teamID) if grantErr == nil { brokerUsers, _ := setupBrokerUsers(cfg) - byID := map[string]brokerUserInfo{} - byName := map[string]brokerUserInfo{} + byID := map[string]protocol.UserInfo{} + byName := map[string]protocol.UserInfo{} for _, user := range brokerUsers { if user.ID != "" { byID[user.ID] = user @@ -4182,7 +3926,7 @@ func setupRepoUsers(cfg config, repo, teamID string) ([]setupRepoUser, error) { return users, nil } -func setupRepoKeys(cfg config, repo, teamID string) ([]brokerKey, error) { +func setupRepoKeys(cfg config, repo, teamID string) ([]protocol.Key, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return nil, err @@ -4196,20 +3940,24 @@ func setupRepoKeys(cfg config, repo, teamID string) ([]brokerKey, error) { return brokerListKeys(brokerURL, actionCfg) } -func setupRepoUserGrants(cfg config, repo, teamID string) ([]brokerRepoUserGrant, error) { +func setupRepoUserGrants(cfg config, repo, teamID string) ([]protocol.RepositoryUserGrant, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return nil, err } - brokerRepo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) + repoInfo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) if err != nil { return nil, err } - var resp brokerRepoUsersResponse - if err := brokerPost(brokerURL, "/repo/users/list", brokerRepoAdminRequest{Repo: brokerRepo}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { return nil, err } - return resp.Users, nil + users, err := endpoints.ListRepositoryUsers(context.Background(), protocol.RepositoryAdminRequest{Repo: repoInfo}) + if err != nil { + return nil, err + } + return users, nil } func strongerSetupRepoRole(a, b string) string { @@ -4288,6 +4036,10 @@ func runSetupManagedTeamRepositoryUserAction(cfg config, teamID, teamName, repo, } actionCfg.provider = firstNonEmpty(actionCfg.provider, cfg.provider, "gcs") actionCfg.teamID = strings.TrimSpace(teamID) + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return "", err + } switch action { case "user-edit": role, ok, err := runSetupSelectWithRaw(reader, rawInput, stdout, setupBreadcrumb("Manage team", teamName, "Repositories", repoName, user, "Role"), setupRepoRoleChoices(), "developer") @@ -4295,12 +4047,12 @@ func runSetupManagedTeamRepositoryUserAction(cfg config, teamID, teamName, repo, return "No changes made.", err } if hasGrant { - if err := brokerPost(brokerURL, "/repo/users/upsert", brokerRepoAdminRequest{Repo: repoForBroker(actionCfg), UserID: grant.UserID, User: firstNonEmpty(grant.User, grant.Username, user), Role: role}, nil); err != nil { + if err := endpoints.UpsertRepositoryUser(context.Background(), protocol.RepositoryAdminRequest{Repo: repoForBroker(actionCfg), UserID: grant.UserID, User: firstNonEmpty(grant.User, grant.Username, user), Role: role}); err != nil { return "", err } } for _, key := range keys { - if err := brokerPost(brokerURL, "/keys/remove", brokerKeyRequest{Repo: repoForBroker(actionCfg), Key: key.PublicKey}, nil); err != nil { + if err := endpoints.RemoveKey(context.Background(), protocol.KeyRequest{Repo: repoForBroker(actionCfg), Key: key.PublicKey}); err != nil { return "", err } } @@ -4320,12 +4072,19 @@ func runSetupManagedTeamRepositoryUserAction(cfg config, teamID, teamName, repo, verb = "removed" } if action == "user-remove" && hasGrant { - if err := brokerPost(brokerURL, "/repo/users/remove", brokerRepoAdminRequest{Repo: repoForBroker(actionCfg), UserID: grant.UserID, User: firstNonEmpty(grant.User, grant.Username, user)}, nil); err != nil { + if err := endpoints.RemoveRepositoryUser(context.Background(), protocol.RepositoryAdminRequest{Repo: repoForBroker(actionCfg), UserID: grant.UserID, User: firstNonEmpty(grant.User, grant.Username, user)}); err != nil { return "", err } } for _, key := range keys { - if err := brokerPost(brokerURL, path, brokerKeyRequest{Repo: repoForBroker(actionCfg), Key: key.PublicKey}, nil); err != nil { + request := protocol.KeyRequest{Repo: repoForBroker(actionCfg), Key: key.PublicKey} + var err error + if path == "/keys/remove" { + err = endpoints.RemoveKey(context.Background(), request) + } else { + err = endpoints.SuspendKey(context.Background(), request, true) + } + if err != nil { return "", err } } @@ -4335,12 +4094,12 @@ func runSetupManagedTeamRepositoryUserAction(cfg config, teamID, teamName, repo, } } -func setupRepoKeysForUser(cfg config, repo, teamID, user string) ([]brokerKey, error) { +func setupRepoKeysForUser(cfg config, repo, teamID, user string) ([]protocol.Key, error) { keys, err := setupRepoKeys(cfg, repo, teamID) if err != nil { return nil, err } - var out []brokerKey + var out []protocol.Key for _, key := range keys { if strings.EqualFold(strings.TrimSpace(key.User), strings.TrimSpace(user)) { out = append(out, key) @@ -4349,10 +4108,10 @@ func setupRepoKeysForUser(cfg config, repo, teamID, user string) ([]brokerKey, e return out, nil } -func setupRepoUserGrantForUser(cfg config, repo, teamID, user string) (brokerRepoUserGrant, bool, error) { +func setupRepoUserGrantForUser(cfg config, repo, teamID, user string) (protocol.RepositoryUserGrant, bool, error) { grants, err := setupRepoUserGrants(cfg, repo, teamID) if err != nil { - return brokerRepoUserGrant{}, false, err + return protocol.RepositoryUserGrant{}, false, err } needle := strings.ToLower(strings.TrimSpace(user)) for _, grant := range grants { @@ -4360,10 +4119,10 @@ func setupRepoUserGrantForUser(cfg config, repo, teamID, user string) (brokerRep return grant, true, nil } } - return brokerRepoUserGrant{}, false, nil + return protocol.RepositoryUserGrant{}, false, nil } -func runSetupPendingRepoInviteSelect(cfg config, repo brokerRepo, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { +func runSetupPendingRepoInviteSelect(cfg config, repo protocol.Repository, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { choices, err := setupPendingRepoInviteChoices(cfg, repo) if err != nil { return "", false, err @@ -4377,17 +4136,21 @@ func runSetupPendingRepoInviteSelect(cfg config, repo brokerRepo, reader *bufio. return runSetupSelectWithRaw(reader, rawInput, stdout, title, choices, "") } -func setupPendingRepoInviteChoices(cfg config, repo brokerRepo) ([]setupChoice, error) { +func setupPendingRepoInviteChoices(cfg config, repo protocol.Repository) ([]setupChoice, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return nil, err } - var resp brokerRepoInvitesResponse - if err := brokerPost(brokerURL, "/keys/invite/list", brokerOwnerTransferRequest{Repo: repo}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { return nil, err } - choices := make([]setupChoice, 0, len(resp.Invites)) - for _, invite := range resp.Invites { + invites, err := endpoints.ListRepositoryInvites(context.Background(), protocol.OwnerTransferRequest{Repo: repo}) + if err != nil { + return nil, err + } + choices := make([]setupChoice, 0, len(invites)) + for _, invite := range invites { user := strings.TrimSpace(invite.User) if user == "" { continue @@ -4398,10 +4161,10 @@ func setupPendingRepoInviteChoices(cfg config, repo brokerRepo) ([]setupChoice, return choices, nil } -func setupBrokerTeamRepoForAction(cfg config, repo, teamID string) (brokerRepo, error) { +func setupBrokerTeamRepoForAction(cfg config, repo, teamID string) (protocol.Repository, error) { actionCfg, err := setupBrokerRepoConfig(cfg, repo) if err != nil { - return brokerRepo{}, err + return protocol.Repository{}, err } actionCfg.provider = firstNonEmpty(actionCfg.provider, cfg.provider, "gcs") actionCfg.teamID = strings.TrimSpace(teamID) @@ -4456,22 +4219,6 @@ func setupRepoRoleCapChoices() []setupChoice { return choices } -func runSetupTeamSelect(cfg config, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { - teams, err := setupBrokerTeamChoices(cfg) - if err != nil { - return "", false, err - } - return runSetupSelectWithRaw(reader, rawInput, stdout, title, teams, "") -} - -func runSetupUserSelect(cfg config, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { - users, err := setupBrokerUserChoices(cfg) - if err != nil { - return "", false, err - } - return runSetupSelectWithRaw(reader, rawInput, stdout, title, users, "") -} - func runSetupAvailableRepoInviteUserSelect(cfg config, repo, teamID string, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { users, err := setupAvailableRepoInviteUserChoices(cfg, repo, teamID) if err != nil { @@ -4508,73 +4255,49 @@ func runSetupRepoSelect(cfg config, reader *bufio.Reader, rawInput io.Reader, st return runSetupSelectWithRaw(reader, rawInput, stdout, title, repos, "") } -func runSetupRepoMultiSelect(cfg config, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) ([]string, bool, error) { - repos, err := setupBrokerRepoChoices(cfg) - if err != nil { - return nil, false, err - } - return runSetupMultiSelectWithRaw(reader, rawInput, stdout, title, repos) -} - -func runSetupTeamMemberSelect(cfg config, teamID string, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { - team, err := setupBrokerTeamInfo(cfg, teamID) - if err != nil { - return "", false, err - } - var choices []setupChoice - for _, member := range team.Members { - user := firstNonEmpty(member.Username, member.UserID) - if user == "" { - continue - } - choices = append(choices, setupChoice{Label: user, Value: user, Help: "team role cap " + firstNonEmpty(member.Role, "read")}) - } - return runSetupSelectWithRaw(reader, rawInput, stdout, title, choices, "") -} - -func runSetupTeamRepoSelect(cfg config, teamID string, reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string) (string, bool, error) { - choices, err := setupBrokerTeamRepoChoices(cfg, teamID) - if err != nil { - return "", false, err - } - return runSetupSelectWithRaw(reader, rawInput, stdout, title, choices, "") -} - func setupBrokerTeamChoices(cfg config) ([]setupChoice, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return nil, err } - var resp brokerTeamsResponse - if err := brokerPost(brokerURL, "/teams/list", brokerRepoAdminRequest{}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return nil, err + } + teams, err := endpoints.ListTeams(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { return nil, err } var choices []setupChoice - for _, team := range resp.Teams { + for _, team := range teams { choices = append(choices, setupChoice{Label: team.Name, Value: team.ID, Help: fmt.Sprintf("%d member(s)", len(team.Members))}) } sort.Slice(choices, func(i, j int) bool { return choices[i].Label < choices[j].Label }) return choices, nil } -func setupBrokerTeamInfo(cfg config, teamID string) (brokerTeamInfo, error) { +func setupBrokerTeamInfo(cfg config, teamID string) (protocol.Team, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { - return brokerTeamInfo{}, err + return protocol.Team{}, err } - var resp brokerTeamsResponse - if err := brokerPost(brokerURL, "/teams/list", brokerRepoAdminRequest{}, &resp); err != nil { - return brokerTeamInfo{}, err + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return protocol.Team{}, err } - for _, team := range resp.Teams { + teams, err := endpoints.ListTeams(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { + return protocol.Team{}, err + } + for _, team := range teams { if team.ID == teamID || strings.EqualFold(team.Name, teamID) { return team, nil } } - return brokerTeamInfo{}, fmt.Errorf("team %s not found", teamID) + return protocol.Team{}, fmt.Errorf("team %s not found", teamID) } -func setupFormatTeamMembers(team brokerTeamInfo) string { +func setupFormatTeamMembers(team protocol.Team) string { if len(team.Members) == 0 { return "No members." } @@ -4594,18 +4317,10 @@ func setupFormatTeamMembers(team brokerTeamInfo) string { return strings.Join(lines, "\n") } -func setupTeamDisplayName(fallback string, team brokerTeamInfo) string { +func setupTeamDisplayName(fallback string, team protocol.Team) string { return firstNonEmpty(team.Name, team.ID, fallback) } -func setupBrokerUserChoices(cfg config) ([]setupChoice, error) { - users, err := setupBrokerUsers(cfg) - if err != nil { - return nil, err - } - return setupBrokerUserChoicesFromUsers(users, nil), nil -} - func setupAvailableTeamUserChoices(cfg config, teamID string) ([]setupChoice, error) { users, err := setupBrokerUsers(cfg) if err != nil { @@ -4640,11 +4355,11 @@ func setupAvailableRepoInviteUserChoices(cfg config, repo, teamID string) ([]set exclude[strings.ToLower(user.User)] = struct{}{} } } - brokerRepo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) + repoInfo, err := setupBrokerTeamRepoForAction(cfg, repo, teamID) if err != nil { return nil, err } - pending, err := setupPendingRepoInviteChoices(cfg, brokerRepo) + pending, err := setupPendingRepoInviteChoices(cfg, repoInfo) if err != nil { return nil, err } @@ -4656,22 +4371,26 @@ func setupAvailableRepoInviteUserChoices(cfg config, repo, teamID string) ([]set return setupBrokerUserChoicesFromUsers(users, exclude), nil } -func setupBrokerUsers(cfg config) ([]brokerUserInfo, error) { +func setupBrokerUsers(cfg config) ([]protocol.UserInfo, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return nil, err } - var resp brokerUsersResponse - if err := brokerPost(brokerURL, "/broker/users/list", brokerRepoAdminRequest{}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { return nil, err } - return resp.Users, nil + users, err := endpoints.ListUsers(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { + return nil, err + } + return users, nil } -func setupBrokerUserByName(cfg config, username string) (brokerUserInfo, bool, error) { +func setupBrokerUserByName(cfg config, username string) (protocol.UserInfo, bool, error) { users, err := setupBrokerUsers(cfg) if err != nil { - return brokerUserInfo{}, false, err + return protocol.UserInfo{}, false, err } needle := strings.ToLower(strings.TrimSpace(username)) for _, user := range users { @@ -4679,10 +4398,10 @@ func setupBrokerUserByName(cfg config, username string) (brokerUserInfo, bool, e return user, true, nil } } - return brokerUserInfo{}, false, nil + return protocol.UserInfo{}, false, nil } -func setupBrokerUserStatus(user brokerUserInfo) string { +func setupBrokerUserStatus(user protocol.UserInfo) string { parts := []string{firstNonEmpty(user.BrokerRole, "user")} if user.Suspended { parts = append(parts, "suspended") @@ -4697,7 +4416,7 @@ func setupBrokerUserStatus(user brokerUserInfo) string { return strings.Join(parts, " · ") } -func setupBrokerUserChoicesFromUsers(users []brokerUserInfo, exclude map[string]struct{}) []setupChoice { +func setupBrokerUserChoicesFromUsers(users []protocol.UserInfo, exclude map[string]struct{}) []setupChoice { var choices []setupChoice groupPending := exclude != nil for _, user := range users { @@ -4785,16 +4504,20 @@ func setupBrokerRepoChoices(cfg config) ([]setupChoice, error) { return choices, nil } -func setupBrokerRepos(cfg config) ([]brokerRepoInfo, error) { +func setupBrokerRepos(cfg config) ([]protocol.RepositoryInfo, error) { brokerURL, err := brokerURLFromConfigOrDiscovery(cfg) if err != nil { return nil, err } - var resp brokerRepoListResponse - if err := brokerPost(brokerURL, "/repos/list", brokerRepoAdminRequest{}, &resp); err != nil { + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { return nil, err } - return resp.Repos, nil + repos, err := endpoints.ListRepositories(context.Background(), protocol.RepositoryAdminRequest{}) + if err != nil { + return nil, err + } + return repos, nil } func runSetupSelectWithRaw(reader *bufio.Reader, rawInput io.Reader, stdout io.Writer, title string, choices []setupChoice, selected string) (string, bool, error) { @@ -5696,7 +5419,7 @@ func setupAvailableCreateProviders() []string { } func (s setupDialogState) selection() setupSelection { - var profiles []setupProfile + var profiles []internalsetup.Profile for i, profile := range s.profiles { if i < len(s.selectedProfiles) && s.selectedProfiles[i] { profiles = append(profiles, profile) @@ -6106,7 +5829,7 @@ func (s setupDialogState) profileIndicesByProvider() map[string][]int { return byProvider } -func setupDialogProviderOrder(profiles []setupProfile, createProviders []string) []string { +func setupDialogProviderOrder(profiles []internalsetup.Profile, createProviders []string) []string { seen := map[string]struct{}{} for _, profile := range profiles { seen[profile.Provider] = struct{}{} @@ -6563,7 +6286,7 @@ func setupProviderLabel(provider string) string { return "gcp" } -func setupProfileDisplayName(profile setupProfile) string { +func setupProfileDisplayName(profile internalsetup.Profile) string { if profile.Existing && strings.TrimSpace(profile.Region) != "" { return profile.Name + "." + profile.Region } @@ -6587,18 +6310,22 @@ func brokerUpsertOwners(brokerURL, bootstrapToken string, publicKeys []string) e if strings.TrimSpace(bootstrapToken) != "" { headers["X-Bgit-Bootstrap-Token"] = strings.TrimSpace(bootstrapToken) } - return brokerPostJSONContextWithHeaders(context.Background(), brokerURL, "/owners/upsert", brokerOwnerRequest{User: "owner", Role: "owner", PublicKeys: publicKeys}, nil, headers) + return brokerPostJSONContextWithHeaders(context.Background(), brokerURL, "/owners/upsert", protocol.OwnerRequest{User: "owner", Role: "owner", PublicKeys: publicKeys}, nil, headers) } func brokerEnsureCoreTeam(brokerURL string) error { - err := brokerPost(brokerURL, "/teams/create", brokerRepoAdminRequest{TeamID: coreTeamID, Name: coreTeamName}, nil) + endpoints, clientErr := brokerEndpointClient(brokerURL) + if clientErr != nil { + return clientErr + } + _, err := endpoints.CreateTeam(context.Background(), protocol.RepositoryAdminRequest{TeamID: coreTeamID, Name: coreTeamName}) if err != nil && !strings.Contains(err.Error(), "team already exists") { return err } return nil } -func upsertGlobalGCPProfile(cfg globalConfig, profile globalGCPProfile) globalConfig { +func upsertGlobalGCPProfile(cfg internalconfig.Global, profile internalconfig.GCPProfile) internalconfig.Global { for i, existing := range cfg.GCPProfiles { if existing.Name == profile.Name { profile.Regions = mergeGlobalProfileRegions(existing.Regions, profile.Regions) @@ -6610,7 +6337,7 @@ func upsertGlobalGCPProfile(cfg globalConfig, profile globalGCPProfile) globalCo return cfg } -func upsertGlobalAWSProfile(cfg globalConfig, profile globalAWSProfile) globalConfig { +func upsertGlobalAWSProfile(cfg internalconfig.Global, profile internalconfig.AWSProfile) internalconfig.Global { for i, existing := range cfg.AWSProfiles { if existing.Name == profile.Name { profile.Regions = mergeGlobalProfileRegions(existing.Regions, profile.Regions) @@ -6622,7 +6349,7 @@ func upsertGlobalAWSProfile(cfg globalConfig, profile globalAWSProfile) globalCo return cfg } -func upsertGlobalLocalProfile(cfg globalConfig, profile globalLocalProfile) globalConfig { +func upsertGlobalLocalProfile(cfg internalconfig.Global, profile internalconfig.LocalProfile) internalconfig.Global { for i, existing := range cfg.LocalProfiles { if existing.Name == profile.Name { if strings.TrimSpace(profile.Root) == "" { @@ -6638,8 +6365,8 @@ func upsertGlobalLocalProfile(cfg globalConfig, profile globalLocalProfile) glob return cfg } -func mergeGlobalProfileRegions(existing, incoming []globalProfileRegion) []globalProfileRegion { - out := append([]globalProfileRegion{}, existing...) +func mergeGlobalProfileRegions(existing, incoming []internalconfig.ProfileRegion) []internalconfig.ProfileRegion { + out := append([]internalconfig.ProfileRegion{}, existing...) for _, next := range incoming { matched := false for i := range out { diff --git a/setup_test.go b/internal/app/setup_test.go similarity index 94% rename from setup_test.go rename to internal/app/setup_test.go index 6a949a2..13cc868 100644 --- a/setup_test.go +++ b/internal/app/setup_test.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -12,6 +12,10 @@ import ( "path/filepath" "strings" "testing" + + internalconfig "github.com/bucketgit/bgit/internal/config" + internalsetup "github.com/bucketgit/bgit/internal/setup" + "github.com/bucketgit/bgit/protocol" ) func TestParseAWSProfileFile(t *testing.T) { @@ -38,7 +42,7 @@ func TestResolveAWSSetupRegionUsesConfiguredRegion(t *testing.T) { bin := t.TempDir() writeFakeCLI(t, bin, "aws", []fakeCLIAction{}) t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) - got, err := resolveAWSSetupRegion(context.Background(), setupProfile{Name: "prod", Region: "eu-west-1"}, "", false, strings.NewReader(""), ioDiscard{}) + got, err := resolveAWSSetupRegion(context.Background(), internalsetup.Profile{Name: "prod", Region: "eu-west-1"}, "", false, strings.NewReader(""), ioDiscard{}) if err != nil { t.Fatal(err) } @@ -52,7 +56,7 @@ func TestResolveAWSSetupRegionPromptsFromEnabledRegions(t *testing.T) { writeFakeCLI(t, bin, "aws", []fakeCLIAction{}) t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) var stdout bytes.Buffer - got, err := resolveAWSSetupRegion(context.Background(), setupProfile{Name: "prod"}, "", true, strings.NewReader("\x1b[B \x04"), &stdout) + got, err := resolveAWSSetupRegion(context.Background(), internalsetup.Profile{Name: "prod"}, "", true, strings.NewReader("\x1b[B \x04"), &stdout) if err != nil { t.Fatal(err) } @@ -70,7 +74,7 @@ func TestResolveAWSSetupRegionYesModeRequiresRegion(t *testing.T) { bin := t.TempDir() writeFakeCLI(t, bin, "aws", []fakeCLIAction{}) t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) - errRegion, err := resolveAWSSetupRegion(context.Background(), setupProfile{Name: "prod"}, "", false, strings.NewReader(""), ioDiscard{}) + errRegion, err := resolveAWSSetupRegion(context.Background(), internalsetup.Profile{Name: "prod"}, "", false, strings.NewReader(""), ioDiscard{}) if err == nil || errRegion != "" || !strings.Contains(err.Error(), "pass --region REGION") { t.Fatalf("region=%q err=%v", errRegion, err) } @@ -78,7 +82,7 @@ func TestResolveAWSSetupRegionYesModeRequiresRegion(t *testing.T) { func TestResolveAWSSetupRegionRequiresAWSCLI(t *testing.T) { t.Setenv("PATH", t.TempDir()) - region, err := resolveAWSSetupRegion(context.Background(), setupProfile{Name: "prod", Region: "eu-west-1"}, "", true, strings.NewReader(""), ioDiscard{}) + region, err := resolveAWSSetupRegion(context.Background(), internalsetup.Profile{Name: "prod", Region: "eu-west-1"}, "", true, strings.NewReader(""), ioDiscard{}) if err == nil || region != "" || !strings.Contains(err.Error(), "AWS CLI is not installed") { t.Fatalf("region=%q err=%v", region, err) } @@ -86,7 +90,7 @@ func TestResolveAWSSetupRegionRequiresAWSCLI(t *testing.T) { func TestResolveGCPSetupRegionUsesDialog(t *testing.T) { var stdout bytes.Buffer - got, err := resolveGCPSetupRegion(setupProfile{Name: "work", Region: "us-central1"}, "", true, strings.NewReader("\x1b[B \x04"), &stdout) + got, err := resolveGCPSetupRegion(internalsetup.Profile{Name: "work", Region: "us-central1"}, "", true, strings.NewReader("\x1b[B \x04"), &stdout) if err != nil { t.Fatal(err) } @@ -102,7 +106,7 @@ func TestResolveGCPSetupRegionUsesDialog(t *testing.T) { func TestResolveGCPSetupRegionUsesExistingConfiguredRegion(t *testing.T) { var stdout bytes.Buffer - got, err := resolveGCPSetupRegion(setupProfile{Name: "work", Region: "europe-west1", Existing: true}, "", true, strings.NewReader("\x04"), &stdout) + got, err := resolveGCPSetupRegion(internalsetup.Profile{Name: "work", Region: "europe-west1", Existing: true}, "", true, strings.NewReader("\x04"), &stdout) if err != nil { t.Fatal(err) } @@ -115,13 +119,13 @@ func TestResolveGCPSetupRegionUsesExistingConfiguredRegion(t *testing.T) { } func TestConfiguredSetupProfilesCarryConfiguredRegion(t *testing.T) { - got := markConfiguredSetupProfiles([]setupProfile{{ + got := markConfiguredSetupProfiles([]internalsetup.Profile{{ Provider: "gcs", Name: "work", Region: "us-central1", - }}, globalConfig{GCPProfiles: []globalGCPProfile{{ + }}, internalconfig.Global{GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: "https://broker.example.test", }}, @@ -135,12 +139,12 @@ func TestConfiguredSetupProfilesCarryConfiguredRegion(t *testing.T) { } func TestConfiguredSetupProfilesExpandConfiguredRegions(t *testing.T) { - got := markConfiguredSetupProfiles([]setupProfile{{ + got := markConfiguredSetupProfiles([]internalsetup.Profile{{ Provider: "gcs", Name: "work", - }}, globalConfig{GCPProfiles: []globalGCPProfile{{ + }}, internalconfig.Global{GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "us-central1", BrokerURL: "https://us.example.test", }, { @@ -160,19 +164,19 @@ func TestConfiguredSetupProfilesExpandConfiguredRegions(t *testing.T) { } func TestConfiguredSetupBrokersSortedAndDetected(t *testing.T) { - cfg := globalConfig{ - GCPProfiles: []globalGCPProfile{{ + cfg := internalconfig.Global{ + GCPProfiles: []internalconfig.GCPProfile{{ Name: "work", ProjectID: "project-123", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "europe-west1", BrokerURL: "https://gcp.example.test", }}, }}, - AWSProfiles: []globalAWSProfile{{ + AWSProfiles: []internalconfig.AWSProfile{{ Name: "prod", AccountID: "123456789012", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "us-east-1", BrokerURL: "https://aws.example.test", }}, @@ -301,9 +305,9 @@ func TestSetupManagedTeamEscapeReturnsToPreviousMenu(t *testing.T) { } func TestSetupFormatsTeamTablesWithHeaders(t *testing.T) { - members := setupFormatTeamMembers(brokerTeamInfo{ + members := setupFormatTeamMembers(protocol.Team{ Name: "core", - Members: []brokerTeamMember{{ + Members: []protocol.TeamMember{{ Username: "owner", Role: "admin", }}, @@ -365,7 +369,7 @@ func TestSetupPendingUserNoteOnlyWhenPendingAndOutsideFrame(t *testing.T) { func TestSetupDialogRendersCheckboxesAndKeys(t *testing.T) { rendered := renderSetupDialog(setupDialogState{ - profiles: []setupProfile{{ + profiles: []internalsetup.Profile{{ Provider: "gcs", Name: "work", ProjectID: "example-test-123456", @@ -386,12 +390,12 @@ func TestSetupDialogRendersCheckboxesAndKeys(t *testing.T) { } func TestSetupDialogPaginatesProfilesPerProvider(t *testing.T) { - var profiles []setupProfile + var profiles []internalsetup.Profile for i := 0; i < 12; i++ { - profiles = append(profiles, setupProfile{Provider: "gcs", Name: "gcp" + string(rune('a'+i))}) + profiles = append(profiles, internalsetup.Profile{Provider: "gcs", Name: "gcp" + string(rune('a'+i))}) } for i := 0; i < 11; i++ { - profiles = append(profiles, setupProfile{Provider: "s3", Name: "aws" + string(rune('a'+i))}) + profiles = append(profiles, internalsetup.Profile{Provider: "s3", Name: "aws" + string(rune('a'+i))}) } rendered := renderSetupDialog(setupDialogState{ profiles: profiles, @@ -413,7 +417,7 @@ func TestSetupDialogPaginatesProfilesPerProvider(t *testing.T) { func TestSetupDialogTabJumpsBetweenProviders(t *testing.T) { state := setupDialogState{ - profiles: []setupProfile{ + profiles: []internalsetup.Profile{ {Provider: "gcs", Name: "work"}, {Provider: "s3", Name: "prod"}, }, @@ -430,7 +434,7 @@ func TestSetupDialogTabJumpsBetweenProviders(t *testing.T) { func TestSetupDialogHandlesKeyboardSelection(t *testing.T) { var stdout bytes.Buffer selected, err := runSetupDialog(strings.NewReader(" \x04"), &stdout, setupSelection{ - Profiles: []setupProfile{{ + Profiles: []internalsetup.Profile{{ Provider: "gcs", Name: "work", Active: true, @@ -454,7 +458,7 @@ func TestSetupDialogHandlesKeyboardSelection(t *testing.T) { func TestSetupDialogPreselectsSSHKeys(t *testing.T) { var stdout bytes.Buffer selected, err := runSetupDialog(strings.NewReader(" \x04"), &stdout, setupSelection{ - Profiles: []setupProfile{{ + Profiles: []internalsetup.Profile{{ Provider: "gcs", Name: "work", }}, @@ -493,7 +497,7 @@ func TestSetupDialogCreatesAWSProfileInApp(t *testing.T) { } func TestSetupCreateProfileDefaultsAvoidExistingDefault(t *testing.T) { - defaults := setupCreateProfileDefaults([]setupProfile{{Provider: "s3", Name: "default"}}, setupOptions{}) + defaults := setupCreateProfileDefaults([]internalsetup.Profile{{Provider: "s3", Name: "default"}}, setupOptions{}) if defaults["s3"] != "" { t.Fatalf("aws default = %q", defaults["s3"]) } @@ -537,7 +541,7 @@ func TestCreateAWSProfileConfiguredUsesAWSConfigureSet(t *testing.T) { func TestSetupDialogDoesNotDeployWithoutProfile(t *testing.T) { var stdout bytes.Buffer _, err := runSetupDialog(strings.NewReader("\x04\x03"), &stdout, setupSelection{ - Profiles: []setupProfile{{ + Profiles: []internalsetup.Profile{{ Provider: "gcs", Name: "work", Active: true, @@ -577,7 +581,7 @@ func TestSetupDialogDeploysIdentityOnlyWhenChanged(t *testing.T) { func TestSetupDialogEOFCancels(t *testing.T) { var stdout bytes.Buffer _, err := runSetupDialog(strings.NewReader(""), &stdout, setupSelection{ - Profiles: []setupProfile{{ + Profiles: []internalsetup.Profile{{ Provider: "gcs", Name: "work", Active: true, @@ -591,7 +595,7 @@ func TestSetupDialogEOFCancels(t *testing.T) { func TestSetupDialogCtrlCCancels(t *testing.T) { var stdout bytes.Buffer _, err := runSetupDialog(strings.NewReader("\x03"), &stdout, setupSelection{ - Profiles: []setupProfile{{ + Profiles: []internalsetup.Profile{{ Provider: "gcs", Name: "work", Active: true, @@ -636,7 +640,7 @@ func TestSetupCommandProvisionsGCPAndWritesGlobalConfig(t *testing.T) { if err := os.WriteFile(pubKey, []byte("ssh-ed25519 AAAAOWNER owner@example\n"), 0o644); err != nil { t.Fatal(err) } - var ownerReq brokerOwnerRequest + var ownerReq protocol.OwnerRequest var bootstrapToken string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -1044,12 +1048,12 @@ func TestGcloudIAMBindingRetryableDetectsServiceAccountPropagation(t *testing.T) func TestBrokerDeleteAWSDeletesStackAndClearsConfig(t *testing.T) { home := t.TempDir() configPath := filepath.Join(home, ".bgit", "config") - if err := writeGlobalConfig(configPath, globalConfig{ + if err := writeGlobalConfig(configPath, internalconfig.Global{ Version: globalConfigVersion, - AWSProfiles: []globalAWSProfile{{ + AWSProfiles: []internalconfig.AWSProfile{{ Name: "prod", AccountID: "123456789012", - Regions: []globalProfileRegion{{ + Regions: []internalconfig.ProfileRegion{{ Name: "eu-west-1", BrokerURL: "https://broker.example.test", BrokerVersion: brokerVersion(), @@ -1111,10 +1115,10 @@ func TestSetupBrokerTeamRepoForActionPreservesTeamID(t *testing.T) { } func TestSetupAvailableTeamUserChoicesGroupPendingAndExcludeMembers(t *testing.T) { - choices := setupBrokerUserChoicesFromUsers([]brokerUserInfo{{ + choices := setupBrokerUserChoicesFromUsers([]protocol.UserInfo{{ Username: "owner", BrokerRole: "admin", - Keys: []brokerKey{{PublicKey: "ssh-ed25519 AAAA owner"}}, + Keys: []protocol.Key{{PublicKey: "ssh-ed25519 AAAA owner"}}, }, { Username: "pending", BrokerRole: "user", @@ -1122,7 +1126,7 @@ func TestSetupAvailableTeamUserChoicesGroupPendingAndExcludeMembers(t *testing.T }, { Username: "developer", BrokerRole: "user", - Keys: []brokerKey{{PublicKey: "ssh-ed25519 AAAA dev"}}, + Keys: []protocol.Key{{PublicKey: "ssh-ed25519 AAAA dev"}}, }}, map[string]struct{}{"owner": {}}) if len(choices) != 2 { t.Fatalf("choices = %#v", choices) @@ -1206,7 +1210,7 @@ func TestSetupBrokerUserManagementChoicesNestUsers(t *testing.T) { } func TestSetupBrokerOwnerUserOnlyShowsTransfer(t *testing.T) { - choices := setupBrokerUserActionChoices(brokerUserInfo{Username: "owner", BrokerRole: "owner", Keys: []brokerKey{{PublicKey: "ssh-ed25519 AAAA owner"}}}) + choices := setupBrokerUserActionChoices(protocol.UserInfo{Username: "owner", BrokerRole: "owner", Keys: []protocol.Key{{PublicKey: "ssh-ed25519 AAAA owner"}}}) if len(choices) != 2 || choices[0].Value != "transfer-owner" || choices[1].Value != "back" { t.Fatalf("choices = %#v", choices) } @@ -1219,7 +1223,7 @@ func TestSetupBrokerOwnerUserOnlyShowsTransfer(t *testing.T) { } func TestSetupBrokerRegularUserShowsDelete(t *testing.T) { - choices := setupBrokerUserActionChoices(brokerUserInfo{Username: "ada", BrokerRole: "user", Keys: []brokerKey{{PublicKey: "ssh-ed25519 AAAA ada"}}}) + choices := setupBrokerUserActionChoices(protocol.UserInfo{Username: "ada", BrokerRole: "user", Keys: []protocol.Key{{PublicKey: "ssh-ed25519 AAAA ada"}}}) var sawDelete bool for _, choice := range choices { if choice.Value == "delete" { diff --git a/ssh.go b/internal/app/ssh.go similarity index 76% rename from ssh.go rename to internal/app/ssh.go index e4f4319..b116764 100644 --- a/ssh.go +++ b/internal/app/ssh.go @@ -1,4 +1,4 @@ -package main +package app import ( "bufio" @@ -7,12 +7,10 @@ import ( "crypto/rand" "crypto/sha256" "encoding/base64" - "encoding/json" "errors" "fmt" "io" "net/http" - "net/url" "os" "os/exec" "path/filepath" @@ -20,6 +18,10 @@ import ( "strings" "time" + brokerclient "github.com/bucketgit/bgit/broker/client" + internalidentity "github.com/bucketgit/bgit/internal/identity" + internalsetup "github.com/bucketgit/bgit/internal/setup" + "github.com/bucketgit/bgit/protocol" "golang.org/x/crypto/ssh" ) @@ -234,73 +236,6 @@ func mergeSSHRepoAuth(cfg config) config { return cfg } -func parseSSHSetupArgs(args []string) (sshSetupOptions, string, error) { - var opts sshSetupOptions - var repoArg string - for i := 0; i < len(args); i++ { - arg := args[i] - name, value, hasValue := strings.Cut(arg, "=") - switch name { - case "--broker": - if !hasValue { - i++ - if i >= len(args) { - return opts, "", errors.New("--broker requires a value") - } - value = args[i] - } - opts.broker = value - case "--key": - if !hasValue { - i++ - if i >= len(args) { - return opts, "", errors.New("--key requires a value") - } - value = args[i] - } - opts.keys = append(opts.keys, value) - case "--region": - if !hasValue { - i++ - if i >= len(args) { - return opts, "", errors.New("--region requires a value") - } - value = args[i] - } - opts.region = value - case "--firestore-database": - if !hasValue { - i++ - if i >= len(args) { - return opts, "", errors.New("--firestore-database requires a value") - } - value = args[i] - } - opts.firestoreDatabase = value - case "--firestore-location": - if !hasValue { - i++ - if i >= len(args) { - return opts, "", errors.New("--firestore-location requires a value") - } - value = args[i] - } - opts.firestoreLocation = value - case "--no-agent": - opts.noAgent = true - default: - if strings.HasPrefix(arg, "-") { - return opts, "", fmt.Errorf("unsupported ssh option %s", arg) - } - if repoArg != "" { - return opts, "", errors.New("ssh commands accept at most one repository URI") - } - repoArg = arg - } - } - return opts, repoArg, nil -} - func sshSetupConfig(base config, repoArg string) (config, error) { if strings.TrimSpace(repoArg) != "" { cfg, _, err := parseRepoURI(repoArg) @@ -537,109 +472,15 @@ func sshRemoteURL(cfg config) string { return fmt.Sprintf("git@%s:%s", defaultSSHHost, repo) } -func writeBrokerConfig(worktree, brokerURL string, stdout io.Writer) error { - if strings.TrimSpace(brokerURL) == "" { - return nil - } - if _, err := runGit(worktree, "config", "--local", "bucketgit.broker", strings.TrimSpace(brokerURL)); err != nil { - return err - } - fmt.Fprintf(stdout, "configured broker %s\n", strings.TrimSpace(brokerURL)) - return nil -} - -type brokerRepo struct { - Provider string `json:"provider"` - Bucket string `json:"bucket"` - Prefix string `json:"prefix"` - Origin string `json:"origin"` - Logical string `json:"logical,omitempty"` - Host string `json:"host,omitempty"` - Profile string `json:"profile,omitempty"` - Region string `json:"region,omitempty"` - TeamID string `json:"team_id,omitempty"` - TeamName string `json:"team_name,omitempty"` -} - const coreTeamID = "t_core" const coreTeamName = "core" -type brokerKey struct { - User string `json:"user"` - Role string `json:"role"` - PublicKey string `json:"public_key"` - Source string `json:"source,omitempty"` - Suspended bool `json:"suspended,omitempty"` -} - -type brokerRepoRequest struct { - Repo brokerRepo `json:"repo"` - AdminUser string `json:"admin_user,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` - Role string `json:"role,omitempty"` -} - -type brokerKeyRequest struct { - Repo brokerRepo `json:"repo"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` - Key string `json:"key,omitempty"` - Source string `json:"source,omitempty"` -} - -type brokerAuthRequest struct { - Repo brokerRepo `json:"repo"` - Operation string `json:"operation"` -} - -type brokerAuthResponse struct { - Allowed bool `json:"allowed"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` -} - -type brokerRefUpdateRequest struct { - Repo brokerRepo `json:"repo"` - Ref string `json:"ref"` - Old string `json:"old"` - New string `json:"new"` - Override bool `json:"override,omitempty"` -} - -type brokerKeysResponse struct { - Keys []brokerKey `json:"keys"` -} - -func brokerUpsertLogicalRepo(brokerURL, provider, logicalRepo string, teamID ...string) error { - logical, err := normalizeLogicalRepoName(logicalRepo) +func brokerListKeys(brokerURL string, cfg config) ([]protocol.Key, error) { + endpoints, err := brokerEndpointClient(brokerURL) if err != nil { - return err - } - team := "" - if len(teamID) > 0 { - team = strings.TrimSpace(teamID[0]) - } - cfg := config{ - provider: provider, - prefix: logical, - logicalRepo: logical, - origin: fmt.Sprintf("git@%s:%s", defaultSSHHost, logical), - } - repo := repoForBroker(cfg) - if team != "" { - repo.TeamID = team - } - req := brokerRepoRequest{Repo: repo} - return brokerPost(brokerURL, "/repos/upsert", req, nil) -} - -func brokerListKeys(brokerURL string, cfg config) ([]brokerKey, error) { - var resp brokerKeysResponse - if err := brokerPost(brokerURL, "/keys/list", brokerKeyRequest{Repo: repoForBroker(cfg)}, &resp); err != nil { return nil, err } - return resp.Keys, nil + return endpoints.ListKeys(context.Background(), protocol.KeyRequest{Repo: repoForBroker(cfg)}) } func brokerAddKeys(brokerURL string, cfg config, user, role string, publicKeys []string) error { @@ -651,18 +492,36 @@ func brokerAddKeysWithSource(brokerURL string, cfg config, user, role, source st if !validBrokerRole(role) { return fmt.Errorf("invalid broker role %q", role) } - req := brokerKeyRequest{ + req := protocol.KeyRequest{ Repo: repoForBroker(cfg), User: user, Role: role, PublicKeys: publicKeys, Source: source, } - return brokerPost(brokerURL, "/keys/add", req, nil) + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + return endpoints.AddKey(context.Background(), req) } func brokerMutateKey(brokerURL, path string, cfg config, key string) error { - return brokerPost(brokerURL, path, brokerKeyRequest{Repo: repoForBroker(cfg), Key: key}, nil) + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + request := protocol.KeyRequest{Repo: repoForBroker(cfg), Key: key} + switch path { + case "/keys/remove": + return endpoints.RemoveKey(context.Background(), request) + case "/keys/suspend": + return endpoints.SuspendKey(context.Background(), request, true) + case "/keys/unsuspend": + return endpoints.SuspendKey(context.Background(), request, false) + default: + return fmt.Errorf("unsupported key mutation endpoint %q", path) + } } func validBrokerRole(role string) bool { @@ -688,14 +547,18 @@ func brokerUpdateRef(brokerURL string, cfg config, ref, oldHash, newHash string) } func brokerUpdateRefWithOverride(brokerURL string, cfg config, ref, oldHash, newHash string, override bool) error { - req := brokerRefUpdateRequest{ + req := protocol.RefUpdateRequest{ Repo: repoForBroker(cfg), Ref: ref, Old: firstNonEmpty(strings.TrimSpace(oldHash), zeroObjectID()), New: firstNonEmpty(strings.TrimSpace(newHash), zeroObjectID()), Override: override, } - return brokerPost(brokerURL, "/refs/update", req, nil) + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + return endpoints.UpdateRef(context.Background(), req) } func optionalBrokerURLForPush() string { @@ -707,6 +570,7 @@ func optionalBrokerURLForPush() string { } func brokerPushError(err error) error { + //lint:ignore ST1005 Multiline CLI remediation is intentionally formatted and capitalized. return fmt.Errorf("%w\n\nBroker ref coordination failed. Retry after fetching, or use --skip-broker for a direct bucket push if this is an operator recovery action.", err) } @@ -719,9 +583,13 @@ func authorizeSSHGitService(cfg config, service string) error { if err != nil { return err } - var resp brokerAuthResponse - req := brokerAuthRequest{Repo: repoForBroker(cfg), Operation: operation} - if err := brokerPost(brokerURL, "/auth/check", req, &resp); err != nil { + req := protocol.AuthRequest{Repo: repoForBroker(cfg), Operation: operation} + endpoints, err := brokerEndpointClient(brokerURL) + if err != nil { + return err + } + resp, err := endpoints.Authorize(context.Background(), req) + if err != nil { return err } if !resp.Allowed { @@ -761,7 +629,7 @@ func brokerURLForSSHService(cfg config) (string, error) { return url, nil } -func repoForBroker(cfg config) brokerRepo { +func repoForBroker(cfg config) protocol.Repository { if cfg.origin == "" { cfg.origin = originForConfig(cfg) } @@ -769,7 +637,7 @@ func repoForBroker(cfg config) brokerRepo { if normalized, err := normalizeLogicalRepoName(logical); err == nil { logical = normalized } - return brokerRepo{ + return protocol.Repository{ Provider: firstNonEmpty(cfg.storageProvider, cfg.provider, "gcs"), Bucket: cfg.bucket, Prefix: strings.Trim(cfg.prefix, "/"), @@ -789,82 +657,86 @@ func brokerTeamIDForConfig(cfg config) string { return teamID } -func brokerPost(brokerURL, path string, req any, resp any) error { - return brokerPostContext(context.Background(), brokerURL, path, req, resp) -} - -func brokerPostContext(ctx context.Context, brokerURL, path string, req any, resp any) error { - if isLocalBrokerURL(brokerURL) { - return localBrokerPostContext(ctx, brokerURL, path, req, resp) - } - return brokerPostJSONContextWithHeaders(ctx, brokerURL, path, req, resp, nil) -} - func brokerPostJSONContextWithHeaders(ctx context.Context, brokerURL, path string, req any, resp any, extraHeaders map[string]string) error { if isLocalBrokerURL(brokerURL) { return localBrokerPostContext(ctx, brokerURL, path, req, resp) } - endpoint := strings.TrimRight(brokerURL, "/") + path - data, err := json.Marshal(req) + c, err := newBrokerHTTPClient(brokerURL) if err != nil { return err } - headerSets := brokerSignatureHeaderSetsForBroker(brokerURL, path, data) - if len(headerSets) == 0 { - headerSets = []map[string]string{{}} - } else { - headerSets = append(headerSets, map[string]string{}) + headers := http.Header{} + for key, value := range extraHeaders { + headers.Set(key, value) } - var lastErr error - for i, headers := range headerSets { - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(data)) - if err != nil { - return err - } - httpReq.Header.Set("content-type", "application/json") - for key, value := range headers { - httpReq.Header.Set(key, value) - } - for key, value := range extraHeaders { - httpReq.Header.Set(key, value) - } - httpResp, err := http.DefaultClient.Do(httpReq) - if err != nil { - return err - } - body, readErr := io.ReadAll(httpResp.Body) - _ = httpResp.Body.Close() - if readErr != nil { - return readErr - } - if httpResp.StatusCode >= 200 && httpResp.StatusCode < 300 { - if fingerprint := headers["X-Bgit-Key-Fingerprint"]; fingerprint != "" { - _ = writeRepoAuthCache(brokerURL, data, fingerprint) - } - if resp != nil && len(body) > 0 { - if err := json.Unmarshal(body, resp); err != nil { - return err + return c.PostJSON(ctx, path, req, resp, headers) +} + +func newBrokerHTTPClient(brokerURL string) (*brokerclient.Client, error) { + return brokerclient.New(brokerURL, brokerclient.Options{ + Signatures: brokerclient.SignatureProviderFunc(func(_ context.Context, baseURL, endpoint string, payload []byte) ([]http.Header, error) { + sets := brokerSignatureHeaderSetsForBroker(baseURL, endpoint, payload) + headers := make([]http.Header, 0, len(sets)) + for _, set := range sets { + header := http.Header{} + for key, value := range set { + header.Set(key, value) } + headers = append(headers, header) } - return nil - } - msg := strings.TrimSpace(string(body)) - if msg == "" { - msg = httpResp.Status - } - lastErr = brokerHTTPError(path, msg) - if httpResp.StatusCode != http.StatusForbidden || i == len(headerSets)-1 || !brokerForbiddenAllowsSignatureRetry(msg) { - return lastErr - } + return headers, nil + }), + AllowUnsignedFallback: true, + DecodeError: func(endpoint string, status int, message string) error { + return brokerHTTPStatusError(endpoint, status, message) + }, + Retry: func(status int, message string) bool { + return status == http.StatusForbidden && brokerForbiddenAllowsSignatureRetry(message) + }, + ObserveSuccess: func(baseURL string, payload []byte, headers http.Header) { + if fingerprint := headers.Get(protocol.HeaderKeyFingerprint); fingerprint != "" { + _ = writeRepoAuthCache(baseURL, payload, fingerprint) + } + }, + }) +} + +type localBrokerCaller struct{ brokerURL string } + +func (c localBrokerCaller) PostJSON(ctx context.Context, endpoint string, request, response any, _ http.Header) error { + return localBrokerPostContext(ctx, c.brokerURL, endpoint, request, response) +} + +func brokerEndpointClient(brokerURL string) (*brokerclient.Endpoints, error) { + if isLocalBrokerURL(brokerURL) { + return brokerclient.NewEndpoints(localBrokerCaller{brokerURL: brokerURL}), nil + } + client, err := newBrokerHTTPClient(brokerURL) + if err != nil { + return nil, err } - return lastErr + return brokerclient.NewEndpoints(client), nil } func brokerHTTPError(path, msg string) error { + return brokerHTTPStatusError(path, 0, msg) +} + +func brokerHTTPStatusError(path string, status int, msg string) error { + kind := error(nil) + lower := strings.ToLower(msg) + switch { + case status == http.StatusConflict: + kind = protocol.ErrConflict + case status == http.StatusUnauthorized || status == http.StatusForbidden: + kind = protocol.ErrUnauthorized + case status == http.StatusNotImplemented || strings.Contains(lower, "unknown broker endpoint"): + kind = protocol.ErrUnsupported + } if brokerLooksIncompatibleWithV2Signatures(msg) { - return fmt.Errorf("broker %s: %s\nbroker is incompatible with this bgit version and must be upgraded for v2 request signatures; run `bgit admin broker upgrade` with a bgit version that can administer this broker", path, msg) + msg += "\nbroker is incompatible with this bgit version and must be upgraded for v2 request signatures; run `bgit admin broker upgrade` with a bgit version that can administer this broker" } - return fmt.Errorf("broker %s: %s", path, msg) + return &protocol.BrokerError{Endpoint: path, Status: status, Message: msg, Kind: kind} } func brokerLooksIncompatibleWithV2Signatures(msg string) bool { @@ -888,18 +760,6 @@ func brokerForbiddenAllowsSignatureRetry(msg string) bool { return strings.Contains(msg, "ssh signature required") } -func brokerSignatureHeaders(payload []byte) map[string]string { - sets := brokerSignatureHeaderSetsForBroker("", "/", payload) - if len(sets) == 0 { - return map[string]string{} - } - return sets[0] -} - -func brokerSignatureHeaderSets(payload []byte) []map[string]string { - return brokerSignatureHeaderSetsForBroker("", "/", payload) -} - func brokerSignatureHeaderSetsForBroker(brokerURL, requestPath string, payload []byte) []map[string]string { signers := explicitBrokerSigners() agentSigners, cleanup, err := sshAgentSigners() @@ -912,137 +772,46 @@ func brokerSignatureHeaderSetsForBroker(brokerURL, requestPath string, payload [ if cleanup != nil { defer cleanup() } - host := brokerSignatureHost(brokerURL) preferred := preferredBrokerKeyFingerprints(brokerURL, payload) - type signedHeaders struct { - fingerprint string - headers map[string]string - } - var signed []signedHeaders - for _, signer := range signers { - timestamp := fmt.Sprintf("%d", time.Now().Unix()) - nonce, err := randomBrokerSignatureNonce() - if err != nil { - continue - } - message := brokerSignatureMessage(http.MethodPost, requestPath, host, timestamp, nonce, payload) - sig, err := signBrokerMessage(signer, message) - if err != nil { - continue - } - fingerprint := ssh.FingerprintSHA256(signer.PublicKey()) - signed = append(signed, signedHeaders{fingerprint: fingerprint, headers: map[string]string{ - "X-Bgit-Signature-Version": "2", - "X-Bgit-Key": strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey()))), - "X-Bgit-Key-Fingerprint": fingerprint, - "X-Bgit-Timestamp": timestamp, - "X-Bgit-Nonce": nonce, - "X-Bgit-Signed-Host": host, - "X-Bgit-Signature": base64.StdEncoding.EncodeToString(ssh.Marshal(sig)), - "X-Bgit-Signature-Message": base64.StdEncoding.EncodeToString(message), - }}) - } - sort.SliceStable(signed, func(i, j int) bool { - return preferredBrokerKeyRank(signed[i].fingerprint, preferred) < preferredBrokerKeyRank(signed[j].fingerprint, preferred) + provider := brokerclient.NewV2Signatures(brokerclient.V2SignatureOptions{ + Signers: signers, + Rank: func(fingerprint string) int { + return preferredBrokerKeyRank(fingerprint, preferred) + }, }) - var sets []map[string]string - for _, item := range signed { - sets = append(sets, item.headers) - } - return sets -} - -func brokerSignatureHost(brokerURL string) string { - u, err := url.Parse(strings.TrimSpace(brokerURL)) - if err != nil || u.Host == "" { - return "" - } - return strings.ToLower(u.Host) -} - -func randomBrokerSignatureNonce() (string, error) { - var nonce [16]byte - if _, err := rand.Read(nonce[:]); err != nil { - return "", err + headers, err := provider.HeaderSets(context.Background(), brokerURL, requestPath, payload) + if err != nil { + return nil } - return base64.RawURLEncoding.EncodeToString(nonce[:]), nil -} - -func signBrokerMessage(signer ssh.Signer, message []byte) (*ssh.Signature, error) { - if signer.PublicKey().Type() == ssh.KeyAlgoRSA { - if algorithmSigner, ok := signer.(ssh.AlgorithmSigner); ok { - if sig, err := algorithmSigner.SignWithAlgorithm(rand.Reader, message, ssh.KeyAlgoRSASHA256); err == nil { - return sig, nil + sets := make([]map[string]string, 0, len(headers)) + for _, header := range headers { + item := map[string]string{} + for key, values := range header { + if len(values) > 0 { + item[key] = values[0] } } + sets = append(sets, item) } - return signer.Sign(rand.Reader, message) + return sets } func explicitBrokerSigners() []ssh.Signer { - var paths []string - for _, envName := range []string{"BGIT_SSH_KEY", "BGIT_SSH_KEYS"} { - for _, value := range filepath.SplitList(os.Getenv(envName)) { - if value = strings.TrimSpace(value); value != "" { - paths = append(paths, value) - } - } - } - var inlineKeys []string - if value := strings.TrimSpace(os.Getenv("GIT_SSH_PRIVATE_KEY")); value != "" { - inlineKeys = append(inlineKeys, value) - } - if value := strings.TrimSpace(brokerIdentityPreference); value != "" && !strings.HasPrefix(value, "SHA256:") { - paths = append(paths, value) - } - seen := map[string]struct{}{} - var signers []ssh.Signer - for _, key := range inlineKeys { - signer, err := ssh.ParsePrivateKey([]byte(key)) - if err != nil { - continue - } - signers = append(signers, signer) - } - for _, path := range paths { - path = expandHome(path) - if _, ok := seen[path]; ok { - continue - } - seen[path] = struct{}{} - data, err := os.ReadFile(path) - if err != nil { - continue - } - signer, err := ssh.ParsePrivateKey(data) - if err != nil { - continue - } - signers = append(signers, signer) - } - return signers + return internalidentity.ExplicitSigners(brokerIdentityPreference) } func preferredBrokerKeyRank(fingerprint string, preferred []string) int { - for i, value := range preferred { - if strings.EqualFold(strings.TrimSpace(value), strings.TrimSpace(fingerprint)) { - return i - } - } - return len(preferred) + 1 + return internalidentity.FingerprintRank(fingerprint, preferred) } func brokerSignatureMessage(method, requestPath, host, timestamp, nonce string, payload []byte) []byte { - sum := sha256.Sum256(payload) - return []byte(strings.Join([]string{ - "bgit-broker-v2", - strings.ToUpper(strings.TrimSpace(method)), - firstNonEmpty(requestPath, "/"), - strings.ToLower(strings.TrimSpace(host)), - strings.TrimSpace(timestamp), - strings.TrimSpace(nonce), - fmt.Sprintf("%x", sum[:]), - }, "\n")) + return protocol.SignatureMessage(protocol.RequestMetadata{ + Method: method, + Path: requestPath, + Host: host, + Timestamp: timestamp, + Nonce: nonce, + }, payload) } func discoverBrokerURL(cfg config, opts sshSetupOptions) (string, error) { @@ -1108,14 +877,6 @@ func discoverAWSBrokerURL(cfg config, opts sshSetupOptions) (string, error) { return "", fmt.Errorf("bgit broker was not found in AWS region %s", region) } -func provisionBrokerURL(cfg config, opts sshSetupOptions, stdout io.Writer) (string, error) { - broker, err := provisionBroker(cfg, opts, stdout) - if err != nil { - return "", err - } - return broker.URL, nil -} - func provisionBroker(cfg config, opts sshSetupOptions, stdout io.Writer) (provisionedBroker, error) { token, err := randomBrokerSecret() if err != nil { @@ -1180,19 +941,11 @@ func provisionGCPBrokerURLWithBootstrap(cfg config, opts sshSetupOptions, stdout return "", err } fmt.Fprintf(stdout, "deploying GCP Cloud Run function bgit-broker in %s\n", region) - cmd := gcloudCommand(cfg.gcloudConfiguration, - "functions", "deploy", "bgit-broker", - "--gen2", - "--runtime", "nodejs22", - "--region", region, - "--source", sourceDir, - "--entry-point", "broker", - "--trigger-http", - "--allow-unauthenticated", - "--service-account", serviceAccount, - "--set-env-vars", gcpBrokerEnvVars(cfg, opts, serviceAccount, ciSecret, ciMaterializerURL, bootstrapHash), - "--quiet", - ) + cmd := internalsetup.GCPFunctionDeployCommand(context.Background(), internalsetup.GCPFunctionDeployment{ + Configuration: cfg.gcloudConfiguration, Name: "bgit-broker", Region: region, + Source: sourceDir, EntryPoint: "broker", ServiceAccount: serviceAccount, + Environment: gcpBrokerEnvVars(cfg, opts, serviceAccount, ciSecret, ciMaterializerURL, bootstrapHash), Public: true, + }) out, err := cmd.CombinedOutput() if err != nil { return "", fmt.Errorf("deploy GCP bgit broker: %w\n%s", err, strings.TrimSpace(string(out))) @@ -1213,19 +966,11 @@ func provisionGCPMaterializerURL(cfg config, region, serviceAccount, ciSecret st return "", err } fmt.Fprintf(stdout, "deploying GCP Cloud Run function bgit-ci-materializer in %s\n", region) - cmd := gcloudCommand(cfg.gcloudConfiguration, - "functions", "deploy", "bgit-ci-materializer", - "--gen2", - "--runtime", "nodejs22", - "--region", region, - "--source", sourceDir, - "--entry-point", "materializer", - "--trigger-http", - "--no-allow-unauthenticated", - "--service-account", serviceAccount, - "--set-env-vars", gcpMaterializerEnvVars(cfg, serviceAccount, ciSecret), - "--quiet", - ) + cmd := internalsetup.GCPFunctionDeployCommand(context.Background(), internalsetup.GCPFunctionDeployment{ + Configuration: cfg.gcloudConfiguration, Name: "bgit-ci-materializer", Region: region, + Source: sourceDir, EntryPoint: "materializer", ServiceAccount: serviceAccount, + Environment: gcpMaterializerEnvVars(cfg, serviceAccount, ciSecret), + }) out, err := cmd.CombinedOutput() if err != nil { return "", fmt.Errorf("deploy GCP bgit CI materializer: %w\n%s", err, strings.TrimSpace(string(out))) @@ -1695,18 +1440,12 @@ func provisionAWSBrokerURLWithBootstrap(cfg config, opts sshSetupOptions, stdout if err != nil { return "", err } - parameterOverrides := []string{"OwnerBootstrapHash=" + bootstrapHash} - args := []string{ - "cloudformation", "deploy", - "--stack-name", "bgit-broker", - "--template-file", templatePath, - "--s3-bucket", s3Bucket, - "--capabilities", "CAPABILITY_NAMED_IAM", - "--region", region, - "--parameter-overrides", - } - args = append(args, parameterOverrides...) - out, err := awsCommand(context.Background(), strings.TrimSpace(cfg.gcloudConfiguration), args...).CombinedOutput() + cmd := internalsetup.AWSStackDeployCommand(context.Background(), internalsetup.AWSStackDeployment{ + Profile: strings.TrimSpace(cfg.gcloudConfiguration), Region: region, StackName: "bgit-broker", + TemplatePath: templatePath, ArtifactBucket: s3Bucket, Capability: "CAPABILITY_NAMED_IAM", + ParameterOverrides: []string{"OwnerBootstrapHash=" + bootstrapHash}, + }) + out, err := cmd.CombinedOutput() if err != nil { return "", fmt.Errorf("deploy AWS bgit broker: %w\n%s", err, strings.TrimSpace(string(out))) } @@ -1739,22 +1478,8 @@ func ensureAWSBrokerDeploymentBucket(cfg config, region string, stdout io.Writer return bucket, nil } -func appendAWSProfile(args []string, profile string) []string { - if strings.TrimSpace(profile) == "" { - return args - } - return append(args, "--profile", strings.TrimSpace(profile)) -} - func awsCommand(ctx context.Context, profile string, args ...string) *exec.Cmd { - cmd := exec.CommandContext(ctx, "aws", args...) - if strings.TrimSpace(profile) != "" { - cmd.Env = append(os.Environ(), - "AWS_PROFILE="+strings.TrimSpace(profile), - "AWS_SDK_LOAD_CONFIG=1", - ) - } - return cmd + return internalsetup.AWSCommand(ctx, profile, args...) } func cleanBrokerURL(out string) string { @@ -1780,9 +1505,7 @@ func collectSSHPublicKeys(opts sshSetupOptions) ([]string, error) { if err != nil { return nil, err } - for _, line := range splitPublicKeyLines(string(data)) { - keys = append(keys, line) - } + keys = append(keys, splitPublicKeyLines(string(data))...) } if !opts.noAgent { agentKeys, err := sshAgentPublicKeys() @@ -1815,16 +1538,6 @@ func sshAgentPublicKeys() ([]string, error) { return splitPublicKeyLines(string(out)), nil } -func writeSSHKeyDefaults(worktree string, keys []string) error { - for i, key := range keys { - name := fmt.Sprintf("bucketgit.sshkey%d", i+1) - if _, err := runGit(worktree, "config", "--local", name, key); err != nil { - return err - } - } - return nil -} - func expandHome(path string) string { if path == "~" { if home, err := os.UserHomeDir(); err == nil { diff --git a/internal/app/ssh_agent_unix.go b/internal/app/ssh_agent_unix.go new file mode 100644 index 0000000..5d49601 --- /dev/null +++ b/internal/app/ssh_agent_unix.go @@ -0,0 +1,12 @@ +//go:build !windows + +package app + +import ( + internalidentity "github.com/bucketgit/bgit/internal/identity" + "golang.org/x/crypto/ssh" +) + +func sshAgentSigners() ([]ssh.Signer, func(), error) { + return internalidentity.AgentSigners() +} diff --git a/internal/app/ssh_agent_windows.go b/internal/app/ssh_agent_windows.go new file mode 100644 index 0000000..1237b7e --- /dev/null +++ b/internal/app/ssh_agent_windows.go @@ -0,0 +1,12 @@ +//go:build windows + +package app + +import ( + internalidentity "github.com/bucketgit/bgit/internal/identity" + "golang.org/x/crypto/ssh" +) + +func sshAgentSigners() ([]ssh.Signer, func(), error) { + return internalidentity.AgentSigners() +} diff --git a/web.go b/internal/app/web.go similarity index 90% rename from web.go rename to internal/app/web.go index 34bc228..e60331f 100644 --- a/web.go +++ b/internal/app/web.go @@ -1,11 +1,9 @@ -package main +package app import ( "archive/zip" "bytes" "context" - "crypto/rand" - "embed" "encoding/base64" "encoding/json" "errors" @@ -24,17 +22,15 @@ import ( "sort" "strconv" "strings" - "sync" "time" "unicode" "unicode/utf8" + internalweb "github.com/bucketgit/bgit/internal/web" + "github.com/bucketgit/bgit/protocol" "golang.org/x/crypto/ssh" ) -//go:embed www/* -var webAssets embed.FS - const ( defaultWebAddr = "127.0.0.1" defaultWebPort = 8042 @@ -61,22 +57,6 @@ type webServer struct { csrfToken string } -type brokerGitStore struct { - brokerURL string - cfg config -} - -type brokerObjectRequest struct { - Repo brokerRepo `json:"repo"` - Path string `json:"path,omitempty"` - Prefix string `json:"prefix,omitempty"` -} - -type brokerObjectResponse struct { - Data string `json:"data,omitempty"` - Paths []string `json:"paths,omitempty"` -} - type webTreeFile struct { path string hash string @@ -160,56 +140,8 @@ type webAPIState struct { } type webPullRequestCache struct { - UpdatedAt int64 `json:"updated_at"` - PRs []brokerPullRequest `json:"prs"` -} - -type brokerIssue struct { - ID int `json:"id,omitempty"` - Type string `json:"type,omitempty"` - Title string `json:"title,omitempty"` - Body string `json:"body,omitempty"` - Status string `json:"status,omitempty"` - Lane string `json:"lane,omitempty"` - Assignee string `json:"assignee,omitempty"` - Position float64 `json:"position,omitempty"` - Archived bool `json:"archived,omitempty"` - Author string `json:"author,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` - Comments []brokerIssueReply `json:"comments,omitempty"` - History []brokerIssueEvent `json:"history,omitempty"` -} - -type brokerIssueReply struct { - User string `json:"user,omitempty"` - Body string `json:"body,omitempty"` - At string `json:"at,omitempty"` -} - -type brokerIssueEvent struct { - User string `json:"user,omitempty"` - Action string `json:"action,omitempty"` - From string `json:"from,omitempty"` - To string `json:"to,omitempty"` - At string `json:"at,omitempty"` - Ref string `json:"ref,omitempty"` - Position string `json:"position,omitempty"` -} - -type brokerIssueRequest struct { - Repo brokerRepo `json:"repo"` - ID int `json:"id,omitempty"` - Type string `json:"type,omitempty"` - Title string `json:"title,omitempty"` - Body string `json:"body,omitempty"` - Lane string `json:"lane,omitempty"` - Assignee string `json:"assignee,omitempty"` - Comment string `json:"comment,omitempty"` - AfterID *int `json:"after_id,omitempty"` - Order int `json:"order,omitempty"` - Archived bool `json:"archived,omitempty"` - IncludeArchived bool `json:"include_archived,omitempty"` + UpdatedAt int64 `json:"updated_at"` + PRs []protocol.PullRequest `json:"prs"` } type boardRenderContext struct { @@ -352,40 +284,6 @@ func openWebRepository(ctx context.Context, cfg config, local bool) (*nativeGitR return seedRepo, remoteRepo, closeStore, cfg, nil } -func (s *brokerGitStore) read(ctx context.Context, objectPath string) ([]byte, error) { - if data, ok, err := s.readWithCapability(ctx, objectPath); ok || err != nil { - return data, err - } - var resp brokerObjectResponse - err := brokerPostContext(ctx, s.brokerURL, "/objects/read", brokerObjectRequest{ - Repo: repoForBroker(s.cfg), - Path: strings.TrimPrefix(objectPath, "/"), - }, &resp) - if err != nil { - if isBrokerNotFoundError(err) { - return nil, fs.ErrNotExist - } - return nil, err - } - data, err := base64.StdEncoding.DecodeString(resp.Data) - if err != nil { - return nil, err - } - return data, nil -} - -func (s *brokerGitStore) list(ctx context.Context, prefix string) ([]string, error) { - var resp brokerObjectResponse - err := brokerPostContext(ctx, s.brokerURL, "/objects/list", brokerObjectRequest{ - Repo: repoForBroker(s.cfg), - Prefix: strings.TrimPrefix(prefix, "/"), - }, &resp) - if err != nil { - return nil, err - } - return resp.Paths, nil -} - func isBrokerNotFoundError(err error) bool { if err == nil { return false @@ -412,7 +310,7 @@ func newWebHandlerWithAPI(repo, apiRepo *nativeGitRepo, cfg config) *webServer { localGitDir = store.root } } - csrfToken, err := randomWebCSRFToken() + csrfToken, err := internalweb.NewCSRFToken() if err != nil { csrfToken = fmt.Sprintf("%d", time.Now().UnixNano()) } @@ -436,6 +334,7 @@ func webRepoTitle(cfg config) string { func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() route := strings.TrimPrefix(r.URL.Path, "/") + match := internalweb.MatchRoute(r.URL.Path) srv := s.serverForRequest(r, strings.HasPrefix(route, "api/")) if s.isMutationRoute(route, r.Method) { if err := s.validateWebMutation(r); err != nil { @@ -443,156 +342,118 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } - switch { - case r.Method != http.MethodGet && r.Method != http.MethodHead && !strings.HasPrefix(route, "api/actions/") && route != "api/user/profile": + if r.Method != http.MethodGet && r.Method != http.MethodHead && !strings.HasPrefix(route, "api/actions/") && route != "api/user/profile" { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) - case route == "assets/bgit-mark.png": + return + } + switch match.Route { + case internalweb.RouteLogo: s.handleWebAsset(w, webLogoPath) - case route == "favicon.ico": + case internalweb.RouteFavicon: s.handleWebAsset(w, webFaviconPath) - case route == "events": + case internalweb.RouteEvents: s.handleEvents(w, r) - case route == "api/state": + case internalweb.RouteAPIState: s.handleAPIState(ctx, w, r) - case route == "api/me": + case internalweb.RouteAPIMe: s.handleAPIMe(ctx, w, r) - case route == "api/actions/commit": + case internalweb.RouteAPICommitAction: s.handleAPIActionCommit(ctx, w, r) - case route == "api/actions/stage": + case internalweb.RouteAPIStageAction: s.handleAPIActionStage(ctx, w, r) - case route == "api/actions/unstage": + case internalweb.RouteAPIUnstageAction: s.handleAPIActionUnstage(ctx, w, r) - case route == "api/actions/discard": + case internalweb.RouteAPIDiscardAction: s.handleAPIActionDiscard(ctx, w, r) - case route == "api/actions/uncommit": + case internalweb.RouteAPIUncommitAction: s.handleAPIActionUncommit(ctx, w, r) - case route == "api/actions/push": + case internalweb.RouteAPIPushAction: s.handleAPIActionPush(ctx, w, r) - case route == "api/actions/pull": + case internalweb.RouteAPIPullAction: s.handleAPIActionPull(ctx, w, r) - case route == "api/actions/pr": + case internalweb.RouteAPIPRAction: s.handleAPIActionPullRequest(ctx, w, r) - case route == "api/actions/issues": + case internalweb.RouteAPIIssueAction: s.handleAPIActionIssue(ctx, w, r) - case route == "api/actions/board": + case internalweb.RouteAPIBoardAction: s.handleAPIActionBoard(ctx, w, r) - case route == "api/actions/ci": + case internalweb.RouteAPICIAction: s.handleAPIActionCI(ctx, w, r) - case route == "api/actions/settings": + case internalweb.RouteAPISettingsAction: s.handleAPIActionSettings(ctx, w, r) - case route == "api/user/profile": + case internalweb.RouteAPIUserProfile: s.handleAPIUserProfile(ctx, w, r) - case route == "api/diff": + case internalweb.RouteAPIDiff: s.handleAPIDiff(ctx, w, r) - case route == "api/refs": + case internalweb.RouteAPIRefs: srv.handleAPIRefs(ctx, w, r) - case route == "api/tree": + case internalweb.RouteAPITree: srv.handleAPITree(ctx, w, r) - case route == "api/commits": + case internalweb.RouteAPICommits: srv.handleAPICommits(ctx, w, r) - case route == "api/prs": + case internalweb.RouteAPIPRs: s.handleAPIPullRequests(ctx, w, r) - case route == "api/issues": + case internalweb.RouteAPIIssues: s.handleAPIIssues(ctx, w, r) - case route == "api/settings": + case internalweb.RouteAPISettings: s.handleAPISettings(ctx, w, r) - case route == "api/settings-fragment": + case internalweb.RouteAPISettingsFragment: s.handleAPISettingsFragment(ctx, w, r) - case route == "api/blob": + case internalweb.RouteAPIBlob: srv.handleAPIBlob(ctx, w, r) - case strings.HasPrefix(route, "api/commit/"): - srv.handleAPICommit(ctx, w, r, strings.TrimPrefix(route, "api/commit/")) - case r.URL.Path == "/": + case internalweb.RouteAPICommit: + srv.handleAPICommit(ctx, w, r, match.Value) + case internalweb.RouteHome: srv.handleTree(ctx, w, r, "") - case route == "commits": + case internalweb.RouteCommits: srv.handleCommits(ctx, w, r) - case route == "prs": + case internalweb.RoutePRs: s.handlePullRequests(ctx, w, r) - case route == "prs/new": + case internalweb.RouteNewPR: s.handleNewPullRequest(ctx, w, r) - case strings.HasPrefix(route, "prs/"): - s.handlePullRequest(ctx, w, r, strings.TrimPrefix(route, "prs/")) - case route == "issues": + case internalweb.RoutePR: + s.handlePullRequest(ctx, w, r, match.Value) + case internalweb.RouteIssues: s.handleIssues(ctx, w, r) - case strings.HasPrefix(route, "issues/"): - s.handleIssue(ctx, w, r, strings.TrimPrefix(route, "issues/")) - case route == "board": + case internalweb.RouteIssue: + s.handleIssue(ctx, w, r, match.Value) + case internalweb.RouteBoard: s.handleBoard(ctx, w, r) - case strings.HasPrefix(route, "board/"): - s.handleStory(ctx, w, r, strings.TrimPrefix(route, "board/")) - case route == "ci": + case internalweb.RouteStory: + s.handleStory(ctx, w, r, match.Value) + case internalweb.RouteCI: s.handleCI(ctx, w, r) - case route == "settings": + case internalweb.RouteSettings: s.handleSettings(ctx, w, r) - case route == "user/settings" || route == "user/settings/": + case internalweb.RouteUserSettings: s.handleUserSettings(ctx, w, r) - case route == "admin": + case internalweb.RouteAdmin: http.Redirect(w, r, "/settings", http.StatusFound) - case route == "archive.zip": + case internalweb.RouteArchive: srv.handleArchiveZip(ctx, w, r) - case strings.HasPrefix(route, "commit/"): - srv.handleCommit(ctx, w, r, strings.TrimPrefix(route, "commit/")) - case strings.HasPrefix(route, "tree/"): - srv.handleTree(ctx, w, r, strings.TrimPrefix(route, "tree/")) - case strings.HasPrefix(route, "blob/"): - srv.handleBlob(ctx, w, r, strings.TrimPrefix(route, "blob/"), false) - case strings.HasPrefix(route, "raw/"): - srv.handleBlob(ctx, w, r, strings.TrimPrefix(route, "raw/"), true) + case internalweb.RouteCommit: + srv.handleCommit(ctx, w, r, match.Value) + case internalweb.RouteTree: + srv.handleTree(ctx, w, r, match.Value) + case internalweb.RouteBlob: + srv.handleBlob(ctx, w, r, match.Value, false) + case internalweb.RouteRaw: + srv.handleBlob(ctx, w, r, match.Value, true) default: http.NotFound(w, r) } } -func randomWebCSRFToken() (string, error) { - var data [32]byte - if _, err := rand.Read(data[:]); err != nil { - return "", err - } - return base64.RawURLEncoding.EncodeToString(data[:]), nil -} - func (s *webServer) isMutationRoute(route, method string) bool { - if method == http.MethodGet || method == http.MethodHead || method == http.MethodOptions { - return false - } - return strings.HasPrefix(route, "api/actions/") || route == "api/user/profile" + return internalweb.IsMutationRoute(route, method) } func (s *webServer) validateWebMutation(r *http.Request) error { - if r.Header.Get("X-Bgit-CSRF") != s.csrfToken { - return errors.New("invalid CSRF token") - } - origin := strings.TrimSpace(r.Header.Get("Origin")) - if origin == "" { - return nil - } - u, err := url.Parse(origin) - if err != nil { - return errors.New("invalid origin") - } - host, _, err := net.SplitHostPort(u.Host) - if err != nil { - host = u.Hostname() - } - host = strings.ToLower(strings.Trim(host, "[]")) - if u.Scheme != "http" || (host != "localhost" && host != "127.0.0.1" && host != "::1") { - return errors.New("foreign origin rejected") - } - return nil + return internalweb.ValidateLocalMutation(s.csrfToken, r) } func (s *webServer) handleWebAsset(w http.ResponseWriter, path string) { - data, err := webAssetBytes(path) - if err != nil { - http.Error(w, "not found", http.StatusNotFound) - return - } - if typ := mime.TypeByExtension(filepath.Ext(path)); typ != "" { - w.Header().Set("Content-Type", typ) - } - w.Header().Set("Cache-Control", "public, max-age=86400") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(data) + internalweb.ServeAsset(w, path) } func (s *webServer) handleEvents(w http.ResponseWriter, r *http.Request) { @@ -658,9 +519,9 @@ func (s *webServer) cachedWhoamiJSON() string { return string(data) } -func (s *webServer) webWhoami(ctx context.Context, refresh bool) (brokerAuthStatus, error) { +func (s *webServer) webWhoami(ctx context.Context, refresh bool) (protocol.AuthStatus, error) { if s.cfg.brokerURL == "" || s.cfg.logicalRepo == "" { - return brokerAuthStatus{}, errors.New("whoami requires a broker-backed repository") + return protocol.AuthStatus{}, errors.New("whoami requires a broker-backed repository") } return brokerWhoami(ctx, s.cfg, refresh) } @@ -809,7 +670,7 @@ func (s *webServer) handleAPICommits(ctx context.Context, w http.ResponseWriter, func (s *webServer) handleAPIPullRequests(ctx context.Context, w http.ResponseWriter, r *http.Request) { refresh := r.URL.Query().Get("refresh") == "1" - prs := []brokerPullRequest{} + var prs []protocol.PullRequest source := "cache" stale := false if !refresh { @@ -843,20 +704,20 @@ func (s *webServer) handleAPIPullRequests(ctx context.Context, w http.ResponseWr } type webSettingsInfo struct { - Repo brokerRepo `json:"repo"` - Title string `json:"title"` - BrokerURL string `json:"broker_url,omitempty"` - Provider string `json:"provider,omitempty"` - Region string `json:"region,omitempty"` - Description string `json:"description,omitempty"` - DefaultBranch string `json:"default_branch,omitempty"` - Visibility string `json:"visibility,omitempty"` - ReadOnly bool `json:"read_only,omitempty"` - IssuesEnabled bool `json:"issues_enabled"` - Keys []brokerKey `json:"keys,omitempty"` - UserGrants []brokerRepoUserGrant `json:"user_grants,omitempty"` - Protections []brokerProtectionRequest `json:"protections,omitempty"` - Errors map[string]string `json:"errors,omitempty"` + Repo protocol.Repository `json:"repo"` + Title string `json:"title"` + BrokerURL string `json:"broker_url,omitempty"` + Provider string `json:"provider,omitempty"` + Region string `json:"region,omitempty"` + Description string `json:"description,omitempty"` + DefaultBranch string `json:"default_branch,omitempty"` + Visibility string `json:"visibility,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` + IssuesEnabled bool `json:"issues_enabled"` + Keys []protocol.Key `json:"keys,omitempty"` + UserGrants []protocol.RepositoryUserGrant `json:"user_grants,omitempty"` + Protections []protocol.Protection `json:"protections,omitempty"` + Errors map[string]string `json:"errors,omitempty"` } type webSettingsCache struct { @@ -864,61 +725,6 @@ type webSettingsCache struct { Info webSettingsInfo `json:"info"` } -type brokerRepoTeamGrant struct { - ID string `json:"id,omitempty"` - TeamID string `json:"team_id,omitempty"` - Role string `json:"role,omitempty"` -} - -type brokerRepoInfoRequest struct { - Repo brokerRepo `json:"repo"` - Description string `json:"description,omitempty"` - DefaultBranch string `json:"default_branch,omitempty"` - Visibility string `json:"visibility,omitempty"` - ReadOnly bool `json:"read_only,omitempty"` - IssuesEnabled bool `json:"issues_enabled"` - Logical string `json:"logical,omitempty"` - TeamID string `json:"team_id,omitempty"` - Name string `json:"name,omitempty"` - UserID string `json:"user_id,omitempty"` - User string `json:"user,omitempty"` - Role string `json:"role,omitempty"` - BrokerRole string `json:"broker_role,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` -} - -type brokerRepoInfoResponse struct { - Repo brokerRepo `json:"repo"` - Description string `json:"description"` - DefaultBranch string `json:"default_branch"` - Visibility string `json:"visibility"` - ReadOnly bool `json:"read_only"` - IssuesEnabled bool `json:"issues_enabled"` -} - -type brokerUserProfileRequest struct { - Repo brokerRepo `json:"repo"` - Bio string `json:"bio,omitempty"` - Avatar string `json:"avatar,omitempty"` -} - -type brokerUserProfileResponse struct { - User string `json:"user"` - Profile brokerUserProfileData `json:"profile"` - Keys []brokerUserProfileKey `json:"keys"` -} - -type brokerUserProfileData struct { - Bio string `json:"bio,omitempty"` - Avatar string `json:"avatar,omitempty"` -} - -type brokerUserProfileKey struct { - PublicKey string `json:"public_key"` - Fingerprint string `json:"fingerprint,omitempty"` - Source string `json:"source,omitempty"` -} - func (s *webServer) handleAPISettings(ctx context.Context, w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) @@ -963,34 +769,30 @@ func (s *webServer) handleAPIIssues(ctx context.Context, w http.ResponseWriter, s.renderJSON(w, map[string]any{"issues": issues}) } -func (s *webServer) listIssues(ctx context.Context) ([]brokerIssue, error) { - return s.listIssuesWithOptions(ctx, brokerIssueRequest{Repo: repoForBroker(s.cfg)}) +func (s *webServer) listIssues(ctx context.Context) ([]protocol.Issue, error) { + return s.listIssuesWithOptions(ctx, protocol.IssueRequest{Repo: repoForBroker(s.cfg)}) } -func (s *webServer) listIssuesWithOptions(ctx context.Context, req brokerIssueRequest) ([]brokerIssue, error) { +func (s *webServer) listIssuesWithOptions(ctx context.Context, req protocol.IssueRequest) ([]protocol.Issue, error) { if strings.TrimSpace(s.cfg.brokerURL) == "" || strings.TrimSpace(s.cfg.logicalRepo) == "" { return nil, errors.New("broker issues unavailable") } if strings.TrimSpace(req.Repo.Logical) == "" { req.Repo = repoForBroker(s.cfg) } - var resp struct { - Issues []brokerIssue `json:"issues"` - } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/issues/list", req, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { return nil, err } - return resp.Issues, nil + return endpoints.ListIssues(ctx, req) } -func (s *webServer) getIssue(ctx context.Context, id int) (brokerIssue, error) { - var resp struct { - Issue brokerIssue `json:"issue"` - } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/issues/view", brokerIssueRequest{Repo: repoForBroker(s.cfg), ID: id}, &resp); err != nil { - return brokerIssue{}, err +func (s *webServer) getIssue(ctx context.Context, id int) (protocol.Issue, error) { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + return protocol.Issue{}, err } - return resp.Issue, nil + return endpoints.GetIssue(ctx, protocol.IssueRequest{Repo: repoForBroker(s.cfg), ID: id}) } func (s *webServer) settingsInfo(ctx context.Context) webSettingsInfo { @@ -1012,8 +814,13 @@ func (s *webServer) settingsInfo(ctx context.Context) webSettingsInfo { info.Errors = nil return info } - var repoInfo brokerRepoInfoResponse - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/repo/info", brokerRepoInfoRequest{Repo: repoForBroker(s.cfg)}, &repoInfo); err != nil { + endpoints, endpointErr := brokerEndpointClient(s.cfg.brokerURL) + if endpointErr != nil { + info.Errors["broker"] = endpointErr.Error() + return info + } + repoInfo, err := endpoints.RepositoryInfo(ctx, protocol.RepositoryAdminRequest{Repo: repoForBroker(s.cfg)}) + if err != nil { info.Errors["repo"] = err.Error() } else { if repoInfo.Repo.Logical != "" || repoInfo.Repo.Bucket != "" { @@ -1030,19 +837,17 @@ func (s *webServer) settingsInfo(ctx context.Context) webSettingsInfo { } else { info.Keys = keys } - var repoUsers brokerRepoUsersResponse - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/repo/users/list", brokerRepoAdminRequest{Repo: repoForBroker(s.cfg)}, &repoUsers); err != nil { + repoUsers, err := endpoints.ListRepositoryUsers(ctx, protocol.RepositoryAdminRequest{Repo: repoForBroker(s.cfg)}) + if err != nil { info.Errors["repo users"] = err.Error() } else { - info.UserGrants = repoUsers.Users - } - var protections struct { - Protections []brokerProtectionRequest `json:"protections"` + info.UserGrants = repoUsers } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/protection/list", brokerProtectionRequest{Repo: repoForBroker(s.cfg)}, &protections); err != nil { + protections, err := endpoints.ListProtections(ctx, protocol.Protection{Repo: repoForBroker(s.cfg)}) + if err != nil { info.Errors["protections"] = err.Error() } else { - info.Protections = protections.Protections + info.Protections = protections } if len(info.Errors) == 0 { info.Errors = nil @@ -1557,17 +1362,17 @@ func (s *webServer) handleAPIActionPullRequest(ctx context.Context, w http.Respo return } var req struct { - ID int `json:"id"` - Action string `json:"action"` - Title string `json:"title"` - Body string `json:"body"` - Source string `json:"source"` - Target string `json:"target"` - Comment string `json:"comment"` - DeleteBranch bool `json:"delete_branch"` - Comments []brokerPullRequestComment `json:"comments"` - TargetNoteID int `json:"target_note_id"` - TargetCommentID int `json:"target_comment_id"` + ID int `json:"id"` + Action string `json:"action"` + Title string `json:"title"` + Body string `json:"body"` + Source string `json:"source"` + Target string `json:"target"` + Comment string `json:"comment"` + DeleteBranch bool `json:"delete_branch"` + Comments []protocol.PullRequestComment `json:"comments"` + TargetNoteID int `json:"target_note_id"` + TargetCommentID int `json:"target_comment_id"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { s.renderJSONError(w, http.StatusBadRequest, err) @@ -1578,9 +1383,9 @@ func (s *webServer) handleAPIActionPullRequest(ctx context.Context, w http.Respo return } var resp struct { - PR brokerPullRequest `json:"pr"` + PR protocol.PullRequest `json:"pr"` } - brokerReq := brokerPullRequestRequest{ + brokerReq := protocol.PullRequestRequest{ Repo: repoForBroker(s.cfg), ID: req.ID, Comment: strings.TrimSpace(req.Comment), @@ -1607,7 +1412,7 @@ func (s *webServer) handleAPIActionPullRequest(ctx context.Context, w http.Respo title = shortRefName(source) + " into " + shortRefName(target) } endpoint = "/prs/create" - brokerReq.PR = brokerPullRequest{Title: title, Body: strings.TrimSpace(req.Body), Source: source, Target: target} + brokerReq.PR = protocol.PullRequest{Title: title, Body: strings.TrimSpace(req.Body), Source: source, Target: target} case "comment": if brokerReq.Comment == "" { s.renderJSONError(w, http.StatusBadRequest, errors.New("comment is required")) @@ -1640,10 +1445,32 @@ func (s *webServer) handleAPIActionPullRequest(ctx context.Context, w http.Respo s.renderJSONError(w, http.StatusBadRequest, fmt.Errorf("unsupported pull request action %q", req.Action)) return } - if err := brokerPostContext(ctx, s.cfg.brokerURL, endpoint, brokerReq, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } + var actionErr error + switch endpoint { + case "/prs/create": + resp.PR, actionErr = endpoints.CreatePullRequest(ctx, brokerReq) + case "/prs/comment": + resp.PR, actionErr = endpoints.CommentPullRequest(ctx, brokerReq) + case "/prs/review": + resp.PR, actionErr = endpoints.ReviewPullRequest(ctx, brokerReq) + case "/prs/reply": + resp.PR, actionErr = endpoints.ReplyPullRequest(ctx, brokerReq) + case "/prs/merge": + resp.PR, actionErr = endpoints.MergePullRequest(ctx, brokerReq) + case "/prs/close": + resp.PR, actionErr = endpoints.ClosePullRequest(ctx, brokerReq) + case "/prs/reopen": + resp.PR, actionErr = endpoints.ReopenPullRequest(ctx, brokerReq) + } + if actionErr != nil { + s.renderJSONError(w, http.StatusBadRequest, actionErr) + return + } prs := s.upsertPullRequestCache(resp.PR) s.renderJSON(w, map[string]any{"ok": true, "pr": resp.PR, "prs": webAPIPullRequests(prs)}) } @@ -1681,14 +1508,15 @@ func (s *webServer) handleAPIActionSettings(ctx context.Context, w http.Response var payload any switch strings.TrimSpace(req.Action) { case "update-repo": + readOnly, issuesEnabled := req.ReadOnly, req.IssuesEnabled endpoint = "/repo/update" - payload = brokerRepoInfoRequest{ + payload = protocol.RepositoryAdminRequest{ Repo: repoForBroker(s.cfg), Description: req.Description, DefaultBranch: req.DefaultBranch, Visibility: req.Visibility, - ReadOnly: req.ReadOnly, - IssuesEnabled: req.IssuesEnabled, + ReadOnly: &readOnly, + IssuesEnabled: &issuesEnabled, } case "add-member": user := strings.TrimSpace(req.User) @@ -1702,10 +1530,10 @@ func (s *webServer) handleAPIActionSettings(ctx context.Context, w http.Response return } endpoint = "/keys/invite/create" - payload = brokerOwnerTransferRequest{Repo: repoForBroker(s.cfg), User: user, Role: role, BrokerURL: s.cfg.brokerURL} + payload = protocol.OwnerTransferRequest{Repo: repoForBroker(s.cfg), User: user, Role: role, BrokerURL: s.cfg.brokerURL} case "remove-member": endpoint = "/keys/remove" - payload = brokerKeyRequest{Repo: repoForBroker(s.cfg), Key: strings.TrimSpace(req.Key)} + payload = protocol.KeyRequest{Repo: repoForBroker(s.cfg), Key: strings.TrimSpace(req.Key)} case "remove-repo-user": user := strings.TrimSpace(req.User) userID := strings.TrimSpace(req.UserID) @@ -1714,16 +1542,16 @@ func (s *webServer) handleAPIActionSettings(ctx context.Context, w http.Response return } endpoint = "/repo/users/remove" - payload = brokerRepoAdminRequest{Repo: repoForBroker(s.cfg), UserID: userID, User: user} + payload = protocol.RepositoryAdminRequest{Repo: repoForBroker(s.cfg), UserID: userID, User: user} case "suspend-member": endpoint = "/keys/suspend" - payload = brokerKeyRequest{Repo: repoForBroker(s.cfg), Key: strings.TrimSpace(req.Key)} + payload = protocol.KeyRequest{Repo: repoForBroker(s.cfg), Key: strings.TrimSpace(req.Key)} case "unsuspend-member": endpoint = "/keys/unsuspend" - payload = brokerKeyRequest{Repo: repoForBroker(s.cfg), Key: strings.TrimSpace(req.Key)} + payload = protocol.KeyRequest{Repo: repoForBroker(s.cfg), Key: strings.TrimSpace(req.Key)} case "transfer-owner": endpoint = "/owners/transfer/confirm" - payload = brokerOwnerTransferRequest{Repo: repoForBroker(s.cfg), BrokerURL: s.cfg.brokerURL} + payload = protocol.OwnerTransferRequest{Repo: repoForBroker(s.cfg), BrokerURL: s.cfg.brokerURL} case "repo-rename": logical, err := normalizeLogicalRepoName(req.Logical) if err != nil { @@ -1731,47 +1559,78 @@ func (s *webServer) handleAPIActionSettings(ctx context.Context, w http.Response return } endpoint = "/repo/rename" - payload = brokerRepoInfoRequest{Repo: repoForBroker(s.cfg), Logical: logical} + payload = protocol.RepositoryAdminRequest{Repo: repoForBroker(s.cfg), Logical: logical} case "repo-delete": endpoint = "/repo/delete" - payload = brokerRepoInfoRequest{Repo: repoForBroker(s.cfg)} + payload = protocol.RepositoryAdminRequest{Repo: repoForBroker(s.cfg)} case "protect-upsert": ref := normalizeDestinationRef(firstNonEmpty(strings.TrimSpace(req.Ref), defaultBranch)) endpoint = "/protection/upsert" - payload = brokerProtectionRequest{Repo: repoForBroker(s.cfg), Ref: ref, RequirePR: req.RequirePR, AllowOverrides: req.AllowOverrides} + payload = protocol.Protection{Repo: repoForBroker(s.cfg), Ref: ref, RequirePR: req.RequirePR, AllowOverrides: req.AllowOverrides} case "protect-remove": endpoint = "/protection/remove" - payload = brokerProtectionRequest{Repo: repoForBroker(s.cfg), Ref: normalizeDestinationRef(req.Ref)} + payload = protocol.Protection{Repo: repoForBroker(s.cfg), Ref: normalizeDestinationRef(req.Ref)} default: s.renderJSONError(w, http.StatusBadRequest, fmt.Errorf("unsupported settings action %q", req.Action)) return } if endpoint != "/repo/update" { switch p := payload.(type) { - case brokerKeyRequest: + case protocol.KeyRequest: if strings.TrimSpace(p.Key) == "" && len(p.PublicKeys) == 0 { s.renderJSONError(w, http.StatusBadRequest, errors.New("member key is required")) return } - case brokerOwnerTransferRequest: + case protocol.OwnerTransferRequest: if endpoint == "/keys/invite/create" && strings.TrimSpace(p.User) == "" { s.renderJSONError(w, http.StatusBadRequest, errors.New("member invite requires user")) return } - case brokerProtectionRequest: + case protocol.Protection: if strings.TrimSpace(p.Ref) == "" { s.renderJSONError(w, http.StatusBadRequest, errors.New("branch protection ref is required")) return } - case brokerRepoInfoRequest: + case protocol.RepositoryAdminRequest: if endpoint == "/repo/rename" && strings.TrimSpace(p.Logical) == "" { s.renderJSONError(w, http.StatusBadRequest, errors.New("logical repo name is required")) return } } } - var brokerResp map[string]any - if err := brokerPostContext(ctx, s.cfg.brokerURL, endpoint, payload, &brokerResp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + s.renderJSONError(w, http.StatusBadRequest, err) + return + } + var brokerResp any + switch endpoint { + case "/repo/update": + err = endpoints.UpdateRepository(ctx, payload.(protocol.RepositoryAdminRequest)) + case "/keys/invite/create": + brokerResp, err = endpoints.CreateRepositoryInvite(ctx, payload.(protocol.OwnerTransferRequest)) + case "/keys/remove": + err = endpoints.RemoveKey(ctx, payload.(protocol.KeyRequest)) + case "/repo/users/remove": + err = endpoints.RemoveRepositoryUser(ctx, payload.(protocol.RepositoryAdminRequest)) + case "/keys/suspend": + err = endpoints.SuspendKey(ctx, payload.(protocol.KeyRequest), true) + case "/keys/unsuspend": + err = endpoints.SuspendKey(ctx, payload.(protocol.KeyRequest), false) + case "/owners/transfer/confirm": + brokerResp, err = endpoints.ConfirmOwnerTransfer(ctx, payload.(protocol.OwnerTransferRequest)) + case "/repo/rename": + err = endpoints.RenameRepository(ctx, payload.(protocol.RepositoryAdminRequest)) + case "/repo/delete": + err = endpoints.DeleteRepository(ctx, payload.(protocol.RepositoryAdminRequest)) + case "/protection/upsert": + err = endpoints.UpsertProtection(ctx, payload.(protocol.Protection)) + case "/protection/remove": + err = endpoints.RemoveProtection(ctx, payload.(protocol.Protection)) + default: + err = fmt.Errorf("unsupported settings endpoint %q", endpoint) + } + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } @@ -1809,7 +1668,7 @@ func (s *webServer) handleAPIActionIssue(ctx context.Context, w http.ResponseWri return } endpoint := "" - payload := brokerIssueRequest{Repo: repoForBroker(s.cfg), ID: req.ID, Title: strings.TrimSpace(req.Title), Body: strings.TrimSpace(req.Body), Comment: strings.TrimSpace(req.Comment)} + payload := protocol.IssueRequest{Repo: repoForBroker(s.cfg), ID: req.ID, Title: strings.TrimSpace(req.Title), Body: strings.TrimSpace(req.Body), Comment: strings.TrimSpace(req.Comment)} switch strings.TrimSpace(req.Action) { case "create": endpoint = "/issues/create" @@ -1831,11 +1690,26 @@ func (s *webServer) handleAPIActionIssue(ctx context.Context, w http.ResponseWri s.renderJSONError(w, http.StatusBadRequest, fmt.Errorf("unsupported issue action %q", req.Action)) return } - var resp map[string]any - if err := brokerPostContext(ctx, s.cfg.brokerURL, endpoint, payload, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } + var actionErr error + switch endpoint { + case "/issues/create": + _, actionErr = endpoints.CreateIssue(ctx, payload) + case "/issues/comment": + actionErr = endpoints.CommentIssue(ctx, payload) + case "/issues/close": + actionErr = endpoints.CloseIssue(ctx, payload) + case "/issues/reopen": + actionErr = endpoints.ReopenIssue(ctx, payload) + } + if actionErr != nil { + s.renderJSONError(w, http.StatusBadRequest, actionErr) + return + } s.renderJSON(w, map[string]any{"ok": true}) } @@ -1862,7 +1736,7 @@ func (s *webServer) handleAPIActionBoard(ctx context.Context, w http.ResponseWri s.renderJSONError(w, http.StatusBadRequest, err) return } - payload := brokerIssueRequest{Repo: repoForBroker(s.cfg), ID: req.ID, Type: "story", Title: strings.TrimSpace(req.Title), Body: strings.TrimSpace(req.Body), Lane: strings.TrimSpace(req.Lane), Assignee: strings.TrimSpace(req.Assignee), Comment: strings.TrimSpace(req.Comment), AfterID: req.AfterID} + payload := protocol.IssueRequest{Repo: repoForBroker(s.cfg), ID: req.ID, Type: "story", Title: strings.TrimSpace(req.Title), Body: strings.TrimSpace(req.Body), Lane: strings.TrimSpace(req.Lane), Assignee: strings.TrimSpace(req.Assignee), Comment: strings.TrimSpace(req.Comment), AfterID: req.AfterID} endpoint := "" switch strings.TrimSpace(req.Action) { case "create": @@ -1921,9 +1795,32 @@ func (s *webServer) handleAPIActionBoard(ctx context.Context, w http.ResponseWri s.renderJSONError(w, http.StatusBadRequest, fmt.Errorf("unsupported board action %q", req.Action)) return } - var resp map[string]any - if err := brokerPostContext(ctx, s.cfg.brokerURL, endpoint, payload, &resp); err != nil { - s.renderJSONError(w, http.StatusBadRequest, boardUpgradeError(err)) + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + s.renderJSONError(w, http.StatusBadRequest, err) + return + } + var actionErr error + switch endpoint { + case "/issues/create": + _, actionErr = endpoints.CreateIssue(ctx, payload) + case "/issues/move": + actionErr = endpoints.MoveIssue(ctx, payload) + case "/issues/reorder": + actionErr = endpoints.ReorderIssue(ctx, payload) + case "/issues/update": + actionErr = endpoints.UpdateIssue(ctx, payload) + case "/issues/archive": + actionErr = endpoints.ArchiveIssue(ctx, payload) + case "/issues/take": + actionErr = endpoints.TakeIssue(ctx, payload) + case "/issues/assign": + actionErr = endpoints.AssignIssue(ctx, payload) + case "/issues/comment": + actionErr = endpoints.CommentIssue(ctx, payload) + } + if actionErr != nil { + s.renderJSONError(w, http.StatusBadRequest, boardUpgradeError(actionErr)) return } s.renderJSON(w, map[string]any{"ok": true}) @@ -1959,14 +1856,21 @@ func (s *webServer) handleAPIActionCI(ctx context.Context, w http.ResponseWriter s.renderJSON(w, map[string]any{"ok": true, "runs": runs}) return case "logs": + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + s.renderJSONError(w, http.StatusBadRequest, err) + return + } var resp struct { - Run brokerCIRun `json:"run"` - Logs string `json:"logs"` + Run protocol.CIRun `json:"run"` + Logs string `json:"logs"` } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/ci/logs", brokerCIRequest{Repo: repoForBroker(s.cfg), ID: req.ID}, &resp); err != nil { + logs, err := endpoints.CILogs(ctx, protocol.CIRequest{Repo: repoForBroker(s.cfg), ID: req.ID}) + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } + resp.Run, resp.Logs = logs.Run, logs.Logs s.renderJSON(w, map[string]any{"ok": true, "run": resp.Run, "logs": resp.Logs}) return case "run": @@ -1987,15 +1891,18 @@ func (s *webServer) handleAPIActionCI(ctx context.Context, w http.ResponseWriter config = detected } } - var resp struct { - Run brokerCIRun `json:"run"` + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + s.renderJSONError(w, http.StatusBadRequest, err) + return } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/ci/run", brokerCIRequest{Repo: repoForBroker(s.cfg), Provider: normalizeCIProvider(req.Provider), Ref: ref, Commit: commit, Config: config}, &resp); err != nil { + run, err := endpoints.RunCI(ctx, protocol.CIRequest{Repo: repoForBroker(s.cfg), Provider: normalizeCIProvider(req.Provider), Ref: ref, Commit: commit, Config: config}) + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } runs, _ := s.listCIRuns(ctx) - s.renderJSON(w, map[string]any{"ok": true, "run": resp.Run, "runs": runs}) + s.renderJSON(w, map[string]any{"ok": true, "run": run, "runs": runs}) } func (s *webServer) webRepositoryState(ctx context.Context, refreshRemote bool, selectedRef string) (webAPIState, error) { @@ -2645,7 +2552,7 @@ func (s *webServer) handleCommits(ctx context.Context, w http.ResponseWriter, r } func (s *webServer) handlePullRequests(ctx context.Context, w http.ResponseWriter, r *http.Request) { - prs := []brokerPullRequest{} + prs := []protocol.PullRequest{} source := "cache" stale := false if cached, err := s.readPullRequestCache(); err == nil { @@ -2757,7 +2664,7 @@ func (s *webServer) handleIssues(ctx context.Context, w http.ResponseWriter, r * func (s *webServer) handleBoard(ctx context.Context, w http.ResponseWriter, r *http.Request) { archived := r.URL.Query().Get("view") == "archived" || r.URL.Query().Get("archived") == "1" - issues, err := s.listIssuesWithOptions(ctx, brokerIssueRequest{Repo: repoForBroker(s.cfg), Type: "story", IncludeArchived: archived}) + issues, err := s.listIssuesWithOptions(ctx, protocol.IssueRequest{Repo: repoForBroker(s.cfg), Type: "story", IncludeArchived: archived}) if err != nil { issues = nil } @@ -2803,17 +2710,15 @@ func (s *webServer) handleCI(ctx context.Context, w http.ResponseWriter, r *http s.renderPage(w, webPageTitle(s.title, "ci"), body.String()) } -func (s *webServer) listCIRuns(ctx context.Context) ([]brokerCIRun, error) { +func (s *webServer) listCIRuns(ctx context.Context) ([]protocol.CIRun, error) { if strings.TrimSpace(s.cfg.brokerURL) == "" || strings.TrimSpace(s.cfg.logicalRepo) == "" { return nil, errors.New("CI requires a broker-backed repository") } - var resp struct { - Runs []brokerCIRun `json:"runs"` - } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/ci/list", brokerCIRequest{Repo: repoForBroker(s.cfg)}, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { return nil, err } - return resp.Runs, nil + return endpoints.ListCIRuns(ctx, protocol.CIRequest{Repo: repoForBroker(s.cfg)}) } func (s *webServer) handleIssue(ctx context.Context, w http.ResponseWriter, r *http.Request, idPart string) { @@ -2924,7 +2829,7 @@ func (s *webServer) handleStory(ctx context.Context, w http.ResponseWriter, r *h s.renderPage(w, webPageTitle(s.title, "story"), body.String()) } -func storyEventText(event brokerIssueEvent) string { +func storyEventText(event protocol.IssueEvent) string { switch event.Action { case "created": return "created the story" @@ -2964,13 +2869,11 @@ func (s *webServer) listBoardAssignees(ctx context.Context) ([]string, error) { if strings.TrimSpace(s.cfg.brokerURL) == "" || strings.TrimSpace(s.cfg.logicalRepo) == "" { return nil, errors.New("board requires a broker-backed repository") } - var resp struct { - Users []string `json:"users"` - } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/issues/assignees", brokerIssueRequest{Repo: repoForBroker(s.cfg)}, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { return nil, err } - return resp.Users, nil + return endpoints.IssueAssignees(ctx, protocol.IssueRequest{Repo: repoForBroker(s.cfg)}) } func (s *webServer) handleSettings(ctx context.Context, w http.ResponseWriter, r *http.Request) { @@ -3021,8 +2924,13 @@ func (s *webServer) handleAPIUserProfile(ctx context.Context, w http.ResponseWri } switch r.Method { case http.MethodGet: - var resp brokerUserProfileResponse - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/profile/get", brokerUserProfileRequest{Repo: repoForBroker(s.cfg)}, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + s.renderJSONError(w, http.StatusBadRequest, err) + return + } + resp, err := endpoints.GetUserProfile(ctx, protocol.UserProfileRequest{Repo: repoForBroker(s.cfg)}) + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } @@ -3036,8 +2944,13 @@ func (s *webServer) handleAPIUserProfile(ctx context.Context, w http.ResponseWri s.renderJSONError(w, http.StatusBadRequest, err) return } - var resp brokerUserProfileResponse - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/profile/update", brokerUserProfileRequest{Repo: repoForBroker(s.cfg), Bio: strings.TrimSpace(req.Bio), Avatar: strings.TrimSpace(req.Avatar)}, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + s.renderJSONError(w, http.StatusBadRequest, err) + return + } + resp, err := endpoints.UpdateUserProfile(ctx, protocol.UserProfileRequest{Repo: repoForBroker(s.cfg), Bio: strings.TrimSpace(req.Bio), Avatar: strings.TrimSpace(req.Avatar)}) + if err != nil { s.renderJSONError(w, http.StatusBadRequest, err) return } @@ -3257,7 +3170,7 @@ func (s *webServer) handlePullRequest(ctx context.Context, w http.ResponseWriter s.renderPage(w, webPageTitle(s.title, fmt.Sprintf("PR #%d", pr.ID)), body.String()) } -func (s *webServer) pullRequestByID(ctx context.Context, id int) (brokerPullRequest, error) { +func (s *webServer) pullRequestByID(ctx context.Context, id int) (protocol.PullRequest, error) { if cached, err := s.readPullRequestCache(); err == nil { for _, pr := range cached.PRs { if pr.ID == id { @@ -3272,10 +3185,10 @@ func (s *webServer) pullRequestByID(ctx context.Context, id int) (brokerPullRequ } } } - return brokerPullRequest{}, errors.New("pull request not found") + return protocol.PullRequest{}, errors.New("pull request not found") } -func (s *webServer) pullRequestFilesHTML(ctx context.Context, pr brokerPullRequest) string { +func (s *webServer) pullRequestFilesHTML(ctx context.Context, pr protocol.PullRequest) string { files, additions, deletions, err := s.pullRequestChangedFiles(ctx, pr) if err != nil { return `
` + html.EscapeString(err.Error()) + `
` @@ -3283,7 +3196,7 @@ func (s *webServer) pullRequestFilesHTML(ctx context.Context, pr brokerPullReque return diffFilesPanelHTML(files, additions, deletions) } -func (s *webServer) pullRequestReviewHTML(ctx context.Context, pr brokerPullRequest) string { +func (s *webServer) pullRequestReviewHTML(ctx context.Context, pr protocol.PullRequest) string { files, additions, deletions, err := s.pullRequestChangedFiles(ctx, pr) if err != nil { return `
` + html.EscapeString(err.Error()) + `
` @@ -3298,15 +3211,15 @@ func (s *webServer) pullRequestReviewHTML(ctx context.Context, pr brokerPullRequ return b.String() } -func prReviewComments(pr brokerPullRequest) []brokerPullRequestComment { - var comments []brokerPullRequestComment +func prReviewComments(pr protocol.PullRequest) []protocol.PullRequestComment { + var comments []protocol.PullRequestComment for _, review := range pr.Reviews { comments = append(comments, review.Comments...) } return comments } -func (s *webServer) pullRequestChangedFiles(ctx context.Context, pr brokerPullRequest) ([]webChangedFile, int, int, error) { +func (s *webServer) pullRequestChangedFiles(ctx context.Context, pr protocol.PullRequest) ([]webChangedFile, int, int, error) { targetRef := firstNonEmpty(pr.Target, branchRef(defaultBranch)) sourceRef := firstNonEmpty(pr.Source, pr.Head) return s.pullRequestChangedFilesForRefs(ctx, targetRef, sourceRef, pr.Head) @@ -3320,7 +3233,7 @@ func (s *webServer) pullRequestChangedFilesForRefs(ctx context.Context, targetRe remote := *s remote.repo = s.apiRepo repo = &remote - targetHash, targetErr = repo.resolvePullRequestRevision(ctx, targetRef, "") + targetHash, _ = repo.resolvePullRequestRevision(ctx, targetRef, "") sourceHash, sourceErr = repo.resolvePullRequestRevision(ctx, sourceRef, sourceFallback) } if targetErr != nil || sourceErr != nil { @@ -3354,7 +3267,7 @@ func (s *webServer) pullRequestMergeability(ctx context.Context, targetRef, sour remote := *s remote.repo = s.apiRepo repo = &remote - targetHash, targetErr = repo.resolvePullRequestRevision(ctx, targetRef, "") + targetHash, _ = repo.resolvePullRequestRevision(ctx, targetRef, "") sourceHash, sourceErr = repo.resolvePullRequestRevision(ctx, sourceRef, "") } if targetErr != nil || sourceErr != nil { @@ -3445,7 +3358,7 @@ func changedFilesUnifiedDiff(files []webChangedFile) string { return b.String() } -func (s *webServer) pullRequestCommitsHTML(ctx context.Context, pr brokerPullRequest) string { +func (s *webServer) pullRequestCommitsHTML(ctx context.Context, pr protocol.PullRequest) string { repo := s targetRef := firstNonEmpty(pr.Target, branchRef(defaultBranch)) sourceRef := firstNonEmpty(pr.Source, pr.Head) @@ -3455,7 +3368,7 @@ func (s *webServer) pullRequestCommitsHTML(ctx context.Context, pr brokerPullReq remote := *s remote.repo = s.apiRepo repo = &remote - targetHash, targetErr = repo.resolvePullRequestRevision(ctx, targetRef, "") + targetHash, _ = repo.resolvePullRequestRevision(ctx, targetRef, "") sourceHash, sourceErr = repo.resolvePullRequestRevision(ctx, sourceRef, pr.Head) } if sourceErr != nil { @@ -3860,10 +3773,14 @@ func (s *webServer) repoPolicyBannerHTML() string { if strings.TrimSpace(s.cfg.brokerURL) == "" || strings.TrimSpace(s.cfg.logicalRepo) == "" { return "" } - var repoInfo brokerRepoInfoResponse ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/repo/info", brokerRepoInfoRequest{Repo: repoForBroker(s.cfg)}, &repoInfo); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + return "" + } + repoInfo, err := endpoints.RepositoryInfo(ctx, protocol.RepositoryAdminRequest{Repo: repoForBroker(s.cfg)}) + if err != nil { return "" } if repoInfo.ReadOnly { @@ -3928,10 +3845,6 @@ func gitHubIconHTML() string { return `` } -func diffIconSVGHTML() string { - return `` -} - func contributorsFromCommits(commits []commitObject) []webContributor { indexes := map[string]int{} contributors := []webContributor{} @@ -4331,12 +4244,12 @@ func (s *webServer) readPullRequestCache() (webPullRequestCache, error) { return webPullRequestCache{}, err } if cache.PRs == nil { - cache.PRs = []brokerPullRequest{} + cache.PRs = []protocol.PullRequest{} } return cache, nil } -func (s *webServer) writePullRequestCache(prs []brokerPullRequest) error { +func (s *webServer) writePullRequestCache(prs []protocol.PullRequest) error { path := s.pullRequestCachePath() if path == "" { return nil @@ -4355,7 +4268,7 @@ func (s *webServer) writePullRequestCache(prs []brokerPullRequest) error { return os.WriteFile(path, data, 0o644) } -func (s *webServer) upsertPullRequestCache(pr brokerPullRequest) []brokerPullRequest { +func (s *webServer) upsertPullRequestCache(pr protocol.PullRequest) []protocol.PullRequest { if pr.ID <= 0 { if cached, err := s.readPullRequestCache(); err == nil { return cached.PRs @@ -4364,7 +4277,7 @@ func (s *webServer) upsertPullRequestCache(pr brokerPullRequest) []brokerPullReq } cache, err := s.readPullRequestCache() if err != nil { - cache.PRs = []brokerPullRequest{} + cache.PRs = []protocol.PullRequest{} } found := false for i := range cache.PRs { @@ -4384,7 +4297,7 @@ func (s *webServer) upsertPullRequestCache(pr brokerPullRequest) []brokerPullReq return cache.PRs } -func (s *webServer) refreshPullRequestCache(ctx context.Context) ([]brokerPullRequest, error) { +func (s *webServer) refreshPullRequestCache(ctx context.Context) ([]protocol.PullRequest, error) { if strings.TrimSpace(s.cfg.brokerURL) == "" || strings.TrimSpace(s.cfg.logicalRepo) == "" { return nil, errors.New("broker pull requests unavailable") } @@ -4398,27 +4311,36 @@ func (s *webServer) refreshPullRequestCache(ctx context.Context) ([]brokerPullRe } } var resp struct { - PRs []brokerPullRequest `json:"prs"` - Deleted []int `json:"deleted"` + PRs []protocol.PullRequest `json:"prs"` + Deleted []int `json:"deleted"` } - if err := brokerPostContext(ctx, s.cfg.brokerURL, "/prs/sync", brokerPullRequestRequest{Repo: repoForBroker(s.cfg), Known: known}, &resp); err != nil { + endpoints, err := brokerEndpointClient(s.cfg.brokerURL) + if err != nil { + return nil, err + } + synced, err := endpoints.SyncPullRequests(ctx, protocol.PullRequestRequest{Repo: repoForBroker(s.cfg), Known: known}) + if err != nil { if !strings.Contains(err.Error(), "unknown broker endpoint") { return nil, err } - if listErr := brokerPostContext(ctx, s.cfg.brokerURL, "/prs/list", brokerPullRequestRequest{Repo: repoForBroker(s.cfg)}, &resp); listErr != nil { - return nil, err + prs, listErr := endpoints.ListPullRequests(ctx, protocol.PullRequestRequest{Repo: repoForBroker(s.cfg)}) + if listErr != nil { + return nil, listErr } + resp.PRs = prs + } else { + resp.PRs, resp.Deleted = synced.PRs, synced.Deleted } if resp.PRs == nil { - resp.PRs = []brokerPullRequest{} + resp.PRs = []protocol.PullRequest{} } prs := mergePullRequestCache(cached.PRs, resp.PRs, resp.Deleted) _ = s.writePullRequestCache(prs) return prs, nil } -func mergePullRequestCache(cached, changed []brokerPullRequest, deleted []int) []brokerPullRequest { - byID := map[int]brokerPullRequest{} +func mergePullRequestCache(cached, changed []protocol.PullRequest, deleted []int) []protocol.PullRequest { + byID := map[int]protocol.PullRequest{} for _, pr := range cached { if pr.ID > 0 { byID[pr.ID] = pr @@ -4432,7 +4354,7 @@ func mergePullRequestCache(cached, changed []brokerPullRequest, deleted []int) [ byID[pr.ID] = pr } } - prs := make([]brokerPullRequest, 0, len(byID)) + prs := make([]protocol.PullRequest, 0, len(byID)) for _, pr := range byID { prs = append(prs, pr) } @@ -4442,7 +4364,7 @@ func mergePullRequestCache(cached, changed []brokerPullRequest, deleted []int) [ return prs } -func webAPIPullRequests(prs []brokerPullRequest) []map[string]any { +func webAPIPullRequests(prs []protocol.PullRequest) []map[string]any { out := make([]map[string]any, 0, len(prs)) for _, pr := range prs { out = append(out, map[string]any{ @@ -4463,7 +4385,7 @@ func webAPIPullRequests(prs []brokerPullRequest) []map[string]any { return out } -func openPullRequestCount(prs []brokerPullRequest) int { +func openPullRequestCount(prs []protocol.PullRequest) int { count := 0 for _, pr := range prs { if pullRequestIsOpen(pr) { @@ -4473,11 +4395,11 @@ func openPullRequestCount(prs []brokerPullRequest) int { return count } -func pullRequestIsOpen(pr brokerPullRequest) bool { +func pullRequestIsOpen(pr protocol.PullRequest) bool { return strings.EqualFold(firstNonEmpty(pr.Status, "open"), "open") } -func pullRequestListHTML(prs []brokerPullRequest) string { +func pullRequestListHTML(prs []protocol.PullRequest) string { if len(prs) == 0 { return `
No pull requests found.
` } @@ -4501,7 +4423,7 @@ func pullRequestListHTML(prs []brokerPullRequest) string { return b.String() } -func issueListHTML(issues []brokerIssue) string { +func issueListHTML(issues []protocol.Issue) string { var b strings.Builder b.WriteString(`
`) count := 0 @@ -4545,7 +4467,7 @@ func ciRunFormHTML(cfg config) string { return b.String() } -func ciRunListHTML(runs []brokerCIRun) string { +func ciRunListHTML(runs []protocol.CIRun) string { if len(runs) == 0 { return `
No CI runs yet.
` } @@ -4568,8 +4490,8 @@ func ciRunListHTML(runs []brokerCIRun) string { return b.String() } -func boardHTML(issues []brokerIssue, ctx boardRenderContext, archived bool) string { - var stories []brokerIssue +func boardHTML(issues []protocol.Issue, ctx boardRenderContext, archived bool) string { + var stories []protocol.Issue for _, issue := range issues { if issue.Type == "story" && issue.Archived == archived { stories = append(stories, issue) @@ -4643,7 +4565,7 @@ func archiveStoryIconHTML() string { return `` } -func storyAssignmentControlsHTML(story brokerIssue, ctx boardRenderContext) string { +func storyAssignmentControlsHTML(story protocol.Issue, ctx boardRenderContext) string { assignee := strings.TrimSpace(story.Assignee) if assignee != "" { var b strings.Builder @@ -4985,7 +4907,7 @@ func kanbanLaneLabel(lane string) string { } } -func storyText(story brokerIssue) string { +func storyText(story protocol.Issue) string { return firstNonEmpty(strings.TrimSpace(story.Body), strings.TrimSpace(story.Title), "Untitled story") } @@ -5002,7 +4924,7 @@ func storySummary(value string) string { return strings.TrimSpace(string(runes[:max-1])) + "..." } -func prHeaderHTML(pr brokerPullRequest, active string) string { +func prHeaderHTML(pr protocol.PullRequest, active string) string { status := firstNonEmpty(pr.Status, "open") title := firstNonEmpty(pr.Title, "Untitled pull request") filesActive := "" @@ -5033,7 +4955,7 @@ func prHeaderHTML(pr brokerPullRequest, active string) string { return b.String() } -func (s *webServer) pullRequestConversationHTML(ctx context.Context, pr brokerPullRequest) string { +func (s *webServer) pullRequestConversationHTML(ctx context.Context, pr protocol.PullRequest) string { contexts := s.prInlineCommentContexts(ctx, pr) var b strings.Builder b.WriteString(`
Conversation
`) @@ -5069,7 +4991,7 @@ func (s *webServer) pullRequestConversationHTML(ctx context.Context, pr brokerPu return b.String() } -func (s *webServer) prInlineCommentContexts(ctx context.Context, pr brokerPullRequest) map[string][]webVisualDiffRow { +func (s *webServer) prInlineCommentContexts(ctx context.Context, pr protocol.PullRequest) map[string][]webVisualDiffRow { contexts := map[string][]webVisualDiffRow{} if len(pr.Comments) == 0 && len(pr.Reviews) == 0 { return contexts @@ -5082,7 +5004,7 @@ func (s *webServer) prInlineCommentContexts(ctx context.Context, pr brokerPullRe for _, file := range files { filesByPath[file.path] = file } - collect := func(note brokerPullRequestNote) { + collect := func(note protocol.PullRequestNote) { for _, comment := range note.Comments { if comment.Kind != "line" || comment.File == "" || comment.Line <= 0 { continue @@ -5110,7 +5032,7 @@ func (s *webServer) prInlineCommentContexts(ctx context.Context, pr brokerPullRe return contexts } -func prCommentAfterContextRows(rows []webVisualDiffRow, comment brokerPullRequestComment) []webVisualDiffRow { +func prCommentAfterContextRows(rows []webVisualDiffRow, comment protocol.PullRequestComment) []webVisualDiffRow { target := -1 targetLine := strconv.Itoa(comment.Line) for i, row := range rows { @@ -5145,7 +5067,7 @@ func prCommentAfterContextRows(rows []webVisualDiffRow, comment brokerPullReques return context } -func prNoteHTML(note brokerPullRequestNote, action string, contexts map[string][]webVisualDiffRow) string { +func prNoteHTML(note protocol.PullRequestNote, action string, contexts map[string][]webVisualDiffRow) string { user := firstNonEmpty(note.User, "unknown") when := note.At if parsed, err := time.Parse(time.RFC3339, note.At); err == nil { @@ -5174,7 +5096,7 @@ func prNoteHTML(note brokerPullRequestNote, action string, contexts map[string][ return b.String() } -func prInlineCommentKey(comment brokerPullRequestComment) string { +func prInlineCommentKey(comment protocol.PullRequestComment) string { return strings.Join([]string{ comment.File, comment.Kind, @@ -5186,7 +5108,7 @@ func prInlineCommentKey(comment brokerPullRequestComment) string { }, "\x00") } -func prInlineCommentHTML(noteID int, comment brokerPullRequestComment, context []webVisualDiffRow) string { +func prInlineCommentHTML(noteID int, comment protocol.PullRequestComment, context []webVisualDiffRow) string { file := firstNonEmpty(comment.File, "Changed file") line := "" if comment.Kind == "line" && comment.Line > 0 { @@ -5251,7 +5173,7 @@ func prInlineCommentHTML(noteID int, comment brokerPullRequestComment, context [ return b.String() } -func prReplyThreadHTML(replies []brokerPullRequestComment, depth int) string { +func prReplyThreadHTML(replies []protocol.PullRequestComment, depth int) string { if len(replies) == 0 { return "" } @@ -5292,7 +5214,7 @@ func prReplyAttrs(noteID, commentID int) string { return attrs } -func prInlineCommentBodyHTML(comment brokerPullRequestComment, replyAttrs string) string { +func prInlineCommentBodyHTML(comment protocol.PullRequestComment, replyAttrs string) string { reply := "" if replyAttrs != "" { reply = `` @@ -6100,56 +6022,26 @@ func anchorID(value string) string { return strings.Trim(b.String(), "-") } -type webEventHub struct { - mu sync.Mutex - clients map[chan string]struct{} -} +type webEventHub struct{ *internalweb.EventHub } func newWebEventHub() *webEventHub { - return &webEventHub{clients: map[chan string]struct{}{}} + return &webEventHub{EventHub: internalweb.NewEventHub()} } func (h *webEventHub) subscribe() chan string { - ch := make(chan string, 8) - h.mu.Lock() - h.clients[ch] = struct{}{} - h.mu.Unlock() - return ch + return h.Subscribe() } func (h *webEventHub) unsubscribe(ch chan string) { - h.mu.Lock() - delete(h.clients, ch) - close(ch) - h.mu.Unlock() + h.Unsubscribe(ch) } func (h *webEventHub) broadcast(name string) { - payload := fmt.Sprintf("event: %s\ndata: {\"time\":%d}\n\n", name, time.Now().UnixMilli()) - h.mu.Lock() - defer h.mu.Unlock() - for ch := range h.clients { - select { - case ch <- payload: - default: - } - } + h.Broadcast(name) } func (h *webEventHub) broadcastJSON(name string, value any) { - data, err := json.Marshal(value) - if err != nil { - return - } - payload := fmt.Sprintf("event: %s\ndata: %s\n\n", name, data) - h.mu.Lock() - defer h.mu.Unlock() - for ch := range h.clients { - select { - case ch <- payload: - default: - } - } + h.BroadcastJSON(name, value) } func monitorWebPath(ctx context.Context, root, eventName string, hub *webEventHub) { @@ -6226,17 +6118,20 @@ func webAssetBytes(path string) ([]byte, error) { return data, nil } } - return webAssets.ReadFile(path) + return internalweb.ReadAsset(path) } func webAssetDiskPath(path string) string { - if _, err := os.Stat(path); err == nil { - return path + for _, candidate := range []string{filepath.Join("internal", "web", path), path} { + if _, err := os.Stat(candidate); err == nil { + return candidate + } } if exe, err := os.Executable(); err == nil { - candidate := filepath.Join(filepath.Dir(exe), path) - if _, statErr := os.Stat(candidate); statErr == nil { - return candidate + for _, candidate := range []string{filepath.Join(filepath.Dir(exe), "internal", "web", path), filepath.Join(filepath.Dir(exe), path)} { + if _, statErr := os.Stat(candidate); statErr == nil { + return candidate + } } } return "" diff --git a/internal/cli/main.go b/internal/cli/main.go new file mode 100644 index 0000000..571bc86 --- /dev/null +++ b/internal/cli/main.go @@ -0,0 +1,16 @@ +package cli + +import ( + "fmt" + "io" + "os" +) + +type Application func(executable string, args []string, stdin io.Reader, stdout, stderr io.Writer) error + +func Main(run Application) { + if err := run(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "fatal:", err) + os.Exit(1) + } +} diff --git a/internal/cli/options.go b/internal/cli/options.go new file mode 100644 index 0000000..2851d43 --- /dev/null +++ b/internal/cli/options.go @@ -0,0 +1,149 @@ +package cli + +import ( + "fmt" + "net/url" + "strings" +) + +type GlobalOptions struct { + Provider string + Bucket string + Prefix string + Branch string + Region string + Auth string + Profile string + Identity string + AuthExplicit bool + AuthFlagExplicit bool + ProfileExplicit bool + VersionRequested bool +} + +type GlobalOptionDefaults struct { + Branch string + Auth string + Profile string + AuthExplicit bool + ProfileExplicit bool +} + +func ParseGlobalOptions(arguments []string, defaults GlobalOptionDefaults) (GlobalOptions, []string, error) { + options := GlobalOptions{ + Branch: defaults.Branch, + Auth: defaults.Auth, + Profile: defaults.Profile, + AuthExplicit: defaults.AuthExplicit, + ProfileExplicit: defaults.ProfileExplicit, + } + var rest []string + for index := 0; index < len(arguments); index++ { + argument := arguments[index] + name, value, hasValue := strings.Cut(argument, "=") + takeValue := func() (string, error) { + if hasValue { + return value, nil + } + index++ + if index >= len(arguments) { + return "", fmt.Errorf("%s requires a value", name) + } + return arguments[index], nil + } + switch name { + case "--bucket": + value, err := takeValue() + if err != nil { + return options, nil, err + } + applyBucket(&options, value) + case "--prefix": + value, err := takeValue() + if err != nil { + return options, nil, err + } + options.Prefix = value + case "--branch": + value, err := takeValue() + if err != nil { + return options, nil, err + } + options.Branch = value + case "--region": + value, err := takeValue() + if err != nil { + return options, nil, err + } + options.Region = value + case "--auth": + value, err := takeValue() + if err != nil { + return options, nil, err + } + options.Auth, options.AuthExplicit, options.AuthFlagExplicit = value, true, true + case "--configuration", "--profile": + value, err := takeValue() + if err != nil { + return options, nil, err + } + options.Profile, options.ProfileExplicit = value, true + case "--identity": + value, err := takeValue() + if err != nil { + return options, nil, err + } + options.Identity = value + case "--version", "-v": + options.VersionRequested = true + default: + rest = append(rest, argument) + } + } + options.Prefix = strings.Trim(options.Prefix, "/") + options.Auth = strings.ToLower(strings.TrimSpace(options.Auth)) + if options.Auth == "" { + options.Auth = "gcloud" + } + if options.Auth != "gcloud" && options.Auth != "adc" { + return options, nil, fmt.Errorf("unsupported auth mode %q", options.Auth) + } + return options, rest, nil +} + +func applyBucket(options *GlobalOptions, raw string) { + provider, bucket, prefix := splitBucketTarget(raw) + if provider != "" { + options.Provider = provider + } + if bucket != "" { + options.Bucket = bucket + } else { + options.Bucket = raw + } + if prefix != "" && options.Prefix == "" { + options.Prefix = prefix + } +} + +func splitBucketTarget(raw string) (string, string, string) { + bucket := strings.TrimSpace(raw) + if bucket == "" { + return "", "", "" + } + if strings.HasPrefix(bucket, "gs://") || strings.HasPrefix(bucket, "gcs://") || strings.HasPrefix(bucket, "s3://") { + parsed, err := url.Parse(bucket) + if err == nil && parsed.Host != "" { + provider := "gcs" + if parsed.Scheme == "s3" { + provider = "s3" + } + return provider, parsed.Host, strings.Trim(parsed.Path, "/") + } + } + bucket = strings.Trim(bucket, "/") + if slash := strings.Index(bucket, "/"); slash >= 0 { + return "", bucket[:slash], strings.Trim(bucket[slash+1:], "/") + } + return "", bucket, "" +} diff --git a/internal/cli/options_test.go b/internal/cli/options_test.go new file mode 100644 index 0000000..da8a11b --- /dev/null +++ b/internal/cli/options_test.go @@ -0,0 +1,45 @@ +package cli + +import "testing" + +func TestParseGlobalOptions(t *testing.T) { + options, rest, err := ParseGlobalOptions([]string{"--bucket", "s3://bucket/repo.git", "--profile=work", "--auth", "adc", "clone"}, GlobalOptionDefaults{Branch: "main", Auth: "gcloud"}) + if err != nil { + t.Fatal(err) + } + if options.Provider != "s3" || options.Bucket != "bucket" || options.Prefix != "repo.git" || options.Profile != "work" || options.Auth != "adc" || !options.AuthFlagExplicit || len(rest) != 1 || rest[0] != "clone" { + t.Fatalf("options=%#v rest=%q", options, rest) + } +} + +func TestParseGlobalOptionsTracksExplicitDefaults(t *testing.T) { + implicit, _, err := ParseGlobalOptions(nil, GlobalOptionDefaults{Auth: "gcloud", Profile: "default"}) + if err != nil { + t.Fatal(err) + } + if implicit.AuthExplicit || implicit.ProfileExplicit { + t.Fatalf("implicit defaults marked explicit: %#v", implicit) + } + + explicit, _, err := ParseGlobalOptions(nil, GlobalOptionDefaults{ + Auth: "adc", + Profile: "work", + AuthExplicit: true, + ProfileExplicit: true, + }) + if err != nil { + t.Fatal(err) + } + if !explicit.AuthExplicit || !explicit.ProfileExplicit { + t.Fatalf("explicit defaults not preserved: %#v", explicit) + } +} + +func TestParseGlobalOptionsRejectsInvalidAuthAndMissingValues(t *testing.T) { + if _, _, err := ParseGlobalOptions([]string{"--auth", "invalid"}, GlobalOptionDefaults{}); err == nil { + t.Fatal("invalid auth accepted") + } + if _, _, err := ParseGlobalOptions([]string{"--identity"}, GlobalOptionDefaults{}); err == nil { + t.Fatal("missing identity accepted") + } +} diff --git a/internal/cli/presentation.go b/internal/cli/presentation.go new file mode 100644 index 0000000..86915fb --- /dev/null +++ b/internal/cli/presentation.go @@ -0,0 +1,98 @@ +package cli + +import ( + "fmt" + "io" +) + +const usageText = `usage: bgit [args]. These are common BucketGit commands: + +start a repository + setup Connect a cloud account and deploy or update BucketGit + init Create a local Git repository backed by BucketGit + clone Clone a BucketGit repository into a new directory + +local web UI + web Browse and manage a repository locally + +work on the current change + add Add file contents to the index + mv Move or rename a file, directory, or symlink + restore Restore working tree files + rm Remove files from the working tree and index + +examine history and state + diff Show changes between commits, commit and working tree, etc + grep Print lines matching a pattern + log Show commit logs + show Show objects + status Show the working tree status + +grow, mark, and tweak history + branch List, create, or delete branches + checkout Switch branches or restore paths + commit Record changes to the repository + merge Join development histories together + reset Reset HEAD, index, or working tree state + tag Create, list, delete, or verify tags + +collaborate + fetch Download objects and refs from BucketGit + pull Fetch and integrate with the current branch + push Update remote refs and upload objects + ls-remote List remote refs + pr Create, review, merge, and close pull requests + ci Run and inspect broker CI builds + board Manage the repository task board + issue Create, comment on, close, and reopen issues + +administer + whoami Show broker identity, role, and capabilities for this repo + repos List repositories visible to local SSH keys + admin Manage broker-backed users, keys, owners, and protection + janitor Run broker maintenance and repair tasks + broker Delete or decommission deployed broker infrastructure + direct Run direct bucket recovery and administration commands + +Run "bgit help " or "bgit direct help" for details. +` + +func Usage(output io.Writer) error { + _, err := io.WriteString(output, usageText) + return err +} + +func UsageWithBanner(output io.Writer, version string) error { + if _, err := fmt.Fprintf(output, + " _____________________________________________________________\n"+ + " | â–„â–„â–„â–„â–„â–„â–„ â–„â–„â–„â–„â–„â–„â–„ |\n"+ + " | ███▀▀███▄ â–„â–„ ██ ███▀▀▀▀▀ ▀▀ ██ |\n"+ + "________| ███▄▄███▀ ██ ██ ▄████ ██ ▄█▀ ▄█▀█▄ ▀██▀▀ ███ ██ ▀██▀▀ |________\n"+ + "\\ | ███ ███▄ ██ ██ ██ ████ ██▄█▀ ██ ███ ███▀ ██ ██ | /\n"+ + " \\ | ████████▀ ▀██▀█ ▀████ ██ ▀█▄ ▀█▄▄▄ ██ ▀██████▀ ██▄ ██ | /\n"+ + " / |_____________________________________________________________| \\\n"+ + "/________) [ bucketgit.com / github.com/bucketgit / bgit %s ] (________\\\n"+ + "\n", version); err != nil { + return err + } + return Usage(output) +} + +type Intent struct { + Command string + Args []string + Help bool + Version bool +} + +func ParseIntent(arguments []string) Intent { + intent := Intent{} + if len(arguments) == 0 { + return intent + } + intent.Command = arguments[0] + intent.Args = append([]string(nil), arguments[1:]...) + intent.Help = len(intent.Args) == 1 && (intent.Args[0] == "help" || intent.Args[0] == "-h" || intent.Args[0] == "--help") + intent.Version = len(intent.Args) == 1 && (intent.Args[0] == "version" || intent.Args[0] == "-v" || intent.Args[0] == "--version") + return intent +} diff --git a/internal/cli/presentation_test.go b/internal/cli/presentation_test.go new file mode 100644 index 0000000..ed02979 --- /dev/null +++ b/internal/cli/presentation_test.go @@ -0,0 +1,30 @@ +package cli + +import ( + "bytes" + "strings" + "testing" +) + +func TestParseIntent(t *testing.T) { + intent := ParseIntent([]string{"board", "--help"}) + if intent.Command != "board" || !intent.Help || intent.Version { + t.Fatalf("intent=%#v", intent) + } + intent = ParseIntent([]string{"status", "--version"}) + if intent.Command != "status" || !intent.Version || intent.Help { + t.Fatalf("intent=%#v", intent) + } +} + +func TestUsageWithBanner(t *testing.T) { + var output bytes.Buffer + if err := UsageWithBanner(&output, "1.2.3"); err != nil { + t.Fatal(err) + } + for _, expected := range []string{"bgit 1.2.3", "usage: bgit", "local web UI", "administer"} { + if !strings.Contains(output.String(), expected) { + t.Fatalf("output does not contain %q", expected) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..1282860 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,79 @@ +package config + +import ( + "errors" + "fmt" + "net/url" + "path/filepath" + "strings" +) + +type Repository struct { + Provider string + Bucket string + Prefix string + Branch string + Origin string +} + +func ParseRepositoryURI(raw string) (Repository, string, error) { + parsed, err := url.Parse(raw) + if err != nil { + return Repository{}, "", err + } + provider, scheme := "", parsed.Scheme + switch scheme { + case "gs", "gcs": + provider, scheme = "gcs", "gs" + case "s3": + provider = "s3" + default: + return Repository{}, "", errors.New("repository URI must start with gs:// or s3://") + } + if parsed.Host == "" { + return Repository{}, "", errors.New("repository URI must include a bucket name") + } + prefix := strings.Trim(parsed.Path, "/") + if prefix == "" { + return Repository{}, "", errors.New("repository URI must include a repository prefix") + } + parts := strings.Split(prefix, "/") + name := parts[len(parts)-1] + origin := fmt.Sprintf("%s://%s/%s", scheme, parsed.Host, prefix) + return Repository{Provider: provider, Bucket: parsed.Host, Prefix: prefix, Branch: "main", Origin: origin}, name, nil +} + +func LocalSelection(value string) (string, string) { + value = strings.TrimPrefix(strings.TrimSpace(value), "local:") + if value == "" { + return "default", "default" + } + for _, separator := range []string{"/", "."} { + if strings.Contains(value, separator) { + parts := strings.SplitN(value, separator, 2) + return fallback(parts[0]), fallback(parts[1]) + } + } + return value, "default" +} + +func NormalizeLogicalRepository(name string) (string, error) { + name = strings.TrimSuffix(strings.TrimSpace(name), ".git") + if name == "" { + return "", errors.New("logical repo name is required") + } + if strings.ContainsAny(name, `/\`) { + return "", fmt.Errorf("logical repo names must be flat; use %q instead of a path", filepath.Base(name)) + } + if name == "." || name == ".." { + return "", errors.New("logical repo name is invalid") + } + return name + ".git", nil +} + +func fallback(value string) string { + if strings.TrimSpace(value) == "" { + return "default" + } + return strings.TrimSpace(value) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..2392be9 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,31 @@ +package config + +import "testing" + +func TestParseRepositoryURI(t *testing.T) { + parsed, name, err := ParseRepositoryURI("gs://bucket/team/demo.git") + if err != nil || parsed.Provider != "gcs" || parsed.Bucket != "bucket" || parsed.Prefix != "team/demo.git" || name != "demo.git" { + t.Fatalf("parsed=%#v name=%q err=%v", parsed, name, err) + } + if _, _, err := ParseRepositoryURI("file://demo.git"); err == nil { + t.Fatal("file URI accepted as cloud bucket URI") + } +} + +func TestLocalSelection(t *testing.T) { + for input, want := range map[string][2]string{"": {"default", "default"}, "local:work/eu": {"work", "eu"}, "work.eu": {"work", "eu"}} { + profile, region := LocalSelection(input) + if [2]string{profile, region} != want { + t.Fatalf("LocalSelection(%q)=(%q,%q), want %v", input, profile, region, want) + } + } +} + +func TestNormalizeLogicalRepository(t *testing.T) { + if got, err := NormalizeLogicalRepository("demo"); err != nil || got != "demo.git" { + t.Fatalf("normalize=%q,%v", got, err) + } + if _, err := NormalizeLogicalRepository("team/demo"); err == nil { + t.Fatal("nested logical name accepted") + } +} diff --git a/global_config.go b/internal/config/global.go similarity index 75% rename from global_config.go rename to internal/config/global.go index a4b27e6..e91d0aa 100644 --- a/global_config.go +++ b/internal/config/global.go @@ -1,4 +1,4 @@ -package main +package config import ( "bytes" @@ -10,23 +10,44 @@ import ( "gopkg.in/yaml.v3" ) -const globalConfigVersion = 1 +func firstNonEmptyValue(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func expandHomePath(value string) string { + if value == "~" || strings.HasPrefix(value, "~/") || strings.HasPrefix(value, `~\`) { + if home, err := os.UserHomeDir(); err == nil { + if value == "~" { + return home + } + return filepath.Join(home, value[2:]) + } + } + return value +} + +const GlobalVersion = 1 -type globalConfig struct { +type Global struct { Version int - Identity globalIdentityConfig - GCPProfiles []globalGCPProfile - AWSProfiles []globalAWSProfile - LocalProfiles []globalLocalProfile - Repos []globalRepoConfig + Identity Identity + GCPProfiles []GCPProfile + AWSProfiles []AWSProfile + LocalProfiles []LocalProfile + Repos []Repo } -type globalIdentityConfig struct { +type Identity struct { Name string Email string } -type globalGCPProfile struct { +type GCPProfile struct { Name string ProjectID string Account string @@ -35,10 +56,10 @@ type globalGCPProfile struct { BrokerURL string BrokerVersion string LastSetupAt string - Regions []globalProfileRegion + Regions []ProfileRegion } -type globalAWSProfile struct { +type AWSProfile struct { Name string AccountID string ARN string @@ -46,10 +67,10 @@ type globalAWSProfile struct { BrokerURL string BrokerVersion string LastSetupAt string - Regions []globalProfileRegion + Regions []ProfileRegion } -type globalLocalProfile struct { +type LocalProfile struct { Name string Root string Autostart bool @@ -57,49 +78,49 @@ type globalLocalProfile struct { BrokerURL string BrokerVersion string LastSetupAt string - Regions []globalProfileRegion + Regions []ProfileRegion } -type globalProfileRegion struct { +type ProfileRegion struct { Name string BrokerURL string BrokerVersion string LastSetupAt string } -type globalRepoConfig struct { +type Repo struct { Name string Profile string BrokerURL string } -func defaultGlobalConfigPath() (string, error) { - home, err := bgitHomeDir() +func DefaultGlobalPath() (string, error) { + home, err := HomeDir() if err != nil { return "", err } return filepath.Join(home, "config.yaml"), nil } -func defaultLocalBrokerRoot() (string, error) { - home, err := bgitHomeDir() +func DefaultLocalBrokerRoot() (string, error) { + home, err := HomeDir() if err != nil { return "", err } return filepath.Join(home, "local-broker"), nil } -func defaultBGitCacheDir() (string, error) { - home, err := bgitHomeDir() +func DefaultCacheDir() (string, error) { + home, err := HomeDir() if err != nil { return "", err } return filepath.Join(home, "cache"), nil } -func bgitHomeDir() (string, error) { +func HomeDir() (string, error) { if value := strings.TrimSpace(os.Getenv("BGIT_HOME")); value != "" { - return expandHome(value), nil + return expandHomePath(value), nil } home, err := os.UserHomeDir() if err != nil { @@ -108,16 +129,16 @@ func bgitHomeDir() (string, error) { return filepath.Join(home, ".bgit"), nil } -func readGlobalConfig(path string) (globalConfig, error) { +func ReadGlobal(path string) (Global, error) { data, err := os.ReadFile(path) if err != nil { - return globalConfig{}, err + return Global{}, err } - cfg, err := parseGlobalConfigYAML(data) + cfg, err := ParseGlobalYAML(data) if err != nil { return cfg, err } - normalizeGlobalConfigProfileRegions(&cfg) + NormalizeProfileRegions(&cfg) return cfg, nil } @@ -177,84 +198,84 @@ type globalRepoYAML struct { BrokerURL string `yaml:"broker_url,omitempty"` } -func parseGlobalConfigYAML(data []byte) (globalConfig, error) { +func ParseGlobalYAML(data []byte) (Global, error) { var raw globalConfigYAML dec := yaml.NewDecoder(bytes.NewReader(data)) dec.KnownFields(true) if err := dec.Decode(&raw); err != nil { - return globalConfig{}, err + return Global{}, err } return globalConfigFromYAML(raw), nil } -func globalConfigFromYAML(raw globalConfigYAML) globalConfig { - cfg := globalConfig{ +func globalConfigFromYAML(raw globalConfigYAML) Global { + cfg := Global{ Version: raw.Version, - Identity: globalIdentityConfig{ + Identity: Identity{ Name: raw.Identity.Name, Email: raw.Identity.Email, }, } if cfg.Version == 0 { - cfg.Version = globalConfigVersion + cfg.Version = GlobalVersion } for name, profile := range raw.GCP.Profiles { - next := globalGCPProfile{ + next := GCPProfile{ Name: name, ProjectID: profile.ProjectID, Account: profile.Account, ServiceAccount: profile.ServiceAccount, } for regionName, region := range profile.Regions { - next.Regions = append(next.Regions, globalProfileRegion{ + next.Regions = append(next.Regions, ProfileRegion{ Name: regionName, BrokerURL: region.BrokerURL, BrokerVersion: region.BrokerVersion, LastSetupAt: region.LastSetupAt, }) } - sortGlobalProfileRegions(next.Regions) + SortProfileRegions(next.Regions) cfg.GCPProfiles = append(cfg.GCPProfiles, next) } for name, profile := range raw.AWS.Profiles { - next := globalAWSProfile{ + next := AWSProfile{ Name: name, AccountID: profile.AccountID, ARN: profile.ARN, } for regionName, region := range profile.Regions { - next.Regions = append(next.Regions, globalProfileRegion{ + next.Regions = append(next.Regions, ProfileRegion{ Name: regionName, BrokerURL: region.BrokerURL, BrokerVersion: region.BrokerVersion, LastSetupAt: region.LastSetupAt, }) } - sortGlobalProfileRegions(next.Regions) + SortProfileRegions(next.Regions) cfg.AWSProfiles = append(cfg.AWSProfiles, next) } for name, profile := range raw.Local.Profiles { - next := globalLocalProfile{Name: name, Root: profile.Root, Autostart: profile.Autostart} + next := LocalProfile{Name: name, Root: profile.Root, Autostart: profile.Autostart} for regionName, region := range profile.Regions { - next.Regions = append(next.Regions, globalProfileRegion{ + next.Regions = append(next.Regions, ProfileRegion{ Name: regionName, BrokerURL: region.BrokerURL, BrokerVersion: region.BrokerVersion, LastSetupAt: region.LastSetupAt, }) } - sortGlobalProfileRegions(next.Regions) + SortProfileRegions(next.Regions) cfg.LocalProfiles = append(cfg.LocalProfiles, next) } for name, repo := range raw.Repos { - cfg.Repos = append(cfg.Repos, globalRepoConfig{Name: name, Profile: repo.Profile, BrokerURL: repo.BrokerURL}) + cfg.Repos = append(cfg.Repos, Repo{Name: name, Profile: repo.Profile, BrokerURL: repo.BrokerURL}) } sortGlobalConfig(&cfg) return cfg } -func globalConfigToYAML(cfg globalConfig) globalConfigYAML { - normalizeGlobalConfigProfileRegions(&cfg) +func globalConfigToYAML(cfg Global) globalConfigYAML { + NormalizeProfileRegions(&cfg) sortGlobalConfig(&cfg) out := globalConfigYAML{ Version: cfg.Version, @@ -268,7 +289,7 @@ func globalConfigToYAML(cfg globalConfig) globalConfigYAML { Repos: map[string]globalRepoYAML{}, } if out.Version == 0 { - out.Version = globalConfigVersion + out.Version = GlobalVersion } for _, profile := range cfg.GCPProfiles { next := globalGCPProfileYAML{ @@ -322,39 +343,39 @@ func globalConfigToYAML(cfg globalConfig) globalConfigYAML { return out } -func sortGlobalConfig(cfg *globalConfig) { +func sortGlobalConfig(cfg *Global) { sort.Slice(cfg.GCPProfiles, func(i, j int) bool { return cfg.GCPProfiles[i].Name < cfg.GCPProfiles[j].Name }) for i := range cfg.GCPProfiles { - sortGlobalProfileRegions(cfg.GCPProfiles[i].Regions) + SortProfileRegions(cfg.GCPProfiles[i].Regions) } sort.Slice(cfg.AWSProfiles, func(i, j int) bool { return cfg.AWSProfiles[i].Name < cfg.AWSProfiles[j].Name }) for i := range cfg.AWSProfiles { - sortGlobalProfileRegions(cfg.AWSProfiles[i].Regions) + SortProfileRegions(cfg.AWSProfiles[i].Regions) } sort.Slice(cfg.LocalProfiles, func(i, j int) bool { return cfg.LocalProfiles[i].Name < cfg.LocalProfiles[j].Name }) for i := range cfg.LocalProfiles { - sortGlobalProfileRegions(cfg.LocalProfiles[i].Regions) + SortProfileRegions(cfg.LocalProfiles[i].Regions) } sort.Slice(cfg.Repos, func(i, j int) bool { return cfg.Repos[i].Name < cfg.Repos[j].Name }) } -func sortGlobalProfileRegions(regions []globalProfileRegion) { +func SortProfileRegions(regions []ProfileRegion) { sort.Slice(regions, func(i, j int) bool { return regions[i].Name < regions[j].Name }) } -func writeGlobalConfig(path string, cfg globalConfig) error { +func WriteGlobal(path string, cfg Global) error { if cfg.Version == 0 { - cfg.Version = globalConfigVersion + cfg.Version = GlobalVersion } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err @@ -366,12 +387,12 @@ func writeGlobalConfig(path string, cfg globalConfig) error { return os.WriteFile(path, data, 0o644) } -func normalizeGlobalConfigProfileRegions(cfg *globalConfig) { +func NormalizeProfileRegions(cfg *Global) { for i := range cfg.GCPProfiles { profile := &cfg.GCPProfiles[i] if len(profile.Regions) == 0 && strings.TrimSpace(profile.BrokerURL) != "" { - profile.Regions = append(profile.Regions, globalProfileRegion{ - Name: firstNonEmpty(profile.Region, "us-central1"), + profile.Regions = append(profile.Regions, ProfileRegion{ + Name: firstNonEmptyValue(profile.Region, "us-central1"), BrokerURL: profile.BrokerURL, BrokerVersion: profile.BrokerVersion, LastSetupAt: profile.LastSetupAt, @@ -385,8 +406,8 @@ func normalizeGlobalConfigProfileRegions(cfg *globalConfig) { for i := range cfg.AWSProfiles { profile := &cfg.AWSProfiles[i] if len(profile.Regions) == 0 && strings.TrimSpace(profile.BrokerURL) != "" { - profile.Regions = append(profile.Regions, globalProfileRegion{ - Name: firstNonEmpty(profile.Region, "us-east-1"), + profile.Regions = append(profile.Regions, ProfileRegion{ + Name: firstNonEmptyValue(profile.Region, "us-east-1"), BrokerURL: profile.BrokerURL, BrokerVersion: profile.BrokerVersion, LastSetupAt: profile.LastSetupAt, @@ -400,8 +421,8 @@ func normalizeGlobalConfigProfileRegions(cfg *globalConfig) { for i := range cfg.LocalProfiles { profile := &cfg.LocalProfiles[i] if len(profile.Regions) == 0 && strings.TrimSpace(profile.BrokerURL) != "" { - profile.Regions = append(profile.Regions, globalProfileRegion{ - Name: firstNonEmpty(profile.Region, "default"), + profile.Regions = append(profile.Regions, ProfileRegion{ + Name: firstNonEmptyValue(profile.Region, "default"), BrokerURL: profile.BrokerURL, BrokerVersion: profile.BrokerVersion, LastSetupAt: profile.LastSetupAt, diff --git a/internal/config/repository_file.go b/internal/config/repository_file.go new file mode 100644 index 0000000..f5c81eb --- /dev/null +++ b/internal/config/repository_file.go @@ -0,0 +1,157 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// RepositoryFile is the parsed representation of a repository's .git/config. +type RepositoryFile struct { + sections map[string]map[string]string + order []string +} + +func ReadRepositoryFile(path string) (*RepositoryFile, error) { + value := &RepositoryFile{sections: map[string]map[string]string{}} + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return value, nil + } + if err != nil { + return nil, err + } + section := "" + for _, line := range strings.Split(string(data), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";") { + continue + } + if strings.HasPrefix(trimmed, "[") && strings.HasSuffix(trimmed, "]") { + section = strings.TrimSpace(strings.Trim(trimmed, "[]")) + value.ensureSection(section) + continue + } + name, setting, ok := strings.Cut(trimmed, "=") + if !ok || section == "" { + continue + } + value.sections[section][strings.TrimSpace(name)] = strings.Trim(strings.TrimSpace(setting), `"`) + } + return value, nil +} + +func (c *RepositoryFile) Get(key string) (string, bool) { + section, name := sectionAndName(key) + values, ok := c.sections[section] + if !ok { + return "", false + } + value, ok := values[name] + return value, ok +} + +func (c *RepositoryFile) Set(key, value string) { + section, name := sectionAndName(key) + c.ensureSection(section) + c.sections[section][name] = value +} + +func (c *RepositoryFile) Unset(key string) { + section, name := sectionAndName(key) + if values, ok := c.sections[section]; ok { + delete(values, name) + } +} + +func (c *RepositoryFile) Keys() []string { + var keys []string + for _, section := range c.orderedSections() { + for name := range c.sections[section] { + keys = append(keys, fullKey(section, name)) + } + } + sort.Strings(keys) + return keys +} + +func (c *RepositoryFile) Write(path string) error { + var output strings.Builder + for _, section := range c.orderedSections() { + values := c.sections[section] + if len(values) == 0 { + continue + } + fmt.Fprintf(&output, "[%s]\n", section) + names := make([]string, 0, len(values)) + for name := range values { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + fmt.Fprintf(&output, "\t%s = %s\n", name, values[name]) + } + output.WriteByte('\n') + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, []byte(output.String()), 0o644) +} + +func (c *RepositoryFile) ensureSection(section string) { + if c.sections == nil { + c.sections = map[string]map[string]string{} + } + if _, ok := c.sections[section]; ok { + return + } + c.sections[section] = map[string]string{} + c.order = append(c.order, section) +} + +func (c *RepositoryFile) orderedSections() []string { + seen := map[string]struct{}{} + var sections []string + for _, section := range c.order { + if _, ok := c.sections[section]; ok { + sections = append(sections, section) + seen[section] = struct{}{} + } + } + for section := range c.sections { + if _, ok := seen[section]; !ok { + sections = append(sections, section) + } + } + sort.SliceStable(sections, func(i, j int) bool { + _, leftSeen := seen[sections[i]] + _, rightSeen := seen[sections[j]] + if leftSeen != rightSeen { + return leftSeen + } + return sections[i] < sections[j] + }) + return sections +} + +func sectionAndName(key string) (string, string) { + parts := strings.Split(key, ".") + if len(parts) <= 2 { + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], parts[1] + } + return fmt.Sprintf(`%s "%s"`, parts[0], parts[1]), strings.Join(parts[2:], ".") +} + +func fullKey(section, name string) string { + if before, after, ok := strings.Cut(section, " "); ok { + return before + "." + strings.Trim(after, `"`) + "." + name + } + return section + "." + name +} diff --git a/internal/identity/agent_unix.go b/internal/identity/agent_unix.go new file mode 100644 index 0000000..a47875f --- /dev/null +++ b/internal/identity/agent_unix.go @@ -0,0 +1,29 @@ +//go:build !windows + +package identity + +import ( + "net" + "os" + "strings" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" +) + +func AgentSigners() ([]ssh.Signer, func(), error) { + socket := strings.TrimSpace(os.Getenv("SSH_AUTH_SOCK")) + if socket == "" { + return nil, nil, nil + } + connection, err := net.Dial("unix", socket) + if err != nil { + return nil, nil, err + } + signers, err := agent.NewClient(connection).Signers() + if err != nil { + _ = connection.Close() + return nil, nil, err + } + return signers, func() { _ = connection.Close() }, nil +} diff --git a/internal/identity/agent_windows.go b/internal/identity/agent_windows.go new file mode 100644 index 0000000..8123411 --- /dev/null +++ b/internal/identity/agent_windows.go @@ -0,0 +1,35 @@ +//go:build windows + +package identity + +import ( + "net" + "os" + "strings" + "time" + + "github.com/Microsoft/go-winio" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" +) + +const windowsOpenSSHAgentPipe = `\\.\pipe\openssh-ssh-agent` + +func AgentSigners() ([]ssh.Signer, func(), error) { + socket := strings.TrimSpace(os.Getenv("SSH_AUTH_SOCK")) + if !strings.HasPrefix(socket, `\\.\pipe\`) { + socket = windowsOpenSSHAgentPipe + } + timeout := 5 * time.Second + var connection net.Conn + connection, err := winio.DialPipe(socket, &timeout) + if err != nil { + return nil, nil, err + } + signers, err := agent.NewClient(connection).Signers() + if err != nil { + _ = connection.Close() + return nil, nil, err + } + return signers, func() { _ = connection.Close() }, nil +} diff --git a/internal/identity/identity.go b/internal/identity/identity.go new file mode 100644 index 0000000..ee37272 --- /dev/null +++ b/internal/identity/identity.go @@ -0,0 +1,41 @@ +package identity + +import ( + "regexp" + "strings" +) + +const DefaultName = "BucketGit Client" + +var emailPattern = regexp.MustCompile(`^[^@\s]+@[^@\s]+\.[^@\s]+$`) + +type Value struct { + Name, Email string + UsesDefault bool +} + +func DefaultEmail(username string) string { + username = strings.ToLower(strings.TrimSpace(username)) + var clean strings.Builder + for _, char := range username { + if (char >= 'a' && char <= 'z') || (char >= '0' && char <= '9') || char == '.' || char == '_' || char == '-' { + clean.WriteRune(char) + } + } + if clean.Len() == 0 { + return "username@bucketgit.com" + } + return clean.String() + "@bucketgit.com" +} + +func Effective(name, email, defaultEmail string) Value { + if strings.TrimSpace(name) == "" { + name = DefaultName + } + if strings.TrimSpace(email) == "" { + email = defaultEmail + } + return Value{Name: strings.TrimSpace(name), Email: strings.TrimSpace(email), UsesDefault: name == DefaultName || email == defaultEmail} +} + +func ValidEmail(value string) bool { return emailPattern.MatchString(strings.TrimSpace(value)) } diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go new file mode 100644 index 0000000..2f8dffb --- /dev/null +++ b/internal/identity/identity_test.go @@ -0,0 +1,16 @@ +package identity + +import "testing" + +func TestDefaultEmailAndEffective(t *testing.T) { + if got := DefaultEmail("Dennis Vink!"); got != "dennisvink@bucketgit.com" { + t.Fatalf("DefaultEmail=%q", got) + } + value := Effective("", "", "dennis@bucketgit.com") + if value.Name != DefaultName || !value.UsesDefault { + t.Fatalf("Effective=%#v", value) + } + if !ValidEmail("user@example.com") || ValidEmail("invalid") { + t.Fatal("email validation mismatch") + } +} diff --git a/internal/identity/signers.go b/internal/identity/signers.go new file mode 100644 index 0000000..7ba4f5e --- /dev/null +++ b/internal/identity/signers.go @@ -0,0 +1,72 @@ +package identity + +import ( + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/ssh" +) + +// ExplicitSigners loads private keys selected through BucketGit environment +// variables or an explicit non-fingerprint identity preference. +func ExplicitSigners(preference string) []ssh.Signer { + var paths []string + for _, environment := range []string{"BGIT_SSH_KEY", "BGIT_SSH_KEYS"} { + for _, value := range filepath.SplitList(os.Getenv(environment)) { + if value = strings.TrimSpace(value); value != "" { + paths = append(paths, value) + } + } + } + var inline []string + if value := strings.TrimSpace(os.Getenv("GIT_SSH_PRIVATE_KEY")); value != "" { + inline = append(inline, value) + } + if value := strings.TrimSpace(preference); value != "" && !strings.HasPrefix(value, "SHA256:") { + paths = append(paths, value) + } + var signers []ssh.Signer + for _, value := range inline { + if signer, err := ssh.ParsePrivateKey([]byte(value)); err == nil { + signers = append(signers, signer) + } + } + seen := map[string]struct{}{} + for _, name := range paths { + name = expandHome(name) + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + data, err := os.ReadFile(name) + if err != nil { + continue + } + if signer, err := ssh.ParsePrivateKey(data); err == nil { + signers = append(signers, signer) + } + } + return signers +} + +func FingerprintRank(fingerprint string, preferred []string) int { + for index, value := range preferred { + if strings.EqualFold(strings.TrimSpace(value), strings.TrimSpace(fingerprint)) { + return index + } + } + return len(preferred) + 1 +} + +func expandHome(value string) string { + if value == "~" || strings.HasPrefix(value, "~/") || strings.HasPrefix(value, `~\`) { + if home, err := os.UserHomeDir(); err == nil { + if value == "~" { + return home + } + return filepath.Join(home, value[2:]) + } + } + return value +} diff --git a/internal/setup/assets.go b/internal/setup/assets.go new file mode 100644 index 0000000..c4431de --- /dev/null +++ b/internal/setup/assets.go @@ -0,0 +1,45 @@ +package setup + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func WriteGCPBrokerSource(assets fs.FS, sourceRoot, destination, version string) error { + return writeNodeSource(assets, sourceRoot, destination, version, map[string]string{"package.json": "package.json", "index.js": "index.js"}) +} + +func WriteGCPMaterializerSource(assets fs.FS, sourceRoot, destination, version string) error { + return writeNodeSource(assets, sourceRoot, destination, version, map[string]string{"package.json": "package.json", "materializer.js": "index.js"}) +} + +func AWSTemplate(assets fs.FS, path, version string) (string, error) { + data, err := fs.ReadFile(assets, path) + if err != nil { + return "", err + } + return strings.ReplaceAll(string(data), "{{BROKER_VERSION}}", version), nil +} + +func writeNodeSource(assets fs.FS, sourceRoot, destination, version string, files map[string]string) error { + if strings.TrimSpace(destination) == "" { + return fmt.Errorf("deployment source destination is required") + } + if err := os.MkdirAll(destination, 0o755); err != nil { + return err + } + for source, target := range files { + data, err := fs.ReadFile(assets, filepath.ToSlash(filepath.Join(sourceRoot, source))) + if err != nil { + return err + } + body := strings.ReplaceAll(string(data), "{{BROKER_VERSION}}", version) + if err := os.WriteFile(filepath.Join(destination, target), []byte(body), 0o644); err != nil { + return err + } + } + return nil +} diff --git a/internal/setup/assets_test.go b/internal/setup/assets_test.go new file mode 100644 index 0000000..81bd36a --- /dev/null +++ b/internal/setup/assets_test.go @@ -0,0 +1,29 @@ +package setup + +import ( + "os" + "path/filepath" + "testing" + "testing/fstest" +) + +func TestDeploymentAssetsSubstituteVersion(t *testing.T) { + assets := fstest.MapFS{ + "gcp/package.json": {Data: []byte(`{"version":"{{BROKER_VERSION}}"}`)}, + "gcp/index.js": {Data: []byte(`exports.version = "{{BROKER_VERSION}}"`)}, + "gcp/materializer.js": {Data: []byte(`exports.materializer = "{{BROKER_VERSION}}"`)}, + "aws/template.yaml": {Data: []byte(`Version: {{BROKER_VERSION}}`)}, + } + destination := t.TempDir() + if err := WriteGCPBrokerSource(assets, "gcp", destination, "1.2.3"); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(destination, "index.js")) + if err != nil || string(data) != `exports.version = "1.2.3"` { + t.Fatalf("data=%q err=%v", data, err) + } + template, err := AWSTemplate(assets, "aws/template.yaml", "1.2.3") + if err != nil || template != "Version: 1.2.3" { + t.Fatalf("template=%q err=%v", template, err) + } +} diff --git a/internal/setup/commands.go b/internal/setup/commands.go new file mode 100644 index 0000000..3427ced --- /dev/null +++ b/internal/setup/commands.go @@ -0,0 +1,55 @@ +// Package setup contains cloud CLI execution and broker deployment +// orchestration. Interactive profile selection remains in internal/cli. +package setup + +import ( + "context" + "os" + "os/exec" + "strings" + "time" +) + +func GCloudCommand(ctx context.Context, configuration string, arguments ...string) *exec.Cmd { + command := exec.CommandContext(ctx, "gcloud", arguments...) + if configuration = strings.TrimSpace(configuration); configuration != "" { + command.Env = append(os.Environ(), "CLOUDSDK_ACTIVE_CONFIG_NAME="+configuration) + } + return command +} + +func AWSCommand(ctx context.Context, profile string, arguments ...string) *exec.Cmd { + command := exec.CommandContext(ctx, "aws", arguments...) + if profile = strings.TrimSpace(profile); profile != "" { + command.Env = append(os.Environ(), "AWS_PROFILE="+profile, "AWS_SDK_LOAD_CONFIG=1") + } + return command +} + +type RetryOptions struct { + Attempts int + Delay time.Duration + Retry func(output []byte, err error) bool + Sleep func(time.Duration) +} + +// RunWithRetry rebuilds a command for each attempt so callers never reuse an +// executed exec.Cmd. It is used for eventually consistent cloud IAM updates. +func RunWithRetry(build func() *exec.Cmd, options RetryOptions) ([]byte, error) { + if options.Attempts <= 0 { + options.Attempts = 1 + } + if options.Sleep == nil { + options.Sleep = time.Sleep + } + var output []byte + var err error + for attempt := 0; attempt < options.Attempts; attempt++ { + output, err = build().CombinedOutput() + if err == nil || options.Retry == nil || !options.Retry(output, err) || attempt == options.Attempts-1 { + return output, err + } + options.Sleep(options.Delay) + } + return output, err +} diff --git a/internal/setup/commands_test.go b/internal/setup/commands_test.go new file mode 100644 index 0000000..a5e4ae9 --- /dev/null +++ b/internal/setup/commands_test.go @@ -0,0 +1,51 @@ +package setup + +import ( + "context" + "os" + "os/exec" + "reflect" + "testing" + "time" +) + +func TestCloudCommandsApplyProfileEnvironment(t *testing.T) { + gcloud := GCloudCommand(context.Background(), "work", "projects", "list") + aws := AWSCommand(context.Background(), "prod", "sts", "get-caller-identity") + if !contains(gcloud.Env, "CLOUDSDK_ACTIVE_CONFIG_NAME=work") { + t.Fatalf("gcloud env=%v", gcloud.Env) + } + if !contains(aws.Env, "AWS_PROFILE=prod") || !contains(aws.Env, "AWS_SDK_LOAD_CONFIG=1") { + t.Fatalf("aws env=%v", aws.Env) + } +} + +func TestRunWithRetryRebuildsCommands(t *testing.T) { + attempts := 0 + var sleeps []time.Duration + _, err := RunWithRetry(func() *exec.Cmd { + attempts++ + command := exec.Command(os.Args[0], "-test.run=TestSetupHelperProcess", "--") + command.Env = append(os.Environ(), "BGIT_SETUP_HELPER=1") + return command + }, RetryOptions{Attempts: 3, Delay: time.Second, Retry: func([]byte, error) bool { return true }, Sleep: func(delay time.Duration) { sleeps = append(sleeps, delay) }}) + if err == nil || attempts != 3 || !reflect.DeepEqual(sleeps, []time.Duration{time.Second, time.Second}) { + t.Fatalf("attempts=%d sleeps=%v err=%v", attempts, sleeps, err) + } +} + +func TestSetupHelperProcess(t *testing.T) { + if os.Getenv("BGIT_SETUP_HELPER") != "1" { + return + } + os.Exit(1) +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/internal/setup/deploy.go b/internal/setup/deploy.go new file mode 100644 index 0000000..9464937 --- /dev/null +++ b/internal/setup/deploy.go @@ -0,0 +1,63 @@ +package setup + +import ( + "context" + "os/exec" +) + +type GCPFunctionDeployment struct { + Configuration string + Name string + Region string + Source string + EntryPoint string + ServiceAccount string + Environment string + Public bool +} + +func GCPFunctionDeployCommand(ctx context.Context, deployment GCPFunctionDeployment) *exec.Cmd { + access := "--no-allow-unauthenticated" + if deployment.Public { + access = "--allow-unauthenticated" + } + return GCloudCommand(ctx, deployment.Configuration, + "functions", "deploy", deployment.Name, + "--gen2", + "--runtime", "nodejs22", + "--region", deployment.Region, + "--source", deployment.Source, + "--entry-point", deployment.EntryPoint, + "--trigger-http", + access, + "--service-account", deployment.ServiceAccount, + "--set-env-vars", deployment.Environment, + "--quiet", + ) +} + +type AWSStackDeployment struct { + Profile string + Region string + StackName string + TemplatePath string + ArtifactBucket string + ParameterOverrides []string + Capability string +} + +func AWSStackDeployCommand(ctx context.Context, deployment AWSStackDeployment) *exec.Cmd { + arguments := []string{ + "cloudformation", "deploy", + "--stack-name", deployment.StackName, + "--template-file", deployment.TemplatePath, + "--s3-bucket", deployment.ArtifactBucket, + "--capabilities", deployment.Capability, + "--region", deployment.Region, + } + if len(deployment.ParameterOverrides) > 0 { + arguments = append(arguments, "--parameter-overrides") + arguments = append(arguments, deployment.ParameterOverrides...) + } + return AWSCommand(ctx, deployment.Profile, arguments...) +} diff --git a/internal/setup/deploy_test.go b/internal/setup/deploy_test.go new file mode 100644 index 0000000..2cfca5d --- /dev/null +++ b/internal/setup/deploy_test.go @@ -0,0 +1,27 @@ +package setup + +import ( + "context" + "reflect" + "testing" +) + +func TestGCPFunctionDeployCommand(t *testing.T) { + command := GCPFunctionDeployCommand(context.Background(), GCPFunctionDeployment{ + Configuration: "work", Name: "bgit-broker", Region: "europe-west1", + Source: "/tmp/source", EntryPoint: "broker", ServiceAccount: "broker@example.iam.gserviceaccount.com", + Environment: "BROKER_VERSION=1.0.0", Public: true, + }) + want := []string{"gcloud", "functions", "deploy", "bgit-broker", "--gen2", "--runtime", "nodejs22", "--region", "europe-west1", "--source", "/tmp/source", "--entry-point", "broker", "--trigger-http", "--allow-unauthenticated", "--service-account", "broker@example.iam.gserviceaccount.com", "--set-env-vars", "BROKER_VERSION=1.0.0", "--quiet"} + if !reflect.DeepEqual(command.Args, want) { + t.Fatalf("args=%q\nwant=%q", command.Args, want) + } +} + +func TestAWSStackDeployCommand(t *testing.T) { + command := AWSStackDeployCommand(context.Background(), AWSStackDeployment{Profile: "work", Region: "eu-west-1", StackName: "bgit-broker", TemplatePath: "/tmp/template.yaml", ArtifactBucket: "artifacts", Capability: "CAPABILITY_NAMED_IAM", ParameterOverrides: []string{"OwnerBootstrapHash=hash"}}) + want := []string{"aws", "cloudformation", "deploy", "--stack-name", "bgit-broker", "--template-file", "/tmp/template.yaml", "--s3-bucket", "artifacts", "--capabilities", "CAPABILITY_NAMED_IAM", "--region", "eu-west-1", "--parameter-overrides", "OwnerBootstrapHash=hash"} + if !reflect.DeepEqual(command.Args, want) { + t.Fatalf("args=%q\nwant=%q", command.Args, want) + } +} diff --git a/internal/setup/profiles.go b/internal/setup/profiles.go new file mode 100644 index 0000000..7b741a6 --- /dev/null +++ b/internal/setup/profiles.go @@ -0,0 +1,221 @@ +package setup + +import ( + "bufio" + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" +) + +const defaultProbeTimeout = 10 * time.Second + +type Profile struct { + Provider string + Name string + Active bool + Existing bool + Account string + ProjectID string + AccountID string + ARN string + Region string + ConfiguredRegions []string +} + +type ProfileDiscoveryOptions struct { + LookPath func(string) (string, error) + Command func(context.Context, string, ...string) *exec.Cmd + HomeDir func() (string, error) + ProbeTimeout time.Duration +} + +func (o ProfileDiscoveryOptions) withDefaults() ProfileDiscoveryOptions { + if o.LookPath == nil { + o.LookPath = exec.LookPath + } + if o.Command == nil { + o.Command = exec.CommandContext + } + if o.HomeDir == nil { + o.HomeDir = os.UserHomeDir + } + if o.ProbeTimeout <= 0 { + o.ProbeTimeout = defaultProbeTimeout + } + return o +} + +func DiscoverProfiles(ctx context.Context, options ProfileDiscoveryOptions) ([]Profile, error) { + options = options.withDefaults() + gcp, err := DiscoverGCPProfiles(ctx, options) + if err != nil { + return nil, err + } + aws, err := DiscoverAWSProfiles(options) + if err != nil { + return nil, err + } + return append(gcp, aws...), nil +} + +func DiscoverGCPProfiles(ctx context.Context, options ProfileDiscoveryOptions) ([]Profile, error) { + options = options.withDefaults() + if _, err := options.LookPath("gcloud"); err != nil { + return nil, nil + } + probeCtx, cancel := context.WithTimeout(ctx, options.ProbeTimeout) + defer cancel() + out, err := options.Command(probeCtx, "gcloud", "config", "configurations", "list", "--format=value(name,is_active)").Output() + if err != nil { + return nil, nil + } + var profiles []Profile + scanner := bufio.NewScanner(bytes.NewReader(out)) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) == 0 { + continue + } + name := fields[0] + profiles = append(profiles, Profile{ + Provider: "gcs", + Name: name, + Active: len(fields) > 1 && strings.EqualFold(fields[1], "true"), + Account: GCloudConfigValue(ctx, name, "account", options), + ProjectID: GCloudConfigValue(ctx, name, "project", options), + Region: firstNonEmpty( + GCloudConfigValue(ctx, name, "run/region", options), + GCloudConfigValue(ctx, name, "functions/region", options), + "us-central1", + ), + }) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return profiles, nil +} + +func GCloudConfigValue(ctx context.Context, profile, key string, options ProfileDiscoveryOptions) string { + options = options.withDefaults() + probeCtx, cancel := context.WithTimeout(ctx, options.ProbeTimeout) + defer cancel() + out, err := options.Command(probeCtx, "gcloud", "--configuration", profile, "config", "get-value", key, "--quiet").Output() + if err != nil { + return "" + } + value := strings.TrimSpace(string(out)) + if value == "(unset)" { + return "" + } + return value +} + +func DiscoverAWSProfiles(options ProfileDiscoveryOptions) ([]Profile, error) { + options = options.withDefaults() + names := AWSProfilesFromFiles(options) + profiles := make([]Profile, 0, len(names)) + for _, name := range names { + profiles = append(profiles, Profile{Provider: "s3", Name: name, Region: ConfiguredAWSProfileRegion(name, options)}) + } + return profiles, nil +} + +func AWSProfilesFromFiles(options ProfileDiscoveryOptions) []string { + options = options.withDefaults() + home, err := options.HomeDir() + if err != nil { + return nil + } + names := map[string]struct{}{} + for _, path := range []string{filepath.Join(home, ".aws", "config"), filepath.Join(home, ".aws", "credentials")} { + for _, name := range ParseAWSProfileFile(path) { + names[name] = struct{}{} + } + } + return sortedKeys(names) +} + +func ParseAWSProfileFile(path string) []string { + data, err := os.ReadFile(path) + if err != nil { + return nil + } + names := map[string]struct{}{} + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "[") || !strings.HasSuffix(line, "]") { + continue + } + name := strings.TrimSpace(strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(line, "["), "]"), "profile ")) + if name != "" { + names[name] = struct{}{} + } + } + return sortedKeys(names) +} + +func ConfiguredAWSProfileRegion(profile string, options ProfileDiscoveryOptions) string { + options = options.withDefaults() + home, err := options.HomeDir() + if err != nil { + return "" + } + for _, path := range []string{filepath.Join(home, ".aws", "config"), filepath.Join(home, ".aws", "credentials")} { + if region := AWSProfileFileValue(path, profile, "region"); region != "" { + return region + } + } + return "" +} + +func AWSProfileFileValue(path, profile, key string) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + section := "" + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + section = strings.TrimSpace(strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(line, "["), "]"), "profile ")) + continue + } + if section != profile { + continue + } + name, value, ok := strings.Cut(line, "=") + if ok && strings.TrimSpace(name) == key { + return strings.TrimSpace(value) + } + } + return "" +} + +func sortedKeys(values map[string]struct{}) []string { + result := make([]string, 0, len(values)) + for value := range values { + result = append(result, value) + } + sort.Strings(result) + return result +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} diff --git a/internal/setup/profiles_test.go b/internal/setup/profiles_test.go new file mode 100644 index 0000000..0bb14d1 --- /dev/null +++ b/internal/setup/profiles_test.go @@ -0,0 +1,62 @@ +package setup + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "reflect" + "testing" +) + +func TestParseAWSProfileFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "config") + if err := os.WriteFile(path, []byte("[default]\nregion=us-east-1\n[profile work]\nregion=eu-west-1\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := ParseAWSProfileFile(path); !reflect.DeepEqual(got, []string{"default", "work"}) { + t.Fatalf("profiles = %#v", got) + } + if got := AWSProfileFileValue(path, "work", "region"); got != "eu-west-1" { + t.Fatalf("region = %q", got) + } +} + +func TestDiscoverGCPProfilesUsesInjectedCommands(t *testing.T) { + command := func(_ context.Context, _ string, args ...string) *exec.Cmd { + output := "" + switch { + case len(args) > 1 && args[0] == "config" && args[1] == "configurations": + output = "default true\n" + case len(args) > 4 && args[0] == "--configuration" && args[3] == "get-value": + switch args[4] { + case "account": + output = "owner@example.com\n" + case "project": + output = "demo-project\n" + case "run/region": + output = "europe-west1\n" + } + } + return exec.Command(os.Args[0], "-test.run=TestProfileCommandHelper", "--", output) + } + profiles, err := DiscoverGCPProfiles(context.Background(), ProfileDiscoveryOptions{ + LookPath: func(string) (string, error) { return "gcloud", nil }, + Command: command, + }) + if err != nil { + t.Fatal(err) + } + if len(profiles) != 1 || profiles[0].ProjectID != "demo-project" || profiles[0].Region != "europe-west1" { + t.Fatalf("profiles = %#v", profiles) + } +} + +func TestProfileCommandHelper(t *testing.T) { + for i, arg := range os.Args { + if arg == "--" && i+1 < len(os.Args) { + _, _ = os.Stdout.WriteString(os.Args[i+1]) + os.Exit(0) + } + } +} diff --git a/internal/web/assets.go b/internal/web/assets.go new file mode 100644 index 0000000..2ffff49 --- /dev/null +++ b/internal/web/assets.go @@ -0,0 +1,13 @@ +// Package web owns the embedded assets and presentation primitives for the +// local BucketGit web application. Repository operations are supplied by the +// application layer rather than imported from the CLI. +package web + +import "embed" + +//go:embed www/* +var assets embed.FS + +func ReadAsset(path string) ([]byte, error) { + return assets.ReadFile(path) +} diff --git a/internal/web/events.go b/internal/web/events.go new file mode 100644 index 0000000..40fee60 --- /dev/null +++ b/internal/web/events.go @@ -0,0 +1,55 @@ +package web + +import ( + "encoding/json" + "fmt" + "sync" + "time" +) + +type EventHub struct { + mu sync.Mutex + clients map[chan string]struct{} +} + +func NewEventHub() *EventHub { + return &EventHub{clients: map[chan string]struct{}{}} +} + +func (h *EventHub) Subscribe() chan string { + channel := make(chan string, 8) + h.mu.Lock() + h.clients[channel] = struct{}{} + h.mu.Unlock() + return channel +} + +func (h *EventHub) Unsubscribe(channel chan string) { + h.mu.Lock() + delete(h.clients, channel) + close(channel) + h.mu.Unlock() +} + +func (h *EventHub) Broadcast(name string) { + h.send(fmt.Sprintf("event: %s\ndata: {\"time\":%d}\n\n", name, time.Now().UnixMilli())) +} + +func (h *EventHub) BroadcastJSON(name string, value any) { + data, err := json.Marshal(value) + if err != nil { + return + } + h.send(fmt.Sprintf("event: %s\ndata: %s\n\n", name, data)) +} + +func (h *EventHub) send(payload string) { + h.mu.Lock() + defer h.mu.Unlock() + for channel := range h.clients { + select { + case channel <- payload: + default: + } + } +} diff --git a/internal/web/events_test.go b/internal/web/events_test.go new file mode 100644 index 0000000..6ddc582 --- /dev/null +++ b/internal/web/events_test.go @@ -0,0 +1,19 @@ +package web + +import ( + "strings" + "testing" +) + +func TestEventHubBroadcastsAndUnsubscribes(t *testing.T) { + hub := NewEventHub() + channel := hub.Subscribe() + hub.BroadcastJSON("state", map[string]bool{"dirty": true}) + if event := <-channel; !strings.Contains(event, "event: state") || !strings.Contains(event, `"dirty":true`) { + t.Fatalf("event=%q", event) + } + hub.Unsubscribe(channel) + if _, ok := <-channel; ok { + t.Fatal("unsubscribed channel remains open") + } +} diff --git a/internal/web/http.go b/internal/web/http.go new file mode 100644 index 0000000..c2f72f8 --- /dev/null +++ b/internal/web/http.go @@ -0,0 +1,67 @@ +package web + +import ( + "crypto/rand" + "encoding/base64" + "errors" + "mime" + "net" + "net/http" + "net/url" + "path/filepath" + "strings" +) + +func NewCSRFToken() (string, error) { + var data [32]byte + if _, err := rand.Read(data[:]); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(data[:]), nil +} + +func IsMutationRoute(route, method string) bool { + if method == http.MethodGet || method == http.MethodHead || method == http.MethodOptions { + return false + } + return strings.HasPrefix(route, "api/actions/") || route == "api/user/profile" +} + +// ValidateLocalMutation verifies the per-process CSRF token and rejects +// browser mutations originating outside the loopback web application. +func ValidateLocalMutation(token string, request *http.Request) error { + if request.Header.Get("X-Bgit-CSRF") != token { + return errors.New("invalid CSRF token") + } + origin := strings.TrimSpace(request.Header.Get("Origin")) + if origin == "" { + return nil + } + parsed, err := url.Parse(origin) + if err != nil { + return errors.New("invalid origin") + } + host, _, err := net.SplitHostPort(parsed.Host) + if err != nil { + host = parsed.Hostname() + } + host = strings.ToLower(strings.Trim(host, "[]")) + if parsed.Scheme != "http" || (host != "localhost" && host != "127.0.0.1" && host != "::1") { + return errors.New("foreign origin rejected") + } + return nil +} + +func ServeAsset(response http.ResponseWriter, path string) { + data, err := ReadAsset(path) + if err != nil { + http.Error(response, "not found", http.StatusNotFound) + return + } + if contentType := mime.TypeByExtension(filepath.Ext(path)); contentType != "" { + response.Header().Set("Content-Type", contentType) + } + response.Header().Set("Cache-Control", "public, max-age=86400") + response.WriteHeader(http.StatusOK) + _, _ = response.Write(data) +} diff --git a/internal/web/http_test.go b/internal/web/http_test.go new file mode 100644 index 0000000..2b3e701 --- /dev/null +++ b/internal/web/http_test.go @@ -0,0 +1,33 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestValidateLocalMutation(t *testing.T) { + tests := []struct { + name string + token string + origin string + ok bool + }{ + {"localhost", "secret", "http://localhost:8042", true}, + {"ipv4", "secret", "http://127.0.0.1:8042", true}, + {"ipv6", "secret", "http://[::1]:8042", true}, + {"foreign", "secret", "https://attacker.example", false}, + {"bad token", "wrong", "http://localhost:8042", false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "http://localhost/api/actions/push", nil) + request.Header.Set("X-Bgit-CSRF", test.token) + request.Header.Set("Origin", test.origin) + err := ValidateLocalMutation("secret", request) + if (err == nil) != test.ok { + t.Fatalf("error=%v ok=%t", err, test.ok) + } + }) + } +} diff --git a/internal/web/routes.go b/internal/web/routes.go new file mode 100644 index 0000000..8d5581a --- /dev/null +++ b/internal/web/routes.go @@ -0,0 +1,98 @@ +package web + +import "strings" + +type Route string + +const ( + RouteUnknown Route = "" + RouteHome Route = "home" + RouteLogo Route = "logo" + RouteFavicon Route = "favicon" + RouteEvents Route = "events" + RouteAPIState Route = "api-state" + RouteAPIMe Route = "api-me" + RouteAPICommitAction Route = "api-action-commit" + RouteAPIStageAction Route = "api-action-stage" + RouteAPIUnstageAction Route = "api-action-unstage" + RouteAPIDiscardAction Route = "api-action-discard" + RouteAPIUncommitAction Route = "api-action-uncommit" + RouteAPIPushAction Route = "api-action-push" + RouteAPIPullAction Route = "api-action-pull" + RouteAPIPRAction Route = "api-action-pr" + RouteAPIIssueAction Route = "api-action-issues" + RouteAPIBoardAction Route = "api-action-board" + RouteAPICIAction Route = "api-action-ci" + RouteAPISettingsAction Route = "api-action-settings" + RouteAPIUserProfile Route = "api-user-profile" + RouteAPIDiff Route = "api-diff" + RouteAPIRefs Route = "api-refs" + RouteAPITree Route = "api-tree" + RouteAPICommits Route = "api-commits" + RouteAPIPRs Route = "api-prs" + RouteAPIIssues Route = "api-issues" + RouteAPISettings Route = "api-settings" + RouteAPISettingsFragment Route = "api-settings-fragment" + RouteAPIBlob Route = "api-blob" + RouteAPICommit Route = "api-commit" + RouteCommits Route = "commits" + RoutePRs Route = "prs" + RouteNewPR Route = "new-pr" + RoutePR Route = "pr" + RouteIssues Route = "issues" + RouteIssue Route = "issue" + RouteBoard Route = "board" + RouteStory Route = "story" + RouteCI Route = "ci" + RouteSettings Route = "settings" + RouteUserSettings Route = "user-settings" + RouteAdmin Route = "admin" + RouteArchive Route = "archive" + RouteCommit Route = "commit" + RouteTree Route = "tree" + RouteBlob Route = "blob" + RouteRaw Route = "raw" +) + +type Match struct { + Route Route + Value string +} + +func MatchRoute(path string) Match { + route := strings.TrimPrefix(path, "/") + exact := map[string]Route{ + "": RouteHome, "assets/bgit-mark.png": RouteLogo, "favicon.ico": RouteFavicon, + "events": RouteEvents, "api/state": RouteAPIState, "api/me": RouteAPIMe, + "api/actions/commit": RouteAPICommitAction, "api/actions/stage": RouteAPIStageAction, + "api/actions/unstage": RouteAPIUnstageAction, "api/actions/discard": RouteAPIDiscardAction, + "api/actions/uncommit": RouteAPIUncommitAction, "api/actions/push": RouteAPIPushAction, + "api/actions/pull": RouteAPIPullAction, "api/actions/pr": RouteAPIPRAction, + "api/actions/issues": RouteAPIIssueAction, "api/actions/board": RouteAPIBoardAction, + "api/actions/ci": RouteAPICIAction, "api/actions/settings": RouteAPISettingsAction, + "api/user/profile": RouteAPIUserProfile, "api/diff": RouteAPIDiff, "api/refs": RouteAPIRefs, + "api/tree": RouteAPITree, "api/commits": RouteAPICommits, "api/prs": RouteAPIPRs, + "api/issues": RouteAPIIssues, "api/settings": RouteAPISettings, + "api/settings-fragment": RouteAPISettingsFragment, "api/blob": RouteAPIBlob, + "commits": RouteCommits, "prs": RoutePRs, "prs/new": RouteNewPR, "issues": RouteIssues, + "board": RouteBoard, "ci": RouteCI, "settings": RouteSettings, + "user/settings": RouteUserSettings, "user/settings/": RouteUserSettings, + "admin": RouteAdmin, "archive.zip": RouteArchive, + } + if matched, ok := exact[route]; ok { + return Match{Route: matched} + } + for _, prefix := range []struct { + Value string + Route Route + }{ + {"api/commit/", RouteAPICommit}, {"prs/", RoutePR}, {"issues/", RouteIssue}, + {"board/", RouteStory}, {"commit/", RouteCommit}, {"tree/", RouteTree}, + {"blob/", RouteBlob}, {"raw/", RouteRaw}, + } { + if strings.HasPrefix(route, prefix.Value) { + return Match{Route: prefix.Route, Value: strings.TrimPrefix(route, prefix.Value)} + } + } + return Match{} +} diff --git a/internal/web/routes_test.go b/internal/web/routes_test.go new file mode 100644 index 0000000..7bc6629 --- /dev/null +++ b/internal/web/routes_test.go @@ -0,0 +1,24 @@ +package web + +import "testing" + +func TestMatchRoutePrecedenceAndValues(t *testing.T) { + tests := []struct { + path string + route Route + value string + }{ + {"/", RouteHome, ""}, + {"/prs/new", RouteNewPR, ""}, + {"/prs/42", RoutePR, "42"}, + {"/api/commit/abc", RouteAPICommit, "abc"}, + {"/raw/docs/readme.md", RouteRaw, "docs/readme.md"}, + {"/missing", RouteUnknown, ""}, + } + for _, test := range tests { + match := MatchRoute(test.path) + if match.Route != test.route || match.Value != test.value { + t.Fatalf("MatchRoute(%q)=%#v want route=%q value=%q", test.path, match, test.route, test.value) + } + } +} diff --git a/www/app.css b/internal/web/www/app.css similarity index 100% rename from www/app.css rename to internal/web/www/app.css diff --git a/www/app.js b/internal/web/www/app.js similarity index 100% rename from www/app.js rename to internal/web/www/app.js diff --git a/www/bgit-mark.png b/internal/web/www/bgit-mark.png similarity index 100% rename from www/bgit-mark.png rename to internal/web/www/bgit-mark.png diff --git a/www/favicon.ico b/internal/web/www/favicon.ico similarity index 100% rename from www/favicon.ico rename to internal/web/www/favicon.ico diff --git a/www/page.html b/internal/web/www/page.html similarity index 100% rename from www/page.html rename to internal/web/www/page.html diff --git a/local_broker_native.go b/local_broker_native.go deleted file mode 100644 index 01068e1..0000000 --- a/local_broker_native.go +++ /dev/null @@ -1,1725 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "net/http" - "net/url" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "golang.org/x/crypto/ssh" -) - -type localBrokerServer struct { - root string - objectRoot string - baseURL string - bootstrapHash string - mu sync.Mutex -} - -type localBrokerRepoState struct { - Repo brokerRepo `json:"repo"` - Keys []brokerKey `json:"keys"` - Refs map[string]string `json:"refs,omitempty"` - Teams []brokerRepoTeamGrant `json:"teams,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` -} - -type localBrokerOwners struct { - Keys []brokerKey `json:"keys"` -} - -type localBrokerRepoIndex struct { - Repos []brokerRepo `json:"repos"` -} - -type localBrokerIssueStore struct { - NextID int `json:"next_id"` - Issues []brokerIssue `json:"issues"` -} - -func isLocalBrokerURL(value string) bool { - return strings.HasPrefix(strings.TrimSpace(value), "local://") -} - -func localBrokerPostContext(ctx context.Context, brokerURL, path string, req any, resp any) error { - _ = ctx - server, err := localBrokerServerForURL(brokerURL) - if err != nil { - return err - } - out, status, err := server.localPost(path, req) - if err != nil { - return err - } - if status < 200 || status >= 300 { - msg := strings.TrimSpace(string(out)) - if msg == "" { - msg = http.StatusText(status) - } - return brokerHTTPError(path, msg) - } - if resp != nil && len(out) > 0 { - if err := json.Unmarshal(out, resp); err != nil { - return err - } - } - return nil -} - -func localBrokerCapabilityRead(ctx context.Context, capability brokerObjectCapabilityResponse) ([]byte, error) { - _ = ctx - server, repo, err := localBrokerCapabilityTarget(capability) - if err != nil { - return nil, err - } - return server.readObject(repo, capability.Object) -} - -func localBrokerCapabilityWrite(ctx context.Context, capability brokerObjectCapabilityResponse, data []byte) error { - _ = ctx - server, repo, err := localBrokerCapabilityTarget(capability) - if err != nil { - return err - } - return server.writeObject(repo, capability.Object, data) -} - -func localBrokerCapabilityDelete(ctx context.Context, capability brokerObjectCapabilityResponse) error { - _ = ctx - server, repo, err := localBrokerCapabilityTarget(capability) - if err != nil { - return err - } - return server.deleteObject(repo, capability.Object) -} - -func localBrokerCapabilityTarget(capability brokerObjectCapabilityResponse) (*localBrokerServer, brokerRepo, error) { - server, err := localBrokerServerForURL(capability.URL) - if err != nil { - return nil, brokerRepo{}, err - } - repo := brokerRepo{Provider: capability.Provider, Bucket: capability.Bucket, Prefix: capability.Prefix, Profile: capability.Profile, Region: capability.Region} - return server, repo, nil -} - -func localBrokerServerForURL(brokerURL string) (*localBrokerServer, error) { - profileName, regionName := localProfileSelection(strings.TrimPrefix(strings.TrimSpace(brokerURL), "local://")) - path, err := defaultGlobalConfigPath() - if err != nil { - return nil, err - } - global, err := readGlobalConfig(path) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } - profile, _, ok := globalLocalProfileForSelection(global, profileName, regionName) - if !ok { - profile = ensureDefaultLocalProfile(global) - } - root := expandHome(profile.Root) - if strings.TrimSpace(root) == "" { - root, err = defaultLocalBrokerRoot() - if err != nil { - return nil, err - } - } - server := &localBrokerServer{root: root, objectRoot: filepath.Join(root, "objects"), baseURL: localBrokerURL(profileName, regionName)} - if err := os.MkdirAll(server.objectRoot, 0o755); err != nil { - return nil, err - } - return server, nil -} - -func (s *localBrokerServer) localPost(path string, req any) ([]byte, int, error) { - switch path { - case "/owners/upsert": - r := req.(brokerOwnerRequest) - owners, _ := s.loadOwners() - for _, key := range r.PublicKeys { - key = normalizeKey(key) - if key == "" { - continue - } - found := false - for i := range owners.Keys { - if normalizeKey(owners.Keys[i].PublicKey) == key { - owners.Keys[i].User = firstNonEmpty(r.User, "owner") - owners.Keys[i].Role = firstNonEmpty(r.Role, "owner") - found = true - } - } - if !found { - owners.Keys = append(owners.Keys, brokerKey{User: firstNonEmpty(r.User, "owner"), Role: firstNonEmpty(r.Role, "owner"), PublicKey: key}) - } - } - if err := s.saveJSON(filepath.Join(s.root, "owners.json"), owners); err != nil { - return nil, 500, err - } - return mustJSON(map[string]bool{"ok": true}), 200, nil - case "/teams/create": - return mustJSON(map[string]bool{"ok": true}), 200, nil - case "/teams/list": - return mustJSON(map[string]any{"teams": []brokerTeamInfo{{ID: coreTeamID, Name: coreTeamName}}}), 200, nil - case "/teams/resolve": - return mustJSON(map[string]any{"team": brokerTeamInfo{ID: coreTeamID, Name: coreTeamName}}), 200, nil - case "/repos/create": - r := req.(brokerRepoAdminRequest) - repo := s.physicalRepo(r.Repo) - if state, err := s.loadRepo(repo); err == nil && state.Repo.Logical != "" { - return mustJSON(map[string]string{"error": "repository already exists"}), http.StatusConflict, nil - } - if state, err := s.loadRepoForRequest(repo); err == nil && state.Repo.Logical != "" { - return mustJSON(map[string]string{"error": "repository already exists"}), http.StatusConflict, nil - } - if err := s.ensureRepoStorage(context.Background(), repo); err != nil { - return nil, 500, err - } - now := time.Now().UTC().Format(time.RFC3339) - state := localBrokerRepoState{Repo: repo, Refs: map[string]string{}, Teams: []brokerRepoTeamGrant{{ID: coreTeamID, Role: firstNonEmpty(r.Role, "developer")}}, CreatedAt: now, UpdatedAt: now} - for _, key := range r.PublicKeys { - if key = normalizeKey(key); key != "" { - state.Keys = append(state.Keys, brokerKey{User: firstNonEmpty(r.User, "owner"), Role: firstNonEmpty(r.Role, "developer"), PublicKey: key}) - } - } - if len(state.Keys) == 0 { - owners, _ := s.loadOwners() - for _, key := range owners.Keys { - state.Keys = append(state.Keys, brokerKey{User: key.User, Role: firstNonEmpty(r.Role, "developer"), PublicKey: key.PublicKey}) - } - } - if err := s.saveRepo(state); err != nil { - return nil, 500, err - } - return mustJSON(map[string]any{"ok": true, "repo": state.Repo}), 200, nil - case "/repos/get": - r := req.(brokerRepoRequest) - state, err := s.loadRepoForRequest(r.Repo) - if err != nil { - return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil - } - return mustJSON(map[string]any{"ok": true, "repo": state.Repo, "teams": state.Teams}), 200, nil - case "/repos/list": - repos := s.localRepoInfos() - return mustJSON(map[string]any{"repos": repos}), 200, nil - case "/auth/status": - r := req.(brokerAuthStatusRequest) - state, _ := s.loadRepoForRequest(r.Repo) - return mustJSON(brokerAuthStatus{BrokerURL: s.baseURL, BrokerVersion: brokerVersion(), Repo: state.Repo, User: "owner", Role: "owner", Capabilities: roleCapabilitiesForLocal("owner"), ResolvedAt: time.Now().UTC().Format(time.RFC3339)}), 200, nil - case "/auth/check": - return mustJSON(brokerAuthResponse{Allowed: true, User: "owner", Role: "owner"}), 200, nil - case "/objects/capability": - r := req.(brokerObjectCapabilityRequest) - state, err := s.loadRepoForRequest(r.Repo) - if err != nil { - return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil - } - objectPath, err := validateLocalBrokerCapabilityPath(r.Operation, r.Path) - if err != nil { - return mustJSON(map[string]string{"error": err.Error()}), http.StatusForbidden, nil - } - object := localBrokerObjectName(state.Repo, objectPath) - return mustJSON(brokerObjectCapabilityResponse{Provider: state.Repo.Provider, Mode: "local", URL: s.baseURL, Bucket: state.Repo.Bucket, Prefix: state.Repo.Prefix, Object: object, Profile: state.Repo.Profile, Region: state.Repo.Region}), 200, nil - case "/objects/read": - r := req.(brokerObjectRequest) - state, err := s.loadRepoForRequest(r.Repo) - if err != nil { - return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil - } - data, err := s.readObject(state.Repo, r.Path) - if err != nil { - return mustJSON(map[string]string{"error": err.Error()}), http.StatusNotFound, nil - } - return mustJSON(brokerObjectResponse{Data: base64.StdEncoding.EncodeToString(data)}), 200, nil - case "/objects/list": - r := req.(brokerObjectRequest) - state, err := s.loadRepoForRequest(r.Repo) - if err != nil { - return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil - } - paths, err := s.listObjects(state.Repo, r.Prefix) - if err != nil { - return nil, 500, err - } - return mustJSON(brokerObjectResponse{Paths: paths}), 200, nil - case "/issues/list", "/issues/view", "/issues/create", "/issues/update", "/issues/comment", "/issues/close", "/issues/reopen", "/issues/move", "/issues/take", "/issues/assign", "/issues/archive", "/issues/reorder": - r := req.(brokerIssueRequest) - state, err := s.loadRepoForRequest(r.Repo) - if err != nil { - return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil - } - return s.localIssueEndpoint(path, r, state, "owner") - case "/refs/update": - r := req.(brokerRefUpdateRequest) - state, err := s.loadRepoForRequest(r.Repo) - if err != nil { - return mustJSON(map[string]string{"error": "repository not found"}), http.StatusNotFound, nil - } - current := s.localRefHash(state.Repo, r.Ref, firstNonEmpty(state.Refs[r.Ref], r.Old)) - if current != r.Old { - return mustJSON(map[string]string{"error": "stale ref"}), http.StatusConflict, nil - } - if state.Refs == nil { - state.Refs = map[string]string{} - } - if r.New == zeroObjectID() { - delete(state.Refs, r.Ref) - _ = s.deleteObject(state.Repo, r.Ref) - _ = s.deleteObject(state.Repo, localBrokerRefRecordPath(r.Ref)) - } else { - state.Refs[r.Ref] = r.New - _ = s.writeObject(state.Repo, r.Ref, []byte(r.New+"\n")) - record := map[string]any{"ref": r.Ref, "hash": r.New, "updated_by": "owner", "updated_at": time.Now().UTC().Format(time.RFC3339)} - _ = s.writeObject(state.Repo, localBrokerRefRecordPath(r.Ref), mustJSON(record)) - } - state.UpdatedAt = time.Now().UTC().Format(time.RFC3339) - if err := s.saveRepo(state); err != nil { - return nil, 500, err - } - return mustJSON(map[string]bool{"ok": true}), 200, nil - default: - return mustJSON(map[string]string{"error": "unknown broker endpoint"}), http.StatusNotFound, nil - } -} - -func (s *localBrokerServer) handle(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/_objects/") { - s.handleObject(w, r) - return - } - if r.Method != http.MethodPost && r.URL.Path != "/health" && r.URL.Path != "/" { - localBrokerJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) - return - } - var body []byte - if r.Body != nil { - body, _ = io.ReadAll(r.Body) - } - var req map[string]any - if len(bytes.TrimSpace(body)) > 0 { - _ = json.Unmarshal(body, &req) - } - switch r.URL.Path { - case "/", "/health": - localBrokerJSON(w, 200, map[string]any{"ok": true, "service": "bgit-local-broker", "version": brokerVersion()}) - case "/owners/upsert": - s.handleOwnersUpsert(w, r, body) - case "/teams/create": - localBrokerJSON(w, 200, map[string]bool{"ok": true}) - case "/teams/list": - localBrokerJSON(w, 200, map[string]any{"teams": []brokerTeamInfo{{ID: coreTeamID, Name: coreTeamName}}}) - case "/teams/resolve": - localBrokerJSON(w, 200, map[string]any{"team": brokerTeamInfo{ID: coreTeamID, Name: coreTeamName}}) - case "/repos/create": - s.handleReposCreate(w, r, body) - case "/repos/get": - s.handleReposGet(w, r, body) - case "/repos/list": - s.handleReposList(w, r) - case "/auth/status": - s.handleAuthStatus(w, r, body) - case "/auth/check": - s.handleAuthCheck(w, r, body) - case "/objects/capability": - s.handleObjectCapability(w, r, body) - case "/objects/read": - s.handleObjectsRead(w, r, body) - case "/objects/list": - s.handleObjectsList(w, r, body) - case "/issues/list", "/issues/view", "/issues/create", "/issues/update", "/issues/comment", "/issues/close", "/issues/reopen", "/issues/move", "/issues/take", "/issues/assign", "/issues/archive", "/issues/reorder": - s.handleIssues(w, r, body) - case "/refs/update": - s.handleRefsUpdate(w, r, body) - default: - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "unknown broker endpoint"}) - } -} - -func (s *localBrokerServer) handleOwnersUpsert(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerOwnerRequest - if !localBrokerDecode(w, body, &req) { - return - } - owners, _ := s.loadOwners() - if len(owners.Keys) > 0 { - if _, ok := s.signedOwner(r, body, owners); !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "owner SSH signature required"}) - return - } - } else if !s.validBootstrap(r) { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "owner bootstrap token required"}) - return - } - for _, key := range req.PublicKeys { - key = normalizeKey(key) - if key == "" { - continue - } - found := false - for i := range owners.Keys { - if normalizeKey(owners.Keys[i].PublicKey) == key { - owners.Keys[i].User = firstNonEmpty(req.User, "owner") - owners.Keys[i].Role = firstNonEmpty(req.Role, "owner") - found = true - } - } - if !found { - owners.Keys = append(owners.Keys, brokerKey{User: firstNonEmpty(req.User, "owner"), Role: firstNonEmpty(req.Role, "owner"), PublicKey: key}) - } - } - if err := s.saveJSON(filepath.Join(s.root, "owners.json"), owners); err != nil { - localBrokerJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - localBrokerJSON(w, 200, map[string]bool{"ok": true}) -} - -func (s *localBrokerServer) handleReposCreate(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerRepoAdminRequest - if !localBrokerDecode(w, body, &req) { - return - } - if _, ok := s.requireOwner(r, body); !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "broker admin SSH signature required"}) - return - } - repo := s.physicalRepo(req.Repo) - state, err := s.loadRepo(repo) - if err == nil && state.Repo.Logical != "" { - localBrokerJSON(w, http.StatusConflict, map[string]string{"error": "repository already exists"}) - return - } - if state, err := s.loadRepoForRequest(repo); err == nil && state.Repo.Logical != "" { - localBrokerJSON(w, http.StatusConflict, map[string]string{"error": "repository already exists"}) - return - } - if err := s.ensureRepoStorage(r.Context(), repo); err != nil { - localBrokerJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - now := time.Now().UTC().Format(time.RFC3339) - state = localBrokerRepoState{Repo: repo, Refs: map[string]string{}, Teams: []brokerRepoTeamGrant{{ID: coreTeamID, Role: firstNonEmpty(req.Role, "developer")}}, CreatedAt: now, UpdatedAt: now} - for _, key := range req.PublicKeys { - if key = normalizeKey(key); key != "" { - state.Keys = append(state.Keys, brokerKey{User: firstNonEmpty(req.User, "owner"), Role: firstNonEmpty(req.Role, "developer"), PublicKey: key}) - } - } - if err := s.saveRepo(state); err != nil { - localBrokerJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - localBrokerJSON(w, 200, map[string]any{"ok": true, "repo": state.Repo}) -} - -func (s *localBrokerServer) handleReposGet(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerRepoRequest - if !localBrokerDecode(w, body, &req) { - return - } - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - if _, ok := s.signedRepoKey(r, body, state, "read"); !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "read SSH signature required"}) - return - } - localBrokerJSON(w, 200, map[string]any{"ok": true, "repo": state.Repo, "teams": state.Teams}) -} - -func (s *localBrokerServer) handleReposList(w http.ResponseWriter, r *http.Request) { - localBrokerJSON(w, 200, map[string]any{"repos": s.localRepoInfos()}) -} - -func (s *localBrokerServer) localRepoInfos() []brokerRepoInfo { - repos := []brokerRepoInfo{} - seen := map[string]bool{} - for _, repo := range s.loadRepoIndex().Repos { - state, err := s.loadRepo(repo) - if err == nil && state.Repo.Logical != "" { - repos = append(repos, brokerRepoInfo{Repo: state.Repo, Logical: state.Repo.Logical, Teams: state.Teams}) - seen[strings.ToLower(state.Repo.Logical)] = true - } - } - _ = filepath.WalkDir(s.objectRoot, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() || filepath.Base(path) != "repo.json" { - return nil - } - var state localBrokerRepoState - if s.readJSON(path, &state) == nil && state.Repo.Logical != "" && !seen[strings.ToLower(state.Repo.Logical)] { - repos = append(repos, brokerRepoInfo{Repo: state.Repo, Logical: state.Repo.Logical, Teams: state.Teams}) - seen[strings.ToLower(state.Repo.Logical)] = true - } - return nil - }) - sort.Slice(repos, func(i, j int) bool { return repos[i].Logical < repos[j].Logical }) - return repos -} - -func (s *localBrokerServer) handleAuthStatus(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerAuthStatusRequest - _ = json.Unmarshal(body, &req) - state, _ := s.loadRepoForRequest(req.Repo) - key, _ := s.signedRepoKey(r, body, state, "read") - role := "" - user := "" - if key != nil { - role = key.Role - user = key.User - } - localBrokerJSON(w, 200, brokerAuthStatus{BrokerURL: s.baseURL, BrokerVersion: brokerVersion(), Repo: state.Repo, User: user, Role: role, Capabilities: roleCapabilitiesForLocal(role), ResolvedAt: time.Now().UTC().Format(time.RFC3339)}) -} - -func (s *localBrokerServer) handleAuthCheck(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerAuthRequest - if !localBrokerDecode(w, body, &req) { - return - } - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - key, ok := s.signedRepoKey(r, body, state, localBrokerOperation(req.Operation)) - if !ok { - localBrokerJSON(w, 200, brokerAuthResponse{Allowed: false}) - return - } - localBrokerJSON(w, 200, brokerAuthResponse{Allowed: true, User: key.User, Role: key.Role}) -} - -func (s *localBrokerServer) handleObjectCapability(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerObjectCapabilityRequest - if !localBrokerDecode(w, body, &req) { - return - } - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - op := req.Operation - if op != "read" { - op = "write" - } - if _, ok := s.signedRepoKey(r, body, state, op); !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": op + " SSH signature required"}) - return - } - objectPath, err := validateLocalBrokerCapabilityPath(req.Operation, req.Path) - if err != nil { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": err.Error()}) - return - } - method := http.MethodGet - if req.Operation == "write" { - method = http.MethodPut - } else if req.Operation == "delete" { - method = http.MethodDelete - } - object := localBrokerObjectName(state.Repo, objectPath) - localBrokerJSON(w, 200, brokerObjectCapabilityResponse{ - Provider: "test", - Mode: "signed_url", - Method: method, - URL: s.baseURL + "/_objects/" + url.PathEscape(state.Repo.Bucket) + "/" + base64.RawURLEncoding.EncodeToString([]byte(object)), - Headers: map[string]string{"content-type": "application/octet-stream"}, - Bucket: state.Repo.Bucket, - Prefix: state.Repo.Prefix, - Object: object, - Profile: state.Repo.Profile, - Region: state.Repo.Region, - }) -} - -func (s *localBrokerServer) handleObjectsRead(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerObjectRequest - if !localBrokerDecode(w, body, &req) { - return - } - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - if _, ok := s.signedRepoKey(r, body, state, "read"); !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "read SSH signature required"}) - return - } - data, err := s.readObject(state.Repo, req.Path) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()}) - return - } - localBrokerJSON(w, 200, brokerObjectResponse{Data: base64.StdEncoding.EncodeToString(data)}) -} - -func (s *localBrokerServer) handleObjectsList(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerObjectRequest - if !localBrokerDecode(w, body, &req) { - return - } - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - if _, ok := s.signedRepoKey(r, body, state, "read"); !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "read SSH signature required"}) - return - } - paths, err := s.listObjects(state.Repo, req.Prefix) - if err != nil { - localBrokerJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - localBrokerJSON(w, 200, brokerObjectResponse{Paths: paths}) -} - -func (s *localBrokerServer) handleIssues(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerIssueRequest - if !localBrokerDecode(w, body, &req) { - return - } - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - operation := "write" - if r.URL.Path == "/issues/list" || r.URL.Path == "/issues/view" { - operation = "read" - } - key, ok := s.signedRepoKey(r, body, state, operation) - if !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": operation + " SSH signature required"}) - return - } - s.mu.Lock() - defer s.mu.Unlock() - data, status, err := s.localIssueEndpoint(r.URL.Path, req, state, firstNonEmpty(key.User, "owner")) - if err != nil { - localBrokerJSON(w, status, map[string]string{"error": err.Error()}) - return - } - if len(data) == 0 { - localBrokerJSON(w, status, map[string]bool{"ok": true}) - return - } - w.Header().Set("content-type", "application/json") - w.WriteHeader(status) - _, _ = w.Write(data) -} - -func (s *localBrokerServer) localIssueEndpoint(path string, req brokerIssueRequest, state localBrokerRepoState, user string) ([]byte, int, error) { - store, err := s.loadIssueStore(state.Repo) - if err != nil { - return nil, 500, err - } - user = firstNonEmpty(strings.TrimSpace(user), "owner") - switch path { - case "/issues/list": - issues := make([]brokerIssue, 0, len(store.Issues)) - for _, issue := range store.Issues { - if req.Type != "" && issue.Type != req.Type { - continue - } - if issue.Archived && !req.IncludeArchived { - continue - } - issues = append(issues, issue) - } - sortBoardStories(issues) - return mustJSON(map[string]any{"issues": issues}), 200, nil - case "/issues/view": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - return mustJSON(map[string]any{"issue": *issue}), 200, nil - case "/issues/create": - title := strings.TrimSpace(req.Title) - body := strings.TrimSpace(req.Body) - if title == "" { - title = storySummary(body) - } - if title == "" { - return nil, http.StatusBadRequest, errors.New("issue title is required") - } - now := time.Now().UTC().Format(time.RFC3339) - if store.NextID <= 0 { - store.NextID = localBrokerNextIssueID(store.Issues) - } - issueType := firstNonEmpty(strings.TrimSpace(req.Type), "issue") - lane := strings.TrimSpace(req.Lane) - position := 0.0 - if issueType == "story" { - lane = normalizeKanbanLane(lane) - position = localBrokerNextStoryPosition(store.Issues, lane) - } - issue := brokerIssue{ - ID: store.NextID, - Type: issueType, - Title: title, - Body: body, - Status: "open", - Lane: lane, - Assignee: strings.TrimSpace(req.Assignee), - Position: position, - Author: user, - CreatedAt: now, - UpdatedAt: now, - History: []brokerIssueEvent{{User: user, Action: "created", At: now}}, - } - store.NextID++ - store.Issues = append(store.Issues, issue) - if issueType == "story" { - localBrokerNormalizeLaneOrder(&store, lane, 0) - if created, err := localBrokerFindIssue(&store, issue.ID); err == nil { - issue = *created - } - } - if err := s.saveIssueStore(state.Repo, store); err != nil { - return nil, 500, err - } - return mustJSON(map[string]any{"issue": issue}), 200, nil - case "/issues/update": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - if req.Type != "" { - issue.Type = req.Type - } - if strings.TrimSpace(req.Title) != "" { - issue.Title = strings.TrimSpace(req.Title) - } - if req.Body != "" { - issue.Body = strings.TrimSpace(req.Body) - } - if issue.Title == "" && issue.Type == "story" { - issue.Title = storySummary(issue.Body) - } - localBrokerIssueEvent(issue, user, "edited", "", "", "") - case "/issues/comment": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - comment := strings.TrimSpace(req.Comment) - if comment == "" { - return nil, http.StatusBadRequest, errors.New("comment is required") - } - now := time.Now().UTC().Format(time.RFC3339) - issue.Comments = append(issue.Comments, brokerIssueReply{User: user, Body: comment, At: now}) - localBrokerIssueEvent(issue, user, "commented", "", "", "") - case "/issues/close", "/issues/reopen": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - status := "closed" - action := "closed" - if path == "/issues/reopen" { - status = "open" - action = "reopened" - } - if issue.Status != status { - issue.Status = status - localBrokerIssueEvent(issue, user, action, "", "", "") - } - case "/issues/move", "/issues/reorder": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - lane, err := parseKanbanLane(firstNonEmpty(req.Lane, issue.Lane)) - if err != nil { - return nil, http.StatusBadRequest, err - } - from := normalizeKanbanLane(issue.Lane) - order := localBrokerStoryOrderFromRequest(req.Order) - if path == "/issues/reorder" { - localBrokerApplyStoryOrder(&store, issue.ID, lane, req.AfterID, order) - if from != lane { - localBrokerNormalizeLaneOrder(&store, from, issue.ID) - } - } else if req.AfterID != nil { - localBrokerApplyStoryOrder(&store, issue.ID, lane, req.AfterID, 0) - if from != lane { - localBrokerNormalizeLaneOrder(&store, from, issue.ID) - } - } else { - issue.Lane = lane - issue.Position = localBrokerNextStoryPosition(store.Issues, lane) - if from != lane { - localBrokerNormalizeLaneOrder(&store, from, issue.ID) - } - localBrokerNormalizeLaneOrder(&store, lane, 0) - } - issue, _ = localBrokerFindIssue(&store, req.ID) - action := "moved" - if path == "/issues/reorder" { - action = "reordered" - } - localBrokerIssueEvent(issue, user, action, from, lane, fmt.Sprintf("%.6f", issue.Position)) - case "/issues/take": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - fromAssignee := issue.Assignee - fromLane := normalizeKanbanLane(issue.Lane) - issue.Assignee = user - lane := normalizeKanbanLane(issue.Lane) - if lane == "backlog" { - lane = "doing" - localBrokerApplyStoryOrder(&store, issue.ID, lane, nil, 0) - localBrokerNormalizeLaneOrder(&store, fromLane, issue.ID) - issue, _ = localBrokerFindIssue(&store, req.ID) - } else if issue.Position == 0 { - localBrokerNormalizeLaneOrder(&store, lane, 0) - issue, _ = localBrokerFindIssue(&store, req.ID) - } - localBrokerIssueEvent(issue, user, "assigned", fromAssignee, user, "") - case "/issues/assign": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - from := issue.Assignee - to := strings.TrimSpace(req.Assignee) - issue.Assignee = to - localBrokerIssueEvent(issue, user, "assigned", from, to, "") - case "/issues/archive": - issue, err := localBrokerFindIssue(&store, req.ID) - if err != nil { - return nil, http.StatusNotFound, err - } - issue.Archived = req.Archived - action := "unarchived" - if req.Archived { - action = "archived" - } - localBrokerIssueEvent(issue, user, action, "", "", "") - default: - return mustJSON(map[string]string{"error": "unknown broker endpoint"}), http.StatusNotFound, nil - } - if path != "/issues/list" && path != "/issues/view" && path != "/issues/create" { - if err := s.saveIssueStore(state.Repo, store); err != nil { - return nil, 500, err - } - } - return mustJSON(map[string]bool{"ok": true}), 200, nil -} - -func (s *localBrokerServer) handleRefsUpdate(w http.ResponseWriter, r *http.Request, body []byte) { - var req brokerRefUpdateRequest - if !localBrokerDecode(w, body, &req) { - return - } - s.mu.Lock() - defer s.mu.Unlock() - state, err := s.loadRepoForRequest(req.Repo) - if err != nil { - localBrokerJSON(w, http.StatusNotFound, map[string]string{"error": "repository not found"}) - return - } - key, ok := s.signedRepoKey(r, body, state, "write") - if !ok { - localBrokerJSON(w, http.StatusForbidden, map[string]string{"error": "write SSH signature required"}) - return - } - unlock, err := s.acquireRefLock(state.Repo, req.Ref) - if err != nil { - localBrokerJSON(w, http.StatusConflict, map[string]string{"error": err.Error()}) - return - } - defer unlock() - current := s.localRefHash(state.Repo, req.Ref, firstNonEmpty(state.Refs[req.Ref], req.Old)) - if current != req.Old { - localBrokerJSON(w, http.StatusConflict, map[string]string{"error": "stale ref"}) - return - } - if state.Refs == nil { - state.Refs = map[string]string{} - } - if req.New == zeroObjectID() { - delete(state.Refs, req.Ref) - _ = s.deleteObject(state.Repo, req.Ref) - _ = s.deleteObject(state.Repo, localBrokerRefRecordPath(req.Ref)) - } else { - state.Refs[req.Ref] = req.New - _ = s.writeObject(state.Repo, req.Ref, []byte(req.New+"\n")) - record := map[string]any{"ref": req.Ref, "hash": req.New, "updated_by": key.User, "updated_at": time.Now().UTC().Format(time.RFC3339)} - data, _ := json.MarshalIndent(record, "", " ") - _ = s.writeObject(state.Repo, localBrokerRefRecordPath(req.Ref), data) - } - state.UpdatedAt = time.Now().UTC().Format(time.RFC3339) - if err := s.saveRepo(state); err != nil { - localBrokerJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - localBrokerJSON(w, 200, map[string]bool{"ok": true}) -} - -func (s *localBrokerServer) acquireRefLock(repo brokerRepo, ref string) (func(), error) { - lockName := base64.RawURLEncoding.EncodeToString([]byte(ref)) + ".lock" - path, err := s.objectPath(repo, filepath.ToSlash(filepath.Join(".bucketgit", "broker-state", "v1", "locks", lockName))) - if err != nil { - return nil, err - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return nil, err - } - deadline := time.Now().UTC().Add(30 * time.Second).Format(time.RFC3339Nano) - for attempt := 0; attempt < 2; attempt++ { - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) - if err == nil { - _, writeErr := f.WriteString(deadline) - closeErr := f.Close() - if writeErr != nil { - _ = os.Remove(path) - return nil, writeErr - } - if closeErr != nil { - _ = os.Remove(path) - return nil, closeErr - } - return func() { _ = os.Remove(path) }, nil - } - if !errors.Is(err, os.ErrExist) { - return nil, err - } - data, readErr := os.ReadFile(path) - if readErr == nil { - if expires, parseErr := time.Parse(time.RFC3339Nano, strings.TrimSpace(string(data))); parseErr == nil && time.Now().UTC().After(expires) { - _ = os.Remove(path) - continue - } - } - return nil, fmt.Errorf("ref %s is locked", ref) - } - return nil, fmt.Errorf("ref %s is locked", ref) -} - -func (s *localBrokerServer) handleObject(w http.ResponseWriter, r *http.Request) { - parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/_objects/"), "/") - if len(parts) < 2 { - http.NotFound(w, r) - return - } - bucket, _ := url.PathUnescape(parts[0]) - objectBytes, err := base64.RawURLEncoding.DecodeString(parts[1]) - if err != nil { - http.NotFound(w, r) - return - } - repo := brokerRepo{Bucket: bucket} - object := string(objectBytes) - switch r.Method { - case http.MethodGet: - data, err := s.readObject(repo, object) - if err != nil { - http.NotFound(w, r) - return - } - _, _ = w.Write(data) - case http.MethodPut: - data, _ := io.ReadAll(r.Body) - if err := s.writeObject(repo, object, data); err != nil { - localBrokerJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - localBrokerJSON(w, 200, map[string]bool{"ok": true}) - case http.MethodDelete: - _ = s.deleteObject(repo, object) - localBrokerJSON(w, 200, map[string]bool{"ok": true}) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func (s *localBrokerServer) physicalRepo(repo brokerRepo) brokerRepo { - if repo.Logical == "" { - repo.Logical = firstNonEmpty(repo.Prefix, "repo.git") - } - if logical, err := normalizeLogicalRepoName(repo.Logical); err == nil { - repo.Logical = logical - } - repo.TeamID = firstNonEmpty(repo.TeamID, coreTeamID) - repo.TeamName = firstNonEmpty(repo.TeamName, coreTeamName) - if repo.Bucket == "" { - repo.Bucket = repo.Logical - repo.Provider = "file" - repo.Prefix = "" - return repo - } - if strings.HasPrefix(repo.Bucket, "s3://") || strings.HasPrefix(repo.Bucket, "gs://") { - if cfg, ok, err := localBrokerCloudConfig(repo.Bucket); err == nil && ok { - repo.Bucket = cfg.bucket - repo.Provider = cfg.provider - repo.Profile = cfg.gcloudConfiguration - repo.Region = cfg.region - } - repo.Prefix = "" - return repo - } - if strings.HasPrefix(repo.Bucket, "file://") { - repo.Provider = "file" - repo.Prefix = "" - return repo - } - if repo.Provider == "s3" || repo.Provider == "gcs" { - repo.Prefix = "" - return repo - } - if repo.Prefix == "" && repo.Provider == "" { - repo.Provider = "file" - } - return repo -} - -func (s *localBrokerServer) loadRepo(repo brokerRepo) (localBrokerRepoState, error) { - var state localBrokerRepoState - if s.repoUsesCloudStorage(repo) { - data, err := s.readObject(repo, ".bucketgit/broker-state/v1/repo.json") - if err != nil { - return state, err - } - err = json.Unmarshal(data, &state) - state.Repo = s.physicalRepo(state.Repo) - return state, err - } - err := s.readJSON(s.repoStatePath(repo), &state) - if err == nil { - state.Repo = s.physicalRepo(state.Repo) - } - return state, err -} - -func (s *localBrokerServer) loadRepoForRequest(repo brokerRepo) (localBrokerRepoState, error) { - physical := s.physicalRepo(repo) - state, err := s.loadRepo(physical) - if err == nil { - return state, nil - } - logical := strings.TrimSpace(physical.Logical) - if logical == "" { - return localBrokerRepoState{}, err - } - if indexed, ok := s.indexedRepo(logical); ok { - state, indexErr := s.loadRepo(indexed) - if indexErr == nil && strings.EqualFold(strings.TrimSpace(state.Repo.Logical), logical) { - return state, nil - } - } - var found localBrokerRepoState - walkErr := filepath.WalkDir(s.objectRoot, func(path string, d fs.DirEntry, walkErr error) error { - if walkErr != nil || d.IsDir() || filepath.Base(path) != "repo.json" { - return nil - } - var candidate localBrokerRepoState - if readErr := s.readJSON(path, &candidate); readErr != nil { - return nil - } - if strings.EqualFold(strings.TrimSpace(candidate.Repo.Logical), logical) { - found = candidate - return fs.SkipAll - } - return nil - }) - if found.Repo.Logical != "" { - return found, nil - } - if walkErr != nil { - return localBrokerRepoState{}, walkErr - } - return localBrokerRepoState{}, err -} - -func (s *localBrokerServer) saveRepo(state localBrokerRepoState) error { - if !s.repoUsesCloudStorage(state.Repo) { - if err := s.saveJSON(s.repoStatePath(state.Repo), state); err != nil { - return err - } - } - if err := s.writeObject(state.Repo, ".bucketgit/broker-state/v1/repo.json", mustJSON(state)); err != nil { - return err - } - return s.upsertRepoIndex(state.Repo) -} - -func (s *localBrokerServer) loadIssueStore(repo brokerRepo) (localBrokerIssueStore, error) { - var store localBrokerIssueStore - data, err := s.readObject(repo, localBrokerIssuesPath()) - if err != nil { - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, os.ErrNotExist) { - return localBrokerIssueStore{NextID: 1}, nil - } - return store, err - } - if len(bytes.TrimSpace(data)) == 0 { - return localBrokerIssueStore{NextID: 1}, nil - } - if err := json.Unmarshal(data, &store); err != nil { - return store, err - } - if store.NextID <= 0 { - store.NextID = localBrokerNextIssueID(store.Issues) - } - return store, nil -} - -func (s *localBrokerServer) saveIssueStore(repo brokerRepo, store localBrokerIssueStore) error { - if store.NextID <= 0 { - store.NextID = localBrokerNextIssueID(store.Issues) - } - return s.writeObject(repo, localBrokerIssuesPath(), mustJSON(store)) -} - -func (s *localBrokerServer) repoIndexPath() string { - return filepath.Join(s.root, "repos.json") -} - -func (s *localBrokerServer) loadRepoIndex() localBrokerRepoIndex { - var index localBrokerRepoIndex - _ = s.readJSON(s.repoIndexPath(), &index) - return index -} - -func (s *localBrokerServer) indexedRepo(logical string) (brokerRepo, bool) { - logical = strings.TrimSpace(logical) - if logical == "" { - return brokerRepo{}, false - } - for _, repo := range s.loadRepoIndex().Repos { - if strings.EqualFold(strings.TrimSpace(repo.Logical), logical) { - return repo, true - } - } - return brokerRepo{}, false -} - -func (s *localBrokerServer) upsertRepoIndex(repo brokerRepo) error { - if strings.TrimSpace(repo.Logical) == "" { - return nil - } - index := s.loadRepoIndex() - for i := range index.Repos { - if strings.EqualFold(strings.TrimSpace(index.Repos[i].Logical), strings.TrimSpace(repo.Logical)) { - index.Repos[i] = repo - return s.saveJSON(s.repoIndexPath(), index) - } - } - index.Repos = append(index.Repos, repo) - sort.Slice(index.Repos, func(i, j int) bool { return index.Repos[i].Logical < index.Repos[j].Logical }) - return s.saveJSON(s.repoIndexPath(), index) -} - -func (s *localBrokerServer) repoStatePath(repo brokerRepo) string { - return filepath.Join(s.bucketDir(repo.Bucket), ".bucketgit", "broker-state", "v1", "repo.json") -} - -func (s *localBrokerServer) bucketDir(bucket string) string { - if strings.HasPrefix(bucket, "file://") { - path := strings.TrimPrefix(bucket, "file://") - if filepath.IsAbs(path) { - return filepath.Clean(path) - } - clean := filepath.Clean(strings.TrimPrefix(path, "/")) - if clean == "." { - clean = "repo.git" - } - return filepath.Join(s.objectRoot, clean) - } - return filepath.Join(s.objectRoot, base64.RawURLEncoding.EncodeToString([]byte(bucket))) -} - -func (s *localBrokerServer) ensureRepoStorage(ctx context.Context, repo brokerRepo) error { - cfg, ok, err := localBrokerRepoCloudConfig(repo) - if err != nil || !ok { - return err - } - return ensureBucket(ctx, cfg) -} - -func (s *localBrokerServer) repoUsesCloudStorage(repo brokerRepo) bool { - _, ok, _ := localBrokerRepoCloudConfig(repo) - return ok -} - -func (s *localBrokerServer) cloudStore(ctx context.Context, repo brokerRepo) (writableGitRemoteStore, func(), bool, error) { - cfg, ok, err := localBrokerRepoCloudConfig(repo) - if err != nil || !ok { - return nil, nil, ok, err - } - switch cfg.provider { - case "s3": - client, err := newS3Client(ctx, cfg, false) - if err != nil { - return nil, nil, true, err - } - return &s3GitStore{client: client, bucket: cfg.bucket, prefix: cfg.prefix}, nil, true, nil - case "gcs": - client, err := newStorageClient(ctx, cfg) - if err != nil { - return nil, nil, true, err - } - return &gcsGitStore{client: client, bucket: cfg.bucket, prefix: cfg.prefix}, func() { _ = client.Close() }, true, nil - default: - return nil, nil, false, nil - } -} - -func localBrokerRepoCloudConfig(repo brokerRepo) (config, bool, error) { - if cfg, ok, err := localBrokerCloudConfig(repo.Bucket); ok || err != nil { - return cfg, ok, err - } - provider := strings.TrimSpace(repo.Provider) - if provider != "s3" && provider != "gcs" { - return config{}, false, nil - } - region := strings.TrimSpace(repo.Region) - profile := strings.TrimSpace(repo.Profile) - if provider == "s3" { - region = firstNonEmpty(region, "us-east-1") - } else { - region = firstNonEmpty(region, "us-central1") - } - profile = firstNonEmpty(profile, "default") - return config{ - provider: provider, - bucket: strings.TrimSpace(repo.Bucket), - prefix: "", - region: region, - auth: defaultStorageAuthMode(), - gcloudConfiguration: profile, - }, true, nil -} - -func localBrokerCloudConfig(bucketURI string) (config, bool, error) { - scheme := "" - rest := "" - switch { - case strings.HasPrefix(bucketURI, "s3://"): - scheme = "s3" - rest = strings.TrimPrefix(bucketURI, "s3://") - case strings.HasPrefix(bucketURI, "gs://"): - scheme = "gs" - rest = strings.TrimPrefix(bucketURI, "gs://") - default: - return config{}, false, nil - } - host := strings.Split(rest, "/")[0] - if strings.TrimSpace(host) == "" { - return config{}, true, errors.New("local broker cloud repo URI must include a bucket name") - } - labels := strings.Split(host, ".") - clean := labels[:0] - for _, label := range labels { - if strings.TrimSpace(label) != "" { - clean = append(clean, strings.TrimSpace(label)) - } - } - if len(clean) == 0 { - return config{}, true, errors.New("local broker cloud repo URI must include a bucket name") - } - profile := "default" - region := "us-east-1" - provider := "s3" - if scheme == "gs" { - provider = "gcs" - region = "us-central1" - } - bucket := strings.Join(clean, ".") - if len(clean) >= 3 && localBrokerLooksCloudRegion(scheme, clean[1]) { - profile = clean[0] - region = clean[1] - bucket = strings.Join(clean[2:], ".") - } else if len(clean) >= 2 { - profile = clean[0] - bucket = strings.Join(clean[1:], ".") - } - if strings.TrimSpace(bucket) == "" { - return config{}, true, errors.New("local broker cloud repo URI must include a bucket name") - } - return config{ - provider: provider, - bucket: bucket, - prefix: "", - region: region, - auth: defaultStorageAuthMode(), - gcloudConfiguration: profile, - }, true, nil -} - -func localBrokerLooksCloudRegion(scheme, value string) bool { - if setupAWSRegionPattern.MatchString(value) || looksLikeGCPRegion(value) { - return true - } - if scheme != "gs" { - return false - } - if !strings.Contains(value, "-") { - return false - } - for i := len(value) - 1; i >= 0; i-- { - if value[i] >= '0' && value[i] <= '9' { - return true - } - } - return false -} - -func (s *localBrokerServer) readObject(repo brokerRepo, object string) ([]byte, error) { - if store, closeStore, ok, err := s.cloudStore(context.Background(), repo); ok || err != nil { - if closeStore != nil { - defer closeStore() - } - if err != nil { - return nil, err - } - return store.read(context.Background(), object) - } - path, err := s.objectPath(repo, object) - if err != nil { - return nil, err - } - return os.ReadFile(path) -} - -func (s *localBrokerServer) writeObject(repo brokerRepo, object string, data []byte) error { - if store, closeStore, ok, err := s.cloudStore(context.Background(), repo); ok || err != nil { - if closeStore != nil { - defer closeStore() - } - if err != nil { - return err - } - return store.write(context.Background(), object, data) - } - path, err := s.objectPath(repo, object) - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, data, 0o644) -} - -func (s *localBrokerServer) deleteObject(repo brokerRepo, object string) error { - if store, closeStore, ok, err := s.cloudStore(context.Background(), repo); ok || err != nil { - if closeStore != nil { - defer closeStore() - } - if err != nil { - return err - } - return store.delete(context.Background(), object) - } - path, err := s.objectPath(repo, object) - if err != nil { - return err - } - return os.Remove(path) -} - -func (s *localBrokerServer) listObjects(repo brokerRepo, prefix string) ([]string, error) { - if store, closeStore, ok, err := s.cloudStore(context.Background(), repo); ok || err != nil { - if closeStore != nil { - defer closeStore() - } - if err != nil { - return nil, err - } - return store.list(context.Background(), prefix) - } - root := s.bucketDir(repo.Bucket) - prefix = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(strings.TrimPrefix(prefix, "/"))), ".") - if prefix == "/" { - prefix = "" - } - searchRoot := root - if strings.TrimSpace(repo.Prefix) != "" { - var err error - searchRoot, err = safeJoinLocalPath(searchRoot, strings.Trim(repo.Prefix, "/")) - if err != nil { - return nil, err - } - } - var paths []string - err := filepath.WalkDir(searchRoot, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - rel, relErr := filepath.Rel(searchRoot, path) - if relErr != nil { - return nil - } - rel = filepath.ToSlash(rel) - if prefix == "" || strings.HasPrefix(rel, prefix) { - paths = append(paths, rel) - } - return nil - }) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } - sort.Strings(paths) - return paths, err -} - -func (s *localBrokerServer) objectPath(repo brokerRepo, object string) (string, error) { - root := s.bucketDir(repo.Bucket) - prefix := strings.Trim(repo.Prefix, "/") - if prefix != "" { - object = prefix + "/" + strings.TrimPrefix(object, "/") - } - clean := strings.TrimPrefix(filepath.ToSlash(filepath.Clean("/"+object)), "/") - return safeJoinLocalPath(root, clean) -} - -func (s *localBrokerServer) loadOwners() (localBrokerOwners, error) { - var owners localBrokerOwners - err := s.readJSON(filepath.Join(s.root, "owners.json"), &owners) - return owners, err -} - -func (s *localBrokerServer) readJSON(path string, dst any) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - return json.Unmarshal(data, dst) -} - -func (s *localBrokerServer) saveJSON(path string, value any) error { - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, mustJSON(value), 0o644) -} - -func (s *localBrokerServer) validBootstrap(r *http.Request) bool { - if s.bootstrapHash == "" { - return false - } - return brokerSecretHash(r.Header.Get("x-bgit-bootstrap-token")) == s.bootstrapHash -} - -func (s *localBrokerServer) requireOwner(r *http.Request, body []byte) (*brokerKey, bool) { - owners, _ := s.loadOwners() - return s.signedOwner(r, body, owners) -} - -func (s *localBrokerServer) signedOwner(r *http.Request, body []byte, owners localBrokerOwners) (*brokerKey, bool) { - for i := range owners.Keys { - key := &owners.Keys[i] - if (key.Role == "owner" || key.Role == "admin") && verifyLocalBrokerSignature(r, body, key.PublicKey) { - return key, true - } - } - return nil, false -} - -func (s *localBrokerServer) signedRepoKey(r *http.Request, body []byte, state localBrokerRepoState, operation string) (*brokerKey, bool) { - for i := range state.Keys { - key := &state.Keys[i] - if roleAllowsLocal(key.Role, operation) && verifyLocalBrokerSignature(r, body, key.PublicKey) { - return key, true - } - } - return nil, false -} - -func verifyLocalBrokerSignature(r *http.Request, body []byte, publicKey string) bool { - if strings.TrimSpace(publicKey) == "" || r.Header.Get("x-bgit-signature-version") != "2" { - localBrokerSignatureDebug("missing key or signature version") - return false - } - if normalizeKey(r.Header.Get("x-bgit-key")) != normalizeKey(publicKey) { - localBrokerSignatureDebug("key mismatch") - return false - } - messageB64 := r.Header.Get("x-bgit-signature-message") - sigB64 := r.Header.Get("x-bgit-signature") - message, err := base64.StdEncoding.DecodeString(messageB64) - if err != nil { - localBrokerSignatureDebug("message decode failed: " + err.Error()) - return false - } - want := brokerSignatureMessage(r.Method, r.URL.RequestURI(), strings.ToLower(r.Host), r.Header.Get("x-bgit-timestamp"), r.Header.Get("x-bgit-nonce"), body) - if !bytes.Equal(message, want) { - localBrokerSignatureDebug(fmt.Sprintf("message mismatch got=%q want=%q host=%q path=%q", string(message), string(want), r.Host, r.URL.RequestURI())) - return false - } - var sig ssh.Signature - sigRaw, err := base64.StdEncoding.DecodeString(sigB64) - if err != nil { - localBrokerSignatureDebug("signature decode failed: " + err.Error()) - return false - } - _ = ssh.Unmarshal(sigRaw, &sig) - if sig.Format == "" || len(sig.Blob) == 0 { - localBrokerSignatureDebug("signature unmarshal failed") - return false - } - pub, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey)) - if err != nil { - localBrokerSignatureDebug("public key parse failed: " + err.Error()) - return false - } - if err := pub.Verify(message, &sig); err != nil { - localBrokerSignatureDebug("signature verify failed: " + err.Error()) - return false - } - return true -} - -func localBrokerSignatureDebug(msg string) { - if os.Getenv("BGIT_LOCAL_BROKER_DEBUG") != "" { - fmt.Fprintln(os.Stderr, "local broker signature:", msg) - } -} - -func localBrokerOperation(operation string) string { - switch strings.ToLower(operation) { - case "write", "push", "upload-pack": - return "write" - default: - return "read" - } -} - -func roleAllowsLocal(role, operation string) bool { - role = normalizeBrokerRole(role) - if role == "owner" || role == "admin" { - return true - } - if operation == "read" { - return role == "read" || role == "triage" || role == "developer" || role == "maintainer" - } - if operation == "write" { - return role == "developer" || role == "maintainer" - } - return false -} - -func roleCapabilitiesForLocal(role string) map[string]bool { - return map[string]bool{ - "read": roleAllowsLocal(role, "read"), - "push": roleAllowsLocal(role, "write"), - "merge": role == "maintainer" || role == "admin" || role == "owner", - } -} - -func validateLocalBrokerCapabilityPath(operation, objectPath string) (string, error) { - path := strings.TrimPrefix(filepath.ToSlash(filepath.Clean("/"+objectPath)), "/") - if path == "" || strings.Contains(path, "..") { - return "", errors.New("invalid object path") - } - if operation == "write" && !(path == "objects" || strings.HasPrefix(path, "objects/")) { - return "", errors.New("write capabilities are restricted to git object paths") - } - if operation == "read" { - if path == "HEAD" || path == "packed-refs" || strings.HasPrefix(path, "refs/") || path == "objects" || strings.HasPrefix(path, "objects/") { - return path, nil - } - return "", errors.New("read capabilities are restricted to git repository paths") - } - if operation == "delete" { - return "", errors.New("delete capabilities are not supported") - } - return path, nil -} - -func localBrokerObjectName(repo brokerRepo, objectPath string) string { - prefix := strings.Trim(repo.Prefix, "/") - if prefix == "" { - return objectPath - } - return prefix + "/" + strings.TrimPrefix(objectPath, "/") -} - -func localBrokerRefRecordPath(ref string) string { - return ".bucketgit/broker-state/v1/refs/" + base64.RawURLEncoding.EncodeToString([]byte(ref)) + ".json" -} - -func localBrokerIssuesPath() string { - return ".bucketgit/broker-state/v1/issues.json" -} - -func localBrokerFindIssue(store *localBrokerIssueStore, id int) (*brokerIssue, error) { - if id <= 0 { - return nil, errors.New("issue id is required") - } - for i := range store.Issues { - if store.Issues[i].ID == id { - return &store.Issues[i], nil - } - } - return nil, errors.New("issue not found") -} - -func localBrokerNextIssueID(issues []brokerIssue) int { - next := 1 - for _, issue := range issues { - if issue.ID >= next { - next = issue.ID + 1 - } - } - return next -} - -func localBrokerNextStoryPosition(issues []brokerIssue, lane string) float64 { - count := 0 - for _, issue := range issues { - if issue.Type == "story" && !issue.Archived && normalizeKanbanLane(issue.Lane) == lane { - count++ - } - } - return float64(count + 1) -} - -func localBrokerStoryOrderFromRequest(order int) int { - if order > 0 { - return order - } - return 0 -} - -func localBrokerApplyStoryOrder(store *localBrokerIssueStore, issueID int, lane string, afterID *int, order int) { - target, err := localBrokerFindIssue(store, issueID) - if err != nil { - return - } - target.Lane = lane - stories := localBrokerLaneStories(store, lane, issueID) - index := len(stories) - if order > 0 { - index = order - 1 - if index < 0 { - index = 0 - } - if index > len(stories) { - index = len(stories) - } - } else if afterID != nil { - index = 0 - if *afterID > 0 { - index = len(stories) - for i, story := range stories { - if story.ID == *afterID { - index = i + 1 - break - } - } - } - } - stories = append(stories, nil) - copy(stories[index+1:], stories[index:]) - stories[index] = target - for i, story := range stories { - story.Lane = lane - story.Position = float64(i + 1) - } -} - -func localBrokerNormalizeLaneOrder(store *localBrokerIssueStore, lane string, excludeID int) { - stories := localBrokerLaneStories(store, lane, excludeID) - for i, story := range stories { - story.Position = float64(i + 1) - } -} - -func localBrokerLaneStories(store *localBrokerIssueStore, lane string, excludeID int) []*brokerIssue { - stories := []*brokerIssue{} - for i := range store.Issues { - issue := &store.Issues[i] - if issue.ID == excludeID || issue.Type != "story" || issue.Archived || normalizeKanbanLane(issue.Lane) != lane { - continue - } - stories = append(stories, issue) - } - sort.SliceStable(stories, func(i, j int) bool { - left := stories[i].Position - right := stories[j].Position - if left != right { - if left == 0 { - return false - } - if right == 0 { - return true - } - return left < right - } - return stories[i].ID < stories[j].ID - }) - return stories -} - -func localBrokerIssueEvent(issue *brokerIssue, user, action, from, to, position string) { - now := time.Now().UTC().Format(time.RFC3339) - issue.UpdatedAt = now - issue.History = append(issue.History, brokerIssueEvent{User: user, Action: action, From: from, To: to, Position: position, At: now}) -} - -func (s *localBrokerServer) localRefHash(repo brokerRepo, ref, fallback string) string { - var record struct { - Hash string `json:"hash"` - } - if path, err := s.objectPath(repo, localBrokerRefRecordPath(ref)); err == nil && s.readJSON(path, &record) == nil && record.Hash != "" { - return record.Hash - } - if data, err := s.readObject(repo, ref); err == nil { - text := strings.TrimSpace(string(data)) - if len(text) == 40 { - return text - } - } - return fallback -} - -func localBrokerDecode(w http.ResponseWriter, data []byte, dst any) bool { - if err := json.Unmarshal(data, dst); err != nil { - localBrokerJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid JSON"}) - return false - } - return true -} - -func localBrokerJSON(w http.ResponseWriter, status int, value any) { - w.Header().Set("content-type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(value) -} - -func mustJSON(value any) []byte { - data, _ := json.MarshalIndent(value, "", " ") - return data -} - -func normalizeKey(key string) string { - key = strings.TrimSpace(key) - parts := strings.Fields(key) - if len(parts) >= 2 { - return parts[0] + " " + parts[1] - } - return key -} diff --git a/protocol/domain.go b/protocol/domain.go new file mode 100644 index 0000000..37b6fad --- /dev/null +++ b/protocol/domain.go @@ -0,0 +1,399 @@ +package protocol + +type Key struct { + User string `json:"user"` + Role string `json:"role"` + PublicKey string `json:"public_key"` + Source string `json:"source,omitempty"` + Suspended bool `json:"suspended,omitempty"` +} +type OwnerRequest struct { + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` +} + +type RepositoryRequest struct { + Repo Repository `json:"repo"` + AdminUser string `json:"admin_user,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + Role string `json:"role,omitempty"` +} + +type KeyRequest struct { + Repo Repository `json:"repo"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + Key string `json:"key,omitempty"` + Source string `json:"source,omitempty"` +} + +type KeysResponse struct { + Keys []Key `json:"keys"` +} + +type RepositoryAdminRequest struct { + Repo Repository `json:"repo"` + Description string `json:"description,omitempty"` + DefaultBranch string `json:"default_branch,omitempty"` + Visibility string `json:"visibility,omitempty"` + ReadOnly *bool `json:"read_only,omitempty"` + IssuesEnabled *bool `json:"issues_enabled,omitempty"` + Logical string `json:"logical,omitempty"` + TeamID string `json:"team_id,omitempty"` + Name string `json:"name,omitempty"` + UserID string `json:"user_id,omitempty"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + BrokerRole string `json:"broker_role,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + Suspended bool `json:"suspended,omitempty"` + BrokerURL string `json:"broker_url,omitempty"` + Token string `json:"token,omitempty"` +} + +type RepositoryInfo struct { + Repo Repository `json:"repo"` + Logical string `json:"logical,omitempty"` + Teams []RepositoryTeamGrant `json:"teams,omitempty"` +} +type RepositoryListResponse struct { + Repos []RepositoryInfo `json:"repos"` +} +type AdminRepositoryInfoResponse struct { + Repo Repository `json:"repo"` + Description string `json:"description,omitempty"` + DefaultBranch string `json:"default_branch,omitempty"` + Visibility string `json:"visibility,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` + IssuesEnabled bool `json:"issues_enabled,omitempty"` +} +type RepositoryTeamsResponse struct { + Teams []RepositoryTeamGrant `json:"teams"` +} +type RepositoryUsersResponse struct { + Users []RepositoryUserGrant `json:"users"` +} +type RepositoryUserGrant struct { + UserID string `json:"user_id,omitempty"` + User string `json:"user,omitempty"` + Username string `json:"username,omitempty"` + Role string `json:"role,omitempty"` +} +type UsersResponse struct { + Users []UserInfo `json:"users"` +} +type UserInfo struct { + ID string `json:"id"` + Username string `json:"username"` + BrokerRole string `json:"broker_role"` + Keys []Key `json:"keys,omitempty"` + Suspended bool `json:"suspended,omitempty"` + Pending bool `json:"pending,omitempty"` +} +type RepositoryInvitesResponse struct { + Invites []RepositoryInvite `json:"invites"` +} +type RepositoryInvite struct { + User string `json:"user"` + Role string `json:"role"` + ExpiresAt string `json:"expires_at"` +} +type TeamsResponse struct { + Teams []Team `json:"teams"` +} +type Team struct { + ID string `json:"id"` + Name string `json:"name"` + Members []TeamMember `json:"members,omitempty"` +} +type TeamMember struct { + UserID string `json:"user_id,omitempty"` + Username string `json:"username,omitempty"` + Role string `json:"role"` +} +type RepositoryTeamGrant struct { + ID string `json:"id,omitempty"` + TeamID string `json:"team_id,omitempty"` + Role string `json:"role,omitempty"` +} + +type OwnerTransferRequest struct { + Repo Repository `json:"repo"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + BrokerURL string `json:"broker_url,omitempty"` + Token string `json:"token,omitempty"` +} +type OwnerTransferResponse struct { + Code string `json:"code"` + AcceptCommand string `json:"accept_command"` + CancelCommand string `json:"cancel_command"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` +} +type Protection struct { + Repo Repository `json:"repo"` + Ref string `json:"ref"` + RequirePR bool `json:"require_pr"` + AllowOverrides bool `json:"allow_overrides"` +} + +type PullRequest struct { + ID int `json:"id,omitempty"` + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` + Source string `json:"source,omitempty"` + Target string `json:"target,omitempty"` + Status string `json:"status,omitempty"` + Author string `json:"author,omitempty"` + Version string `json:"version,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` + Approvals int `json:"approvals,omitempty"` + Checks []string `json:"checks,omitempty"` + Head string `json:"head,omitempty"` + Comments []PullRequestNote `json:"comments,omitempty"` + Reviews []PullRequestNote `json:"reviews,omitempty"` + MergedBy string `json:"merged_by,omitempty"` + MergedAt string `json:"merged_at,omitempty"` + ClosedBy string `json:"closed_by,omitempty"` + ClosedAt string `json:"closed_at,omitempty"` +} +type PullRequestNote struct { + ID int `json:"id,omitempty"` + User string `json:"user,omitempty"` + Body string `json:"body,omitempty"` + State string `json:"state,omitempty"` + Source string `json:"source,omitempty"` + At string `json:"at,omitempty"` + Comments []PullRequestComment `json:"comments,omitempty"` + Replies []PullRequestComment `json:"replies,omitempty"` + Head string `json:"head,omitempty"` +} +type PullRequestComment struct { + ID int `json:"id,omitempty"` + User string `json:"user,omitempty"` + Body string `json:"body,omitempty"` + File string `json:"file,omitempty"` + Kind string `json:"kind,omitempty"` + Side string `json:"side,omitempty"` + Hunk string `json:"hunk,omitempty"` + HunkIndex int `json:"hunk_index,omitempty"` + OldStart int `json:"old_start,omitempty"` + NewStart int `json:"new_start,omitempty"` + Offset int `json:"offset,omitempty"` + Line int `json:"line,omitempty"` + LineText string `json:"line_text,omitempty"` + LineHash string `json:"line_hash,omitempty"` + Head string `json:"head,omitempty"` + Outdated bool `json:"outdated,omitempty"` + At string `json:"at,omitempty"` + Replies []PullRequestComment `json:"replies,omitempty"` +} +type PullRequestRequest struct { + Repo Repository `json:"repo"` + ID int `json:"id,omitempty"` + PR PullRequest `json:"pr,omitempty"` + Known map[string]string `json:"known,omitempty"` + Merge bool `json:"merge,omitempty"` + DeleteBranch bool `json:"delete_branch,omitempty"` + Comment string `json:"comment,omitempty"` + Review string `json:"review,omitempty"` + Comments []PullRequestComment `json:"comments,omitempty"` + TargetNoteID int `json:"target_note_id,omitempty"` + TargetCommentID int `json:"target_comment_id,omitempty"` +} +type PullRequestsResponse struct { + PRs []PullRequest `json:"prs"` + Deleted []int `json:"deleted,omitempty"` +} +type PullRequestResponse struct { + PR PullRequest `json:"pr"` +} + +type CIRun struct { + ID int `json:"id,omitempty"` + Provider string `json:"provider,omitempty"` + Ref string `json:"ref,omitempty"` + Commit string `json:"commit,omitempty"` + Config string `json:"config,omitempty"` + Status string `json:"status,omitempty"` + Result string `json:"result,omitempty"` + URL string `json:"url,omitempty"` + Message string `json:"message,omitempty"` + ProviderBuildID string `json:"provider_build_id,omitempty"` + ProviderBuildName string `json:"provider_build_name,omitempty"` + LogGroup string `json:"log_group,omitempty"` + LogStream string `json:"log_stream,omitempty"` + Author string `json:"author,omitempty"` + StartedAt string `json:"started_at,omitempty"` + FinishedAt string `json:"finished_at,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} +type CIRequest struct { + Repo Repository `json:"repo"` + ID int `json:"id,omitempty"` + Provider string `json:"provider,omitempty"` + Ref string `json:"ref,omitempty"` + Commit string `json:"commit,omitempty"` + Config string `json:"config,omitempty"` +} +type CILogResponse struct { + Run CIRun `json:"run"` + Logs string `json:"logs"` +} + +type Issue struct { + ID int `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` + Status string `json:"status,omitempty"` + Lane string `json:"lane,omitempty"` + Assignee string `json:"assignee,omitempty"` + Position float64 `json:"position,omitempty"` + Archived bool `json:"archived,omitempty"` + Author string `json:"author,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` + Comments []IssueReply `json:"comments,omitempty"` + History []IssueEvent `json:"history,omitempty"` +} +type IssueReply struct { + User string `json:"user,omitempty"` + Body string `json:"body,omitempty"` + At string `json:"at,omitempty"` +} +type IssueEvent struct { + User string `json:"user,omitempty"` + Action string `json:"action,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + At string `json:"at,omitempty"` + Ref string `json:"ref,omitempty"` + Position string `json:"position,omitempty"` +} +type IssueRequest struct { + Repo Repository `json:"repo"` + ID int `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` + Lane string `json:"lane,omitempty"` + Assignee string `json:"assignee,omitempty"` + Comment string `json:"comment,omitempty"` + AfterID *int `json:"after_id,omitempty"` + Order int `json:"order,omitempty"` + Archived bool `json:"archived,omitempty"` + IncludeArchived bool `json:"include_archived,omitempty"` +} + +type AuthStatusRequest struct { + Repo Repository `json:"repo"` +} +type Identity struct { + User string `json:"user,omitempty"` + Source string `json:"source,omitempty"` + KeyFingerprint string `json:"key_fingerprint,omitempty"` + PublicKey string `json:"public_key,omitempty"` +} +type AuthStatus struct { + BrokerURL string `json:"broker_url,omitempty"` + BrokerVersion string `json:"broker_version,omitempty"` + Repo Repository `json:"repo,omitempty"` + Identity Identity `json:"identity,omitempty"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + Capabilities map[string]bool `json:"capabilities,omitempty"` + ResolvedAt string `json:"resolved_at,omitempty"` + CachedAt string `json:"cached_at,omitempty"` + Stale bool `json:"stale,omitempty"` + Error string `json:"error,omitempty"` +} +type RepositoriesMineResponse struct { + Repos []RepositoryMembership `json:"repos"` +} +type RepositoryMembership struct { + RepoID string `json:"repo_id,omitempty"` + Logical string `json:"logical,omitempty"` + Repo Repository `json:"repo,omitempty"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + Source string `json:"source,omitempty"` + KeyFingerprint string `json:"key_fingerprint,omitempty"` + Suspended bool `json:"suspended,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} + +type RepositoryInfoRequest struct { + Repo Repository `json:"repo"` + Description string `json:"description,omitempty"` + DefaultBranch string `json:"default_branch,omitempty"` + Visibility string `json:"visibility,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` + IssuesEnabled bool `json:"issues_enabled"` + Logical string `json:"logical,omitempty"` + TeamID string `json:"team_id,omitempty"` + Name string `json:"name,omitempty"` + UserID string `json:"user_id,omitempty"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` + BrokerRole string `json:"broker_role,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` +} +type RepositoryInfoResponse struct { + Repo Repository `json:"repo"` + Description string `json:"description"` + DefaultBranch string `json:"default_branch"` + Visibility string `json:"visibility"` + ReadOnly bool `json:"read_only"` + IssuesEnabled bool `json:"issues_enabled"` +} +type UserProfileRequest struct { + Repo Repository `json:"repo"` + Bio string `json:"bio,omitempty"` + Avatar string `json:"avatar,omitempty"` +} +type UserProfileResponse struct { + User string `json:"user"` + Profile UserProfile `json:"profile"` + Keys []UserProfileKey `json:"keys"` +} +type UserProfile struct { + Bio string `json:"bio,omitempty"` + Avatar string `json:"avatar,omitempty"` +} +type UserProfileKey struct { + PublicKey string `json:"public_key"` + Fingerprint string `json:"fingerprint,omitempty"` + Source string `json:"source,omitempty"` +} + +type Lane string + +const ( + LaneBacklog Lane = "backlog" + LaneReady Lane = "ready" + LaneDoing Lane = "doing" + LaneReview Lane = "review" + LaneDone Lane = "done" +) + +type PullRequestStatus string + +const ( + PullRequestOpen PullRequestStatus = "open" + PullRequestMerged PullRequestStatus = "merged" + PullRequestClosed PullRequestStatus = "closed" +) + +type CIStatus string + +const ( + CIQueued CIStatus = "queued" + CIBuilding CIStatus = "building" + CIFinished CIStatus = "finished" +) diff --git a/protocol/errors.go b/protocol/errors.go new file mode 100644 index 0000000..1456eaa --- /dev/null +++ b/protocol/errors.go @@ -0,0 +1,43 @@ +package protocol + +import ( + "errors" + "fmt" +) + +var ( + ErrConflict = errors.New("bucketgit conflict") + ErrUnauthorized = errors.New("bucketgit unauthorized") + ErrUnsupported = errors.New("bucketgit unsupported") + ErrSignatureExpired = errors.New("bucketgit signature timestamp expired") + ErrReplay = errors.New("bucketgit signature replay") +) + +type BrokerError struct { + Endpoint string + Status int + Code string + Message string + Kind error +} + +func (e *BrokerError) Error() string { + if e == nil { + return "" + } + prefix := "broker" + if e.Endpoint != "" { + prefix += " " + e.Endpoint + } + if e.Message == "" { + return prefix + } + return fmt.Sprintf("%s: %s", prefix, e.Message) +} + +func (e *BrokerError) Unwrap() error { + if e == nil { + return nil + } + return e.Kind +} diff --git a/protocol/protocol.go b/protocol/protocol.go new file mode 100644 index 0000000..63b430d --- /dev/null +++ b/protocol/protocol.go @@ -0,0 +1,144 @@ +// Package protocol defines the stable data model shared by BucketGit clients +// and brokers. JSON field names are part of the broker compatibility contract. +package protocol + +import "fmt" + +const ( + SignatureVersion = "2" + SignaturePrefix = "bgit-broker-v2" +) + +const ( + HeaderSignatureVersion = "X-Bgit-Signature-Version" + HeaderKey = "X-Bgit-Key" + HeaderKeyFingerprint = "X-Bgit-Key-Fingerprint" + HeaderTimestamp = "X-Bgit-Timestamp" + HeaderNonce = "X-Bgit-Nonce" + HeaderSignedHost = "X-Bgit-Signed-Host" + HeaderSignature = "X-Bgit-Signature" + HeaderSignatureMessage = "X-Bgit-Signature-Message" +) + +type Provider string + +const ( + ProviderFile Provider = "file" + ProviderS3 Provider = "s3" + ProviderGCS Provider = "gcs" +) + +type Operation string + +const ( + OperationRead Operation = "read" + OperationWrite Operation = "write" + OperationDelete Operation = "delete" + OperationMerge Operation = "merge" +) + +type Role string + +const ( + RoleOwner Role = "owner" + RoleAdmin Role = "admin" + RoleMaintainer Role = "maintainer" + RoleDeveloper Role = "developer" + RoleTriage Role = "triage" + RoleRead Role = "read" +) + +func (r Role) Valid() bool { + switch r { + case RoleOwner, RoleAdmin, RoleMaintainer, RoleDeveloper, RoleTriage, RoleRead: + return true + default: + return false + } +} + +type Repository struct { + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Origin string `json:"origin"` + Logical string `json:"logical,omitempty"` + Host string `json:"host,omitempty"` + Profile string `json:"profile,omitempty"` + Region string `json:"region,omitempty"` + TeamID string `json:"team_id,omitempty"` + TeamName string `json:"team_name,omitempty"` +} + +func (r Repository) Validate() error { + if r.Logical == "" && (r.Bucket == "" || r.Prefix == "") { + return fmt.Errorf("repository requires a logical name or bucket and prefix") + } + return nil +} + +type AuthRequest struct { + Repo Repository `json:"repo"` + Operation string `json:"operation"` +} + +type AuthResponse struct { + Allowed bool `json:"allowed"` + User string `json:"user,omitempty"` + Role string `json:"role,omitempty"` +} + +type RefUpdateRequest struct { + Repo Repository `json:"repo"` + Ref string `json:"ref"` + Old string `json:"old"` + New string `json:"new"` + Override bool `json:"override,omitempty"` +} + +type RefsRequest struct { + Repo Repository `json:"repo"` +} + +type RefsResponse struct { + Refs map[string]string `json:"refs"` +} + +type ObjectCapabilityRequest struct { + Repo Repository `json:"repo"` + Path string `json:"path"` + Operation string `json:"operation"` + Size int64 `json:"size,omitempty"` + Resumable bool `json:"resumable,omitempty"` +} + +type AWSCredentials struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` +} + +type ObjectCapabilityResponse struct { + Provider string `json:"provider"` + Mode string `json:"mode"` + Method string `json:"method,omitempty"` + URL string `json:"url,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Object string `json:"object,omitempty"` + Profile string `json:"profile,omitempty"` + Region string `json:"region,omitempty"` + Credentials AWSCredentials `json:"credentials,omitempty"` +} + +type ObjectRequest struct { + Repo Repository `json:"repo"` + Path string `json:"path,omitempty"` + Prefix string `json:"prefix,omitempty"` +} + +type ObjectResponse struct { + Data string `json:"data,omitempty"` + Paths []string `json:"paths,omitempty"` +} diff --git a/protocol/signing.go b/protocol/signing.go new file mode 100644 index 0000000..9d4382d --- /dev/null +++ b/protocol/signing.go @@ -0,0 +1,76 @@ +package protocol + +import ( + "crypto/sha256" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" +) + +type RequestMetadata struct { + Method string + Path string + Host string + Timestamp string + Nonce string +} + +// ValidateRequestMetadata validates the replay-sensitive fields of a v2 +// signature before cryptographic verification. The callback must atomically +// return true only for a nonce that has not been observed before. +func ValidateRequestMetadata(meta RequestMetadata, now time.Time, maxSkew time.Duration, acceptNonce func(string, time.Time) bool) error { + timestamp, err := strconv.ParseInt(strings.TrimSpace(meta.Timestamp), 10, 64) + if err != nil { + return fmt.Errorf("invalid signature timestamp: %w", err) + } + if maxSkew <= 0 { + return errors.New("signature timestamp skew must be positive") + } + signedAt := time.Unix(timestamp, 0) + delta := now.Sub(signedAt) + if delta < 0 { + delta = -delta + } + if delta > maxSkew { + return ErrSignatureExpired + } + nonce := strings.TrimSpace(meta.Nonce) + if len(nonce) < 16 || len(nonce) > 256 { + return errors.New("signature nonce must contain 16 to 256 characters") + } + for _, char := range nonce { + if char <= 0x20 || char > 0x7e { + return errors.New("signature nonce contains invalid characters") + } + } + if acceptNonce == nil || !acceptNonce(nonce, signedAt.Add(maxSkew)) { + return ErrReplay + } + return nil +} + +// SignatureMessage returns the canonical v2 message signed by BucketGit SSH +// identities. Its byte representation is a wire compatibility contract. +func SignatureMessage(meta RequestMetadata, payload []byte) []byte { + method := strings.ToUpper(strings.TrimSpace(meta.Method)) + if method == "" { + method = http.MethodPost + } + requestPath := strings.TrimSpace(meta.Path) + if requestPath == "" { + requestPath = "/" + } + sum := sha256.Sum256(payload) + return []byte(strings.Join([]string{ + SignaturePrefix, + method, + requestPath, + strings.ToLower(strings.TrimSpace(meta.Host)), + strings.TrimSpace(meta.Timestamp), + strings.TrimSpace(meta.Nonce), + fmt.Sprintf("%x", sum[:]), + }, "\n")) +} diff --git a/protocol/signing_test.go b/protocol/signing_test.go new file mode 100644 index 0000000..2786e8c --- /dev/null +++ b/protocol/signing_test.go @@ -0,0 +1,103 @@ +package protocol + +import ( + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "os" + "testing" + "time" + + "golang.org/x/crypto/ssh" +) + +func TestSignatureMessageV2Golden(t *testing.T) { + got := string(SignatureMessage(RequestMetadata{ + Method: http.MethodPost, + Path: "/repos/mine", + Host: "Broker.Example.Com", + Timestamp: "1770000000", + Nonce: "nonce", + }, []byte(`{"repo":"demo"}`))) + want := "bgit-broker-v2\nPOST\n/repos/mine\nbroker.example.com\n1770000000\nnonce\n99caed5a504d67241e3f491cfb6f3088703b3291c70788b1064070947e653755" + if got != want { + t.Fatalf("signature message:\n got %q\nwant %q", got, want) + } +} + +func TestValidateRequestMetadata(t *testing.T) { + now := time.Unix(1770000000, 0) + accepted := false + err := ValidateRequestMetadata(RequestMetadata{Timestamp: "1770000000", Nonce: "0123456789abcdef"}, now, 5*time.Minute, func(nonce string, expires time.Time) bool { + accepted = nonce == "0123456789abcdef" && expires.Equal(now.Add(5*time.Minute)) + return accepted + }) + if err != nil || !accepted { + t.Fatalf("ValidateRequestMetadata = %v, accepted=%v", err, accepted) + } + if err := ValidateRequestMetadata(RequestMetadata{Timestamp: "1769990000", Nonce: "0123456789abcdef"}, now, time.Minute, func(string, time.Time) bool { return true }); !errors.Is(err, ErrSignatureExpired) { + t.Fatalf("expired error = %v", err) + } + if err := ValidateRequestMetadata(RequestMetadata{Timestamp: "1770000000", Nonce: "0123456789abcdef"}, now, time.Minute, func(string, time.Time) bool { return false }); !errors.Is(err, ErrReplay) { + t.Fatalf("replay error = %v", err) + } +} + +func TestSignatureV2Fixture(t *testing.T) { + data, err := os.ReadFile("../spec/testdata/signing-v2.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Vectors []struct { + Method, Path, Host, Timestamp, Nonce, Body, Message string + Signatures []struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` + Signature string `json:"signature"` + } `json:"signatures"` + } `json:"vectors"` + } + if err := json.Unmarshal(data, &fixture); err != nil { + t.Fatal(err) + } + for _, vector := range fixture.Vectors { + got := string(SignatureMessage(RequestMetadata{ + Method: vector.Method, Path: vector.Path, Host: vector.Host, + Timestamp: vector.Timestamp, Nonce: vector.Nonce, + }, []byte(vector.Body))) + if got != vector.Message { + t.Fatalf("fixture message = %q, want %q", got, vector.Message) + } + for _, signed := range vector.Signatures { + key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(signed.PublicKey)) + if err != nil { + t.Fatalf("parse %s key: %v", signed.Name, err) + } + wire, err := base64.StdEncoding.DecodeString(signed.Signature) + if err != nil { + t.Fatalf("decode %s signature: %v", signed.Name, err) + } + var signature ssh.Signature + if err := ssh.Unmarshal(wire, &signature); err != nil { + t.Fatalf("parse %s signature: %v", signed.Name, err) + } + if err := key.Verify([]byte(got), &signature); err != nil { + t.Fatalf("verify %s signature: %v", signed.Name, err) + } + } + } +} + +func TestRepositoryValidation(t *testing.T) { + if err := (Repository{Logical: "demo.git"}).Validate(); err != nil { + t.Fatal(err) + } + if err := (Repository{Bucket: "bucket", Prefix: "repo.git"}).Validate(); err != nil { + t.Fatal(err) + } + if err := (Repository{}).Validate(); err == nil { + t.Fatal("expected empty repository to be invalid") + } +} diff --git a/repository/fuzz_test.go b/repository/fuzz_test.go new file mode 100644 index 0000000..e78f0cd --- /dev/null +++ b/repository/fuzz_test.go @@ -0,0 +1,19 @@ +package repository + +import "testing" + +func FuzzParseTree(f *testing.F) { + f.Add([]byte("100644 README.md\x0001234567890123456789")) + f.Add([]byte{}) + f.Fuzz(func(t *testing.T, data []byte) { _, _ = parseTree(data) }) +} + +func FuzzParsePackIndex(f *testing.F) { + f.Add([]byte{0xff, 't', 'O', 'c', 0, 0, 0, 2}) + f.Fuzz(func(t *testing.T, data []byte) { _, _, _ = parsePackIndex(data) }) +} + +func FuzzApplyDelta(f *testing.F) { + f.Add([]byte("base"), []byte{4, 4, 4, 't', 'e', 's', 't'}) + f.Fuzz(func(t *testing.T, base, delta []byte) { _, _ = applyDelta(base, delta) }) +} diff --git a/repository/pack.go b/repository/pack.go new file mode 100644 index 0000000..f2d4d82 --- /dev/null +++ b/repository/pack.go @@ -0,0 +1,339 @@ +package repository + +import ( + "bytes" + "compress/zlib" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "sort" + "strings" +) + +type packIndex struct { + packPath string + hashes []OID + offsets []uint64 +} + +func (r *Repository) packedObject(ctx context.Context, oid OID) (Object, error) { + if err := r.loadPackIndexes(ctx); err != nil { + return Object{}, err + } + for _, index := range r.packs { + position := sort.Search(len(index.hashes), func(i int) bool { return index.hashes[i] >= oid }) + if position < len(index.hashes) && index.hashes[position] == oid { + object, err := r.objectAtPackOffset(ctx, index, index.offsets[position]) + if err != nil { + return Object{}, err + } + object.OID = oid + return object, nil + } + } + return Object{}, fs.ErrNotExist +} + +func (r *Repository) loadPackIndexes(ctx context.Context) error { + r.mu.RLock() + loaded := r.packsLoaded + r.mu.RUnlock() + if loaded { + return nil + } + paths, err := r.objects.List(ctx, "objects/pack") + if err != nil { + return err + } + var indexes []packIndex + for _, path := range paths { + if !strings.HasSuffix(path, ".idx") { + continue + } + data, err := r.objects.Read(ctx, path) + if err != nil { + return err + } + hashes, offsets, err := parsePackIndex(data) + if err != nil { + return fmt.Errorf("parse %s: %w", path, err) + } + indexes = append(indexes, packIndex{packPath: strings.TrimSuffix(path, ".idx") + ".pack", hashes: hashes, offsets: offsets}) + } + r.mu.Lock() + if !r.packsLoaded { + r.packs, r.packsLoaded = indexes, true + } + r.mu.Unlock() + return nil +} + +func parsePackIndex(data []byte) ([]OID, []uint64, error) { + const fanoutBytes = 256 * 4 + if len(data) < 8+fanoutBytes || !bytes.Equal(data[:4], []byte{0xff, 't', 'O', 'c'}) { + return nil, nil, errors.New("unsupported or truncated pack index") + } + if version := binary.BigEndian.Uint32(data[4:8]); version != 2 { + return nil, nil, fmt.Errorf("unsupported pack index version %d", version) + } + position := 8 + count64 := uint64(binary.BigEndian.Uint32(data[position+fanoutBytes-4 : position+fanoutBytes])) + position += fanoutBytes + if count64 > uint64((len(data)-position)/20) { + return nil, nil, errors.New("truncated pack index object ids") + } + count := int(count64) + hashes := make([]OID, count) + for i := range count { + hashes[i] = OID(hex.EncodeToString(data[position+i*20 : position+(i+1)*20])) + } + position += count * 20 + if count > (len(data)-position)/4 { + return nil, nil, errors.New("truncated pack index CRC table") + } + position += count * 4 + if count > (len(data)-position)/4 { + return nil, nil, errors.New("truncated pack index offsets") + } + rawOffsets := make([]uint32, count) + for i := range count { + rawOffsets[i] = binary.BigEndian.Uint32(data[position+i*4 : position+(i+1)*4]) + } + position += count * 4 + offsets := make([]uint64, count) + for i, raw := range rawOffsets { + if raw&0x80000000 == 0 { + offsets[i] = uint64(raw) + continue + } + large := uint64(raw & 0x7fffffff) + remaining := len(data) - position + maxInt := int(^uint(0) >> 1) + if remaining < 8 || large > uint64(maxInt) || int(large) > (remaining-8)/8 { + return nil, nil, errors.New("truncated pack index large offsets") + } + start := position + int(large)*8 + offsets[i] = binary.BigEndian.Uint64(data[start : start+8]) + } + return hashes, offsets, nil +} + +func (r *Repository) objectAtPackOffset(ctx context.Context, index packIndex, offset uint64) (Object, error) { + key := fmt.Sprintf("%s:%d", index.packPath, offset) + r.mu.RLock() + if object, ok := r.offsetCache[key]; ok { + r.mu.RUnlock() + return cloneObject(object), nil + } + r.mu.RUnlock() + pack, err := r.objects.Read(ctx, index.packPath) + if err != nil { + return Object{}, err + } + object, err := r.decodePackedObject(ctx, index, pack, offset) + if err != nil { + return Object{}, err + } + r.mu.Lock() + r.offsetCache[key] = object + r.mu.Unlock() + return cloneObject(object), nil +} + +func (r *Repository) decodePackedObject(ctx context.Context, index packIndex, pack []byte, offset uint64) (Object, error) { + if len(pack) < 12 || !bytes.Equal(pack[:4], []byte("PACK")) || offset > uint64(len(pack)) { + return Object{}, errors.New("invalid pack file or object offset") + } + position := int(offset) + typ, headerLength, err := parsePackObjectHeader(pack[position:]) + if err != nil { + return Object{}, err + } + body := position + headerLength + switch typ { + case 1, 2, 3, 4: + data, err := inflatePackedData(pack[body:], r.maxObjectSize) + return Object{Type: packType(typ), Data: data}, err + case 6: + baseOffset, consumed, err := parseOFSDeltaBase(pack[body:], uint64(position)) + if err != nil { + return Object{}, err + } + delta, err := inflatePackedData(pack[body+consumed:], r.maxObjectSize) + if err != nil { + return Object{}, err + } + base, err := r.objectAtPackOffset(ctx, index, baseOffset) + if err != nil { + return Object{}, err + } + data, err := applyDeltaWithLimit(base.Data, delta, r.maxObjectSize) + return Object{Type: base.Type, Data: data}, err + case 7: + if len(pack)-body < 20 { + return Object{}, errors.New("truncated ref delta") + } + baseOID := OID(hex.EncodeToString(pack[body : body+20])) + delta, err := inflatePackedData(pack[body+20:], r.maxObjectSize) + if err != nil { + return Object{}, err + } + base, err := r.Object(ctx, baseOID) + if err != nil { + return Object{}, err + } + data, err := applyDeltaWithLimit(base.Data, delta, r.maxObjectSize) + return Object{Type: base.Type, Data: data}, err + default: + return Object{}, fmt.Errorf("unsupported pack object type %d", typ) + } +} + +func parsePackObjectHeader(data []byte) (int, int, error) { + if len(data) == 0 { + return 0, 0, errors.New("truncated pack object header") + } + byteValue := data[0] + typ, position := int((byteValue>>4)&7), 1 + for byteValue&0x80 != 0 { + if position >= len(data) { + return 0, 0, errors.New("truncated pack object header") + } + byteValue = data[position] + position++ + } + return typ, position, nil +} + +func inflatePackedData(data []byte, maxSize int64) ([]byte, error) { + reader, err := zlib.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer reader.Close() + return readBounded(reader, maxSize, "packed object") +} + +func packType(value int) ObjectType { + return map[int]ObjectType{1: CommitObject, 2: TreeObject, 3: BlobObject, 4: TagObject}[value] +} + +func parseOFSDeltaBase(data []byte, current uint64) (uint64, int, error) { + if len(data) == 0 { + return 0, 0, errors.New("truncated ofs delta") + } + currentByte := data[0] + offset, position := uint64(currentByte&0x7f), 1 + for currentByte&0x80 != 0 { + if position >= len(data) { + return 0, 0, errors.New("truncated ofs delta") + } + currentByte = data[position] + position++ + offset = ((offset + 1) << 7) | uint64(currentByte&0x7f) + } + if offset > current { + return 0, 0, errors.New("invalid ofs delta base") + } + return current - offset, position, nil +} + +func applyDelta(base, delta []byte) ([]byte, error) { + return applyDeltaWithLimit(base, delta, DefaultMaxObjectSize) +} + +func applyDeltaWithLimit(base, delta []byte, maxSize int64) ([]byte, error) { + baseSize, consumed, err := readDeltaVarint(delta) + if err != nil { + return nil, err + } + if baseSize != len(base) { + return nil, errors.New("delta base size mismatch") + } + position := consumed + resultSize, consumed, err := readDeltaVarint(delta[position:]) + if err != nil { + return nil, err + } + position += consumed + if resultSize < 0 || int64(resultSize) > maxSize { + return nil, fmt.Errorf("delta result exceeds %d bytes", maxSize) + } + capacity := resultSize + if capacity > len(base)+len(delta) { + capacity = len(base) + len(delta) + } + if capacity > 1<<20 { + capacity = 1 << 20 + } + result := make([]byte, 0, capacity) + for position < len(delta) { + opcode := delta[position] + position++ + if opcode&0x80 != 0 { + offset, size := 0, 0 + for i := range 4 { + if opcode&(1<= len(delta) { + return nil, errors.New("truncated delta copy offset") + } + offset |= int(delta[position]) << (8 * i) + position++ + } + } + for i := range 3 { + if opcode&(1<= len(delta) { + return nil, errors.New("truncated delta copy size") + } + size |= int(delta[position]) << (8 * i) + position++ + } + } + if size == 0 { + size = 0x10000 + } + if offset < 0 || size < 0 || offset > len(base)-size { + return nil, errors.New("delta copy exceeds base object") + } + if size > resultSize-len(result) { + return nil, errors.New("delta copy exceeds declared result size") + } + result = append(result, base[offset:offset+size]...) + continue + } + if opcode == 0 || int(opcode) > len(delta)-position { + return nil, errors.New("invalid delta insert") + } + if int(opcode) > resultSize-len(result) { + return nil, errors.New("delta insert exceeds declared result size") + } + result = append(result, delta[position:position+int(opcode)]...) + position += int(opcode) + } + if len(result) != resultSize { + return nil, errors.New("delta result size mismatch") + } + return result, nil +} + +func readDeltaVarint(data []byte) (int, int, error) { + value, shift := uint64(0), uint(0) + for i, current := range data { + if shift >= 63 { + return 0, 0, errors.New("delta varint overflow") + } + value |= uint64(current&0x7f) << shift + if current&0x80 == 0 { + if value > uint64(^uint(0)>>1) { + return 0, 0, errors.New("delta size overflows int") + } + return int(value), i + 1, nil + } + shift += 7 + } + return 0, 0, errors.New("truncated delta varint") +} diff --git a/repository/repository.go b/repository/repository.go new file mode 100644 index 0000000..0667059 --- /dev/null +++ b/repository/repository.go @@ -0,0 +1,594 @@ +// Package repository implements provider-neutral Git repository reads over a +// BucketGit object store. +package repository + +import ( + "bytes" + "compress/zlib" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "sort" + "strconv" + "strings" + "sync" + + "github.com/bucketgit/bgit/store" +) + +type OID string + +func ParseOID(value string) (OID, error) { + value = strings.TrimSpace(value) + if len(value) != sha1.Size*2 { + return "", fmt.Errorf("invalid SHA-1 object id %q", value) + } + if _, err := hex.DecodeString(value); err != nil { + return "", fmt.Errorf("invalid SHA-1 object id %q: %w", value, err) + } + return OID(strings.ToLower(value)), nil +} + +func (o OID) String() string { return string(o) } + +type ObjectType string + +const ( + CommitObject ObjectType = "commit" + TreeObject ObjectType = "tree" + BlobObject ObjectType = "blob" + TagObject ObjectType = "tag" +) + +type Object struct { + OID OID + Type ObjectType + Data []byte +} + +type Signature struct { + Name string + Email string + Timestamp int64 + Timezone string +} + +type Commit struct { + OID OID + Tree OID + Parents []OID + Author Signature + Committer Signature + Subject string + Body string +} + +type TreeEntry struct { + Mode string + Name string + OID OID + Type ObjectType +} + +type Option func(*Repository) + +const DefaultMaxObjectSize int64 = 128 << 20 + +// WithObjectVerification controls verification of loose-object hashes. It is +// enabled by default. +func WithObjectVerification(enabled bool) Option { + return func(r *Repository) { r.verifyObjects = enabled } +} + +// WithMaxObjectSize limits decompressed loose, packed, and delta-expanded Git +// objects. Values <= 0 restore DefaultMaxObjectSize. +func WithMaxObjectSize(bytes int64) Option { + return func(r *Repository) { + if bytes <= 0 { + bytes = DefaultMaxObjectSize + } + r.maxObjectSize = bytes + } +} + +type Repository struct { + objects store.Reader + refs store.RefStore + verifyObjects bool + maxObjectSize int64 + mu sync.RWMutex + cache map[OID]Object + packsLoaded bool + packs []packIndex + offsetCache map[string]Object +} + +func Open(objects store.Reader, refs store.RefStore, options ...Option) *Repository { + r := &Repository{objects: objects, refs: refs, verifyObjects: true, maxObjectSize: DefaultMaxObjectSize, cache: map[OID]Object{}, offsetCache: map[string]Object{}} + for _, option := range options { + option(r) + } + return r +} + +func (r *Repository) Resolve(ctx context.Context, revision string) (OID, error) { + if oid, err := ParseOID(revision); err == nil { + return oid, nil + } + base, distance := ancestorRevision(revision) + if distance > 0 { + oid, err := r.Resolve(ctx, base) + if err != nil { + return "", err + } + for range distance { + commit, err := r.Commit(ctx, oid) + if err != nil { + return "", err + } + if len(commit.Parents) == 0 { + return "", fs.ErrNotExist + } + oid = commit.Parents[0] + } + return oid, nil + } + refs, err := r.ListRefs(ctx) + if err != nil { + return "", err + } + for _, candidate := range revisionCandidates(revision) { + if oid, ok := refs[candidate]; ok { + return oid, nil + } + } + return "", fs.ErrNotExist +} + +func (r *Repository) Object(ctx context.Context, oid OID) (Object, error) { + parsed, err := ParseOID(oid.String()) + if err != nil { + return Object{}, err + } + r.mu.RLock() + if object, ok := r.cache[parsed]; ok { + r.mu.RUnlock() + return cloneObject(object), nil + } + r.mu.RUnlock() + if r.objects == nil { + return Object{}, errors.New("repository object reader is nil") + } + compressed, err := r.objects.Read(ctx, "objects/"+parsed.String()[:2]+"/"+parsed.String()[2:]) + var object Object + var raw []byte + if err == nil { + object, raw, err = decodeLooseObject(parsed, compressed, r.maxObjectSize) + } else if errors.Is(err, fs.ErrNotExist) { + object, err = r.packedObject(ctx, parsed) + if err == nil { + raw = objectBytes(object.Type, object.Data) + } + } + if err != nil { + return Object{}, err + } + if r.verifyObjects { + hash := sha1.Sum(raw) + if hex.EncodeToString(hash[:]) != parsed.String() { + return Object{}, fmt.Errorf("object %s hash mismatch", parsed) + } + } + r.mu.Lock() + r.cache[parsed] = object + r.mu.Unlock() + return cloneObject(object), nil +} + +func objectBytes(typ ObjectType, data []byte) []byte { + return append([]byte(string(typ)+" "+strconv.Itoa(len(data))+"\x00"), data...) +} + +func (r *Repository) Commit(ctx context.Context, oid OID) (Commit, error) { + object, err := r.Object(ctx, oid) + if err != nil { + return Commit{}, err + } + if object.Type == TagObject { + target, err := tagTarget(object.Data) + if err != nil { + return Commit{}, err + } + return r.Commit(ctx, target) + } + if object.Type != CommitObject { + return Commit{}, fmt.Errorf("%s is not a commit", oid) + } + return parseCommit(object.OID, object.Data) +} + +func (r *Repository) Tree(ctx context.Context, revision string) ([]TreeEntry, error) { + oid, err := r.Resolve(ctx, revision) + if err != nil { + return nil, err + } + object, err := r.Object(ctx, oid) + if err != nil { + return nil, err + } + if object.Type == CommitObject || object.Type == TagObject { + commit, err := r.Commit(ctx, oid) + if err != nil { + return nil, err + } + oid = commit.Tree + } + return r.TreeEntries(ctx, oid) +} + +func (r *Repository) TreeEntries(ctx context.Context, oid OID) ([]TreeEntry, error) { + object, err := r.Object(ctx, oid) + if err != nil { + return nil, err + } + if object.Type != TreeObject { + return nil, fmt.Errorf("%s is not a tree", oid) + } + return parseTree(object.Data) +} + +func (r *Repository) ListRefs(ctx context.Context) (map[string]OID, error) { + if r.refs != nil { + values, err := r.refs.ListRefs(ctx) + if err != nil { + return nil, err + } + return parseRefs(values), nil + } + if r.objects == nil { + return nil, errors.New("repository object reader is nil") + } + values := map[string]string{} + for _, prefix := range []string{"refs/heads/", "refs/tags/"} { + paths, err := r.objects.List(ctx, prefix) + if err != nil { + return nil, err + } + for _, path := range paths { + data, err := r.objects.Read(ctx, path) + if err != nil { + return nil, err + } + values[path] = strings.TrimSpace(string(data)) + } + } + packed, err := r.objects.Read(ctx, "packed-refs") + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + if err == nil { + for _, line := range strings.Split(string(packed), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && !strings.HasPrefix(line, "^") && !strings.HasPrefix(line, "#") { + values[fields[1]] = fields[0] + } + } + } + return parseRefs(values), nil +} + +func (r *Repository) IsAncestor(ctx context.Context, ancestor, descendant OID) (bool, error) { + seen := map[OID]struct{}{} + stack := []OID{descendant} + for len(stack) > 0 { + oid := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if oid == ancestor { + return true, nil + } + if _, ok := seen[oid]; ok { + continue + } + seen[oid] = struct{}{} + commit, err := r.Commit(ctx, oid) + if err != nil { + return false, err + } + stack = append(stack, commit.Parents...) + } + return false, nil +} + +func (r *Repository) MergeBase(ctx context.Context, a, b OID) (OID, error) { + aDepth, err := r.ancestorDepths(ctx, a) + if err != nil { + return "", err + } + bDepth, err := r.ancestorDepths(ctx, b) + if err != nil { + return "", err + } + best := OID("") + bestDistance := int(^uint(0) >> 1) + for oid, left := range aDepth { + if right, ok := bDepth[oid]; ok && left+right < bestDistance { + best, bestDistance = oid, left+right + } + } + if best == "" { + return "", fs.ErrNotExist + } + return best, nil +} + +func (r *Repository) ReachableObjects(ctx context.Context, wants, haves []OID) ([]OID, error) { + excluded := map[OID]struct{}{} + for _, oid := range haves { + if err := r.collectReachable(ctx, oid, excluded); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + included := map[OID]struct{}{} + for _, oid := range wants { + if err := r.collectReachable(ctx, oid, included); err != nil { + return nil, err + } + } + result := make([]OID, 0, len(included)) + for oid := range included { + if _, skip := excluded[oid]; !skip { + result = append(result, oid) + } + } + sort.Slice(result, func(i, j int) bool { return result[i] < result[j] }) + return result, nil +} + +func (r *Repository) collectReachable(ctx context.Context, oid OID, seen map[OID]struct{}) error { + if _, ok := seen[oid]; ok { + return nil + } + object, err := r.Object(ctx, oid) + if err != nil { + return err + } + seen[oid] = struct{}{} + switch object.Type { + case CommitObject: + commit, err := parseCommit(oid, object.Data) + if err != nil { + return err + } + if err := r.collectReachable(ctx, commit.Tree, seen); err != nil { + return err + } + for _, parent := range commit.Parents { + if err := r.collectReachable(ctx, parent, seen); err != nil { + return err + } + } + case TreeObject: + entries, err := parseTree(object.Data) + if err != nil { + return err + } + for _, entry := range entries { + if err := r.collectReachable(ctx, entry.OID, seen); err != nil { + return err + } + } + case TagObject: + target, err := tagTarget(object.Data) + if err != nil { + return err + } + return r.collectReachable(ctx, target, seen) + } + return nil +} + +func (r *Repository) ancestorDepths(ctx context.Context, head OID) (map[OID]int, error) { + depths := map[OID]int{head: 0} + queue := []OID{head} + for len(queue) > 0 { + oid := queue[0] + queue = queue[1:] + commit, err := r.Commit(ctx, oid) + if err != nil { + return nil, err + } + for _, parent := range commit.Parents { + depth := depths[oid] + 1 + if current, ok := depths[parent]; ok && current <= depth { + continue + } + depths[parent] = depth + queue = append(queue, parent) + } + } + return depths, nil +} + +func decodeLooseObject(oid OID, compressed []byte, maxSize int64) (Object, []byte, error) { + reader, err := zlib.NewReader(bytes.NewReader(compressed)) + if err != nil { + return Object{}, nil, err + } + raw, err := readBounded(reader, maxSize, "loose object") + closeErr := reader.Close() + if err != nil { + return Object{}, nil, err + } + if closeErr != nil { + return Object{}, nil, closeErr + } + nul := bytes.IndexByte(raw, 0) + if nul < 0 { + return Object{}, nil, errors.New("invalid git object header") + } + fields := strings.Fields(string(raw[:nul])) + if len(fields) != 2 { + return Object{}, nil, errors.New("invalid git object header") + } + size, err := strconv.Atoi(fields[1]) + if err != nil || size < 0 || size != len(raw)-nul-1 { + return Object{}, nil, errors.New("invalid git object size") + } + typ := ObjectType(fields[0]) + if typ != CommitObject && typ != TreeObject && typ != BlobObject && typ != TagObject { + return Object{}, nil, fmt.Errorf("unsupported git object type %q", typ) + } + data := append([]byte(nil), raw[nul+1:]...) + return Object{OID: oid, Type: typ, Data: data}, raw, nil +} + +func readBounded(reader io.Reader, maxSize int64, label string) ([]byte, error) { + if maxSize <= 0 { + maxSize = DefaultMaxObjectSize + } + data, err := io.ReadAll(io.LimitReader(reader, maxSize+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > maxSize { + return nil, fmt.Errorf("%s exceeds %d bytes", label, maxSize) + } + return data, nil +} + +func parseCommit(oid OID, data []byte) (Commit, error) { + commit := Commit{OID: oid} + header, message, _ := strings.Cut(string(data), "\n\n") + for _, line := range strings.Split(header, "\n") { + key, value, ok := strings.Cut(line, " ") + if !ok { + continue + } + switch key { + case "tree": + commit.Tree, _ = ParseOID(value) + case "parent": + if parent, err := ParseOID(value); err == nil { + commit.Parents = append(commit.Parents, parent) + } + case "author": + commit.Author = parseSignature(value) + case "committer": + commit.Committer = parseSignature(value) + } + } + if commit.Tree == "" { + return Commit{}, errors.New("commit missing valid tree") + } + commit.Body = strings.TrimRight(message, "\n") + for _, line := range strings.Split(message, "\n") { + if strings.TrimSpace(line) != "" { + commit.Subject = strings.TrimSpace(line) + break + } + } + return commit, nil +} + +func ParseCommitData(oid OID, data []byte) (Commit, error) { + if _, err := ParseOID(oid.String()); err != nil { + return Commit{}, err + } + return parseCommit(oid, data) +} + +func parseSignature(value string) Signature { + start, end := strings.LastIndex(value, " <"), strings.LastIndex(value, ">") + if start < 0 || end < start { + return Signature{Name: strings.TrimSpace(value)} + } + result := Signature{Name: strings.TrimSpace(value[:start]), Email: value[start+2 : end]} + fields := strings.Fields(value[end+1:]) + if len(fields) > 0 { + result.Timestamp, _ = strconv.ParseInt(fields[0], 10, 64) + } + if len(fields) > 1 { + result.Timezone = fields[1] + } + return result +} + +func parseTree(data []byte) ([]TreeEntry, error) { + var entries []TreeEntry + for len(data) > 0 { + space := bytes.IndexByte(data, ' ') + if space <= 0 { + return nil, errors.New("invalid tree entry mode") + } + nulRelative := bytes.IndexByte(data[space+1:], 0) + if nulRelative < 0 { + return nil, errors.New("invalid tree entry name") + } + nameEnd := space + 1 + nulRelative + if len(data) < nameEnd+1+sha1.Size { + return nil, errors.New("truncated tree entry object id") + } + mode := string(data[:space]) + oid, err := ParseOID(hex.EncodeToString(data[nameEnd+1 : nameEnd+1+sha1.Size])) + if err != nil { + return nil, err + } + typ := BlobObject + if mode == "40000" || mode == "040000" { + typ = TreeObject + } + entries = append(entries, TreeEntry{Mode: mode, Name: string(data[space+1 : nameEnd]), OID: oid, Type: typ}) + data = data[nameEnd+1+sha1.Size:] + } + return entries, nil +} + +func tagTarget(data []byte) (OID, error) { + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "object ") { + return ParseOID(strings.TrimSpace(strings.TrimPrefix(line, "object "))) + } + } + return "", errors.New("tag missing object") +} + +func parseRefs(values map[string]string) map[string]OID { + refs := make(map[string]OID, len(values)) + for ref, value := range values { + if !strings.HasPrefix(ref, "refs/") { + continue + } + if oid, err := ParseOID(value); err == nil { + refs[ref] = oid + } + } + return refs +} + +func revisionCandidates(revision string) []string { + if strings.HasPrefix(revision, "refs/") { + return []string{revision} + } + return []string{revision, "refs/heads/" + revision, "refs/tags/" + revision} +} + +func ancestorRevision(revision string) (string, int) { + base, suffix, ok := strings.Cut(revision, "~") + if !ok || base == "" || suffix == "" { + return revision, 0 + } + distance, err := strconv.Atoi(suffix) + if err != nil || distance < 1 { + return revision, 0 + } + return base, distance +} + +func cloneObject(object Object) Object { + object.Data = append([]byte(nil), object.Data...) + return object +} diff --git a/repository/repository_test.go b/repository/repository_test.go new file mode 100644 index 0000000..585dad3 --- /dev/null +++ b/repository/repository_test.go @@ -0,0 +1,169 @@ +package repository + +import ( + "archive/tar" + "bytes" + "compress/zlib" + "context" + "crypto/sha1" + "encoding/binary" + "encoding/hex" + "io" + "io/fs" + "sort" + "testing" +) + +type memoryStore map[string][]byte + +func (m memoryStore) Read(_ context.Context, path string) ([]byte, error) { + data, ok := m[path] + if !ok { + return nil, fs.ErrNotExist + } + return append([]byte(nil), data...), nil +} + +func (m memoryStore) List(_ context.Context, prefix string) ([]string, error) { + var paths []string + for path := range m { + if len(path) >= len(prefix) && path[:len(prefix)] == prefix { + paths = append(paths, path) + } + } + sort.Strings(paths) + return paths, nil +} + +func putObject(t *testing.T, store memoryStore, typ ObjectType, data []byte) OID { + t.Helper() + raw := append([]byte(string(typ)+" "+itoa(len(data))+"\x00"), data...) + hash := sha1.Sum(raw) + oid := hex.EncodeToString(hash[:]) + var compressed bytes.Buffer + w := zlib.NewWriter(&compressed) + if _, err := w.Write(raw); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + store["objects/"+oid[:2]+"/"+oid[2:]] = compressed.Bytes() + return OID(oid) +} + +func TestRepositoryResolveCommitAndTree(t *testing.T) { + objects := memoryStore{} + blob := putObject(t, objects, BlobObject, []byte("hello\n")) + treeData := append([]byte("100644 README.md\x00"), mustOIDBytes(t, blob)...) + tree := putObject(t, objects, TreeObject, treeData) + commit := putObject(t, objects, CommitObject, []byte("tree "+tree.String()+"\nauthor A 1 +0000\ncommitter A 1 +0000\n\nInitial\n")) + objects["refs/heads/main"] = []byte(commit.String() + "\n") + + repo := Open(objects, nil) + resolved, err := repo.Resolve(context.Background(), "main") + if err != nil || resolved != commit { + t.Fatalf("Resolve(main) = %s, %v", resolved, err) + } + parsed, err := repo.Commit(context.Background(), commit) + if err != nil || parsed.Tree != tree || parsed.Subject != "Initial" { + t.Fatalf("Commit = %#v, %v", parsed, err) + } + entries, err := repo.Tree(context.Background(), "main") + if err != nil || len(entries) != 1 || entries[0].OID != blob || entries[0].Name != "README.md" { + t.Fatalf("Tree = %#v, %v", entries, err) + } +} + +func TestRepositoryRejectsHashMismatch(t *testing.T) { + objects := memoryStore{} + oid := putObject(t, objects, BlobObject, []byte("hello")) + objects["objects/"+oid.String()[:2]+"/"+oid.String()[2:]][5] ^= 1 + if _, err := Open(objects, nil).Object(context.Background(), oid); err == nil { + t.Fatal("Object accepted corrupted data") + } +} + +func TestParsePackIndexV2Offsets(t *testing.T) { + const fanoutBytes = 256 * 4 + data := make([]byte, 8+fanoutBytes+20+4+4+8) + copy(data[:4], []byte{0xff, 't', 'O', 'c'}) + binary.BigEndian.PutUint32(data[4:8], 2) + for i := 0; i < 256; i++ { + binary.BigEndian.PutUint32(data[8+i*4:12+i*4], 1) + } + oidBytes := bytes.Repeat([]byte{0xab}, 20) + position := 8 + fanoutBytes + copy(data[position:position+20], oidBytes) + position += 20 + 4 + binary.BigEndian.PutUint32(data[position:position+4], 0x80000000) + position += 4 + binary.BigEndian.PutUint64(data[position:position+8], 1<<33) + hashes, offsets, err := parsePackIndex(data) + if err != nil { + t.Fatal(err) + } + if len(hashes) != 1 || hashes[0].String() != hex.EncodeToString(oidBytes) || len(offsets) != 1 || offsets[0] != 1<<33 { + t.Fatalf("hashes=%v offsets=%v", hashes, offsets) + } + if _, _, err := parsePackIndex(data[:len(data)-1]); err == nil { + t.Fatal("truncated large offset was accepted") + } +} + +func TestSnapshotDiffAndArchive(t *testing.T) { + objects := memoryStore{} + oldBlob := putObject(t, objects, BlobObject, []byte("old\n")) + oldTree := putObject(t, objects, TreeObject, append([]byte("100644 README.md\x00"), mustOIDBytes(t, oldBlob)...)) + oldCommit := putObject(t, objects, CommitObject, []byte("tree "+oldTree.String()+"\nauthor A 1 +0000\ncommitter A 1 +0000\n\nOld\n")) + newBlob := putObject(t, objects, BlobObject, []byte("new\n")) + newTree := putObject(t, objects, TreeObject, append([]byte("100755 README.md\x00"), mustOIDBytes(t, newBlob)...)) + newCommit := putObject(t, objects, CommitObject, []byte("tree "+newTree.String()+"\nparent "+oldCommit.String()+"\nauthor A 2 +0000\ncommitter A 2 +0000\n\nNew\n")) + objects["refs/heads/old"] = []byte(oldCommit.String() + "\n") + objects["refs/heads/main"] = []byte(newCommit.String() + "\n") + repo := Open(objects, nil) + snapshot, err := repo.Snapshot(t.Context(), "main") + if err != nil || string(snapshot["README.md"].Data) != "new\n" || snapshot["README.md"].Mode != "100755" { + t.Fatalf("snapshot=%#v err=%v", snapshot, err) + } + changes, err := repo.Diff(t.Context(), "old", "main") + if err != nil || len(changes) != 1 || changes[0].Status != Modified || changes[0].Path != "README.md" { + t.Fatalf("changes=%#v err=%v", changes, err) + } + var archive bytes.Buffer + if err := repo.ArchiveTar(t.Context(), "main", &archive); err != nil { + t.Fatal(err) + } + reader := tar.NewReader(&archive) + header, err := reader.Next() + if err != nil || header.Name != "README.md" || header.Mode != 0o755 { + t.Fatalf("header=%#v err=%v", header, err) + } + data, err := io.ReadAll(reader) + if err != nil || string(data) != "new\n" { + t.Fatalf("data=%q err=%v", data, err) + } +} + +func mustOIDBytes(t *testing.T, oid OID) []byte { + t.Helper() + data, err := hex.DecodeString(oid.String()) + if err != nil { + t.Fatal(err) + } + return data +} + +func itoa(value int) string { + if value == 0 { + return "0" + } + var digits [20]byte + i := len(digits) + for value > 0 { + i-- + digits[i] = byte('0' + value%10) + value /= 10 + } + return string(digits[i:]) +} diff --git a/repository/snapshot.go b/repository/snapshot.go new file mode 100644 index 0000000..8c02fb4 --- /dev/null +++ b/repository/snapshot.go @@ -0,0 +1,198 @@ +package repository + +import ( + "archive/tar" + "context" + "fmt" + "io" + "path" + "sort" + "strings" + "time" +) + +type File struct { + Path string + Mode string + OID OID + Data []byte +} + +type ChangeStatus string + +const ( + Added ChangeStatus = "added" + Modified ChangeStatus = "modified" + Deleted ChangeStatus = "deleted" +) + +type Change struct { + Path string + Status ChangeStatus + Old *File + New *File +} + +// Snapshot returns all blobs reachable from a revision, keyed by canonical +// repository-relative path. +func (r *Repository) Snapshot(ctx context.Context, revision string) (map[string]File, error) { + oid, err := r.Resolve(ctx, revision) + if err != nil { + return nil, err + } + object, err := r.Object(ctx, oid) + if err != nil { + return nil, err + } + if object.Type == CommitObject || object.Type == TagObject { + commit, err := r.Commit(ctx, oid) + if err != nil { + return nil, err + } + oid = commit.Tree + } + files := map[string]File{} + if err := r.walkTree(ctx, oid, "", files); err != nil { + return nil, err + } + return files, nil +} + +func (r *Repository) walkTree(ctx context.Context, tree OID, prefix string, files map[string]File) error { + if err := ctx.Err(); err != nil { + return err + } + entries, err := r.TreeEntries(ctx, tree) + if err != nil { + return err + } + for _, entry := range entries { + name, err := snapshotPath(prefix, entry.Name) + if err != nil { + return err + } + if entry.Type == TreeObject || strings.HasPrefix(entry.Mode, "04") || entry.Mode == "40000" { + if err := r.walkTree(ctx, entry.OID, name, files); err != nil { + return err + } + continue + } + if entry.Type != BlobObject { + continue + } + object, err := r.Object(ctx, entry.OID) + if err != nil { + return err + } + files[name] = File{Path: name, Mode: entry.Mode, OID: entry.OID, Data: append([]byte(nil), object.Data...)} + } + return nil +} + +func snapshotPath(prefix, name string) (string, error) { + if name == "" || strings.Contains(name, "/") || name == "." || name == ".." { + return "", fmt.Errorf("invalid tree entry name %q", name) + } + result := path.Join(prefix, name) + if result == "." || strings.HasPrefix(result, "../") || strings.HasPrefix(result, "/") { + return "", fmt.Errorf("invalid tree path %q", result) + } + return result, nil +} + +// Diff compares two repository snapshots without imposing a presentation +// format. Changes are sorted by path. +func (r *Repository) Diff(ctx context.Context, leftRevision, rightRevision string) ([]Change, error) { + left, err := r.Snapshot(ctx, leftRevision) + if err != nil { + return nil, err + } + right, err := r.Snapshot(ctx, rightRevision) + if err != nil { + return nil, err + } + paths := make([]string, 0, len(left)+len(right)) + seen := map[string]struct{}{} + for name := range left { + seen[name] = struct{}{} + paths = append(paths, name) + } + for name := range right { + if _, ok := seen[name]; !ok { + paths = append(paths, name) + } + } + sort.Strings(paths) + changes := make([]Change, 0) + for _, name := range paths { + oldFile, oldOK := left[name] + newFile, newOK := right[name] + if oldOK && newOK && oldFile.OID == newFile.OID && oldFile.Mode == newFile.Mode { + continue + } + change := Change{Path: name} + if oldOK { + copy := oldFile + change.Old = © + } + if newOK { + copy := newFile + change.New = © + } + switch { + case !oldOK: + change.Status = Added + case !newOK: + change.Status = Deleted + default: + change.Status = Modified + } + changes = append(changes, change) + } + return changes, nil +} + +// ArchiveTar writes a deterministic tar archive for a revision. +func (r *Repository) ArchiveTar(ctx context.Context, revision string, output io.Writer) error { + files, err := r.Snapshot(ctx, revision) + if err != nil { + return err + } + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + writer := tar.NewWriter(output) + for _, name := range names { + if err := ctx.Err(); err != nil { + _ = writer.Close() + return err + } + file := files[name] + header := &tar.Header{Name: file.Path, Mode: archiveMode(file.Mode), Size: int64(len(file.Data)), ModTime: time.Unix(0, 0), Format: tar.FormatPAX} + if file.Mode == "120000" { + header.Typeflag = tar.TypeSymlink + header.Linkname = string(file.Data) + header.Size = 0 + } + if err := writer.WriteHeader(header); err != nil { + _ = writer.Close() + return err + } + if header.Typeflag != tar.TypeSymlink { + if _, err := writer.Write(file.Data); err != nil { + _ = writer.Close() + return err + } + } + } + return writer.Close() +} + +func archiveMode(mode string) int64 { + if mode == "100755" { + return 0o755 + } + return 0o644 +} diff --git a/repository/testdata/fuzz/FuzzApplyDelta/4447bc3b1cfecf12 b/repository/testdata/fuzz/FuzzApplyDelta/4447bc3b1cfecf12 new file mode 100644 index 0000000..2b627d5 --- /dev/null +++ b/repository/testdata/fuzz/FuzzApplyDelta/4447bc3b1cfecf12 @@ -0,0 +1,3 @@ +go test fuzz v1 +[]byte("0000") +[]byte("\x04采\x87\x87\x8700") diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 0000000..b4d36b4 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,11 @@ +# BucketGit Protocol Specification + +This directory is the implementation-independent compatibility contract shared +by the Go client/local broker and the AWS/GCP JavaScript brokers. + +- `broker-v2.md` documents signed request construction and control-plane + invariants. +- `testdata/signing-v2.json` contains canonical signing vectors. + +JSON schemas for repository, auth, capabilities, refs, issues, pull requests, +and CI will be added as those models move into the public `protocol` package. diff --git a/spec/broker-v2.md b/spec/broker-v2.md new file mode 100644 index 0000000..2fcfe70 --- /dev/null +++ b/spec/broker-v2.md @@ -0,0 +1,56 @@ +# BucketGit Broker Protocol v2 + +## Signed Requests + +Broker mutations and authenticated reads use SSH-key signatures. The canonical +message is UTF-8 text with seven newline-separated fields and no trailing +newline: + +```text +bgit-broker-v2 +METHOD +/request/path +lowercase-host +unix-timestamp +nonce +sha256-lowercase-hex-of-request-body +``` + +Required headers: + +```text +X-Bgit-Signature-Version: 2 +X-Bgit-Key: +X-Bgit-Key-Fingerprint: +X-Bgit-Timestamp: +X-Bgit-Nonce: +X-Bgit-Signed-Host: +X-Bgit-Signature: +X-Bgit-Signature-Message: +``` + +The broker validates timestamp tolerance, consumes each key/nonce pair once, +reconstructs the canonical message, and verifies the SSH signature. RSA keys +use RSA-SHA2-256 where supported; Ed25519 remains supported. + +## Repository Identity + +Requests identify repositories using the JSON model in Go package `protocol`. +Logical repository name and team identity are the stable control-plane keys. +Provider, bucket, prefix, profile, and region describe physical storage and may +be hidden from normal clients by deployed brokers. + +## Object Capabilities + +Capabilities are short-lived and restricted to canonical repository-relative +paths. Clients cannot infer authorization from storage access. Deployed brokers +retain ownership of final ref updates and apply role, branch protection, PR, +and compare-and-swap policy. + +## Compatibility + +- Readers ignore unknown JSON fields. +- New optional fields use `omitempty` where omission is compatible. +- Existing fields are not renamed or removed within protocol v2. +- Missing optional endpoints produce a scoped upgrade warning and do not alter + unrelated repository data. diff --git a/spec/schemas/auth.json b/spec/schemas/auth.json new file mode 100644 index 0000000..ea11a45 --- /dev/null +++ b/spec/schemas/auth.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/auth.json","title":"BucketGit authorization request","type":"object","properties":{"repo":{"$ref":"repository.json"},"operation":{"enum":["read","write","delete","merge"]}},"required":["repo","operation"],"additionalProperties":false} diff --git a/spec/schemas/ci-run.json b/spec/schemas/ci-run.json new file mode 100644 index 0000000..72e9be7 --- /dev/null +++ b/spec/schemas/ci-run.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/ci-run.json","title":"BucketGit CI run","type":"object","properties":{"id":{"type":"integer","minimum":1},"provider":{"enum":["aws","gcp"]},"ref":{"type":"string"},"commit":{"type":"string","pattern":"^[0-9a-f]{40}$"},"config":{"type":"string"},"status":{"enum":["queued","building","finished"]},"result":{"type":"string"},"url":{"type":"string"}},"required":["id","provider","ref","commit","status"],"additionalProperties":true} diff --git a/spec/schemas/error.json b/spec/schemas/error.json new file mode 100644 index 0000000..9c233d6 --- /dev/null +++ b/spec/schemas/error.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/error.json","title":"BucketGit broker error","type":"object","properties":{"error":{"type":"string"},"code":{"type":"string"}},"required":["error"],"additionalProperties":false} diff --git a/spec/schemas/issue.json b/spec/schemas/issue.json new file mode 100644 index 0000000..74678ae --- /dev/null +++ b/spec/schemas/issue.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/issue.json","title":"BucketGit issue or board story","type":"object","properties":{"id":{"type":"integer","minimum":1},"type":{"enum":["issue","story"]},"title":{"type":"string"},"body":{"type":"string"},"status":{"type":"string"},"lane":{"enum":["backlog","ready","doing","review","done"]},"assignee":{"type":"string"},"position":{"type":"number"},"archived":{"type":"boolean"},"author":{"type":"string"},"created_at":{"type":"string"},"updated_at":{"type":"string"}},"required":["id","type","title"],"additionalProperties":true} diff --git a/spec/schemas/object-capability.json b/spec/schemas/object-capability.json new file mode 100644 index 0000000..3a11290 --- /dev/null +++ b/spec/schemas/object-capability.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/object-capability.json","title":"BucketGit object capability request","type":"object","properties":{"repo":{"$ref":"repository.json"},"path":{"type":"string","minLength":1},"operation":{"enum":["read","write","delete"]},"size":{"type":"integer","minimum":0},"resumable":{"type":"boolean"}},"required":["repo","path","operation"],"additionalProperties":false} diff --git a/spec/schemas/pull-request.json b/spec/schemas/pull-request.json new file mode 100644 index 0000000..f022fa9 --- /dev/null +++ b/spec/schemas/pull-request.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/pull-request.json","title":"BucketGit pull request","type":"object","properties":{"id":{"type":"integer","minimum":1},"title":{"type":"string"},"body":{"type":"string"},"source":{"type":"string","pattern":"^refs/"},"target":{"type":"string","pattern":"^refs/"},"status":{"enum":["open","merged","closed"]},"author":{"type":"string"},"head":{"type":"string","pattern":"^[0-9a-f]{40}$"}},"required":["id","title","source","target","status"],"additionalProperties":true} diff --git a/spec/schemas/ref-update.json b/spec/schemas/ref-update.json new file mode 100644 index 0000000..1c5ee4c --- /dev/null +++ b/spec/schemas/ref-update.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/ref-update.json","title":"BucketGit ref update","type":"object","properties":{"repo":{"$ref":"repository.json"},"ref":{"type":"string","pattern":"^refs/"},"old":{"type":"string","pattern":"^[0-9a-f]{40}$"},"new":{"type":"string","pattern":"^[0-9a-f]{40}$"},"override":{"type":"boolean"}},"required":["repo","ref","old","new"],"additionalProperties":false} diff --git a/spec/schemas/repository.json b/spec/schemas/repository.json new file mode 100644 index 0000000..114c020 --- /dev/null +++ b/spec/schemas/repository.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://bucketgit.com/spec/repository.json","title":"BucketGit repository","type":"object","properties":{"provider":{"enum":["file","s3","gcs"]},"bucket":{"type":"string"},"prefix":{"type":"string"},"origin":{"type":"string"},"logical":{"type":"string"},"host":{"type":"string"},"profile":{"type":"string"},"region":{"type":"string"},"team_id":{"type":"string"},"team_name":{"type":"string"}},"required":["provider","bucket","prefix","origin"],"additionalProperties":false} diff --git a/spec/testdata/error-responses/conflict.json b/spec/testdata/error-responses/conflict.json new file mode 100644 index 0000000..06fc088 --- /dev/null +++ b/spec/testdata/error-responses/conflict.json @@ -0,0 +1 @@ +{"status":409,"body":{"error":"stale ref","code":"conflict"},"kind":"conflict"} diff --git a/spec/testdata/error-responses/unauthorized.json b/spec/testdata/error-responses/unauthorized.json new file mode 100644 index 0000000..7723b99 --- /dev/null +++ b/spec/testdata/error-responses/unauthorized.json @@ -0,0 +1 @@ +{"status":403,"body":{"error":"write SSH signature required","code":"unauthorized"},"kind":"unauthorized"} diff --git a/spec/testdata/signing-v2.json b/spec/testdata/signing-v2.json new file mode 100644 index 0000000..ad6f5f0 --- /dev/null +++ b/spec/testdata/signing-v2.json @@ -0,0 +1,27 @@ +{ + "version": "2", + "vectors": [ + { + "name": "post-repos-mine", + "method": "POST", + "path": "/repos/mine", + "host": "Broker.Example.Com", + "timestamp": "1770000000", + "nonce": "0123456789abcdef", + "body": "{\"repo\":\"demo\"}", + "message": "bgit-broker-v2\nPOST\n/repos/mine\nbroker.example.com\n1770000000\n0123456789abcdef\n99caed5a504d67241e3f491cfb6f3088703b3291c70788b1064070947e653755", + "signatures": [ + { + "name": "ed25519", + "public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHm1Vi6P5lT5QHixEuipi6eQH4U65pW+1+DjkQutBJZk", + "signature": "AAAAC3NzaC1lZDI1NTE5AAAAQL1Dd2oLj6Fou6TiU6ydVXaCyXV3Tu2natBXNjeb6jE1uWnB1P2BvXo6Lc9rYo6ZyCkrLgukrhnalwxzYgAq/QE=" + }, + { + "name": "rsa-sha2-256", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfLiTIreqGwT7BSb7TF9t+44b3A+QCHO8htBJFGIe8y6Agzf8ku861f8PI2cWyZfhLWjC2pFg5LO7ib++NCiTH9/t04k4qFsJOCuvnfryhnfvR0MthsgoHZX1++AZ0PfO1Qp9rkUXNgqHUVx/aDGkSvkdWLcgVnvlBnDTTiPly4yjMwZhag9MQFdJD9xdwZbBBfup8vndBBkdM9V8dbjmQQaj5nHRTtCEgr6x8CMAkUWhOc8slZEQmAdWU1r96etHF4rdkYNjxV6GfksqYhydHZzoDIqQBKqfprnX9+/Cjooy4TEAoKxcxYgV6oUZtDyBHVaz3H9LCH5WhzFY318eP", + "signature": "AAAADHJzYS1zaGEyLTI1NgAAAQBNoPCVVCHdVqbbORh2X8k8fiO4IO9wDAIY7dKksoTsuLtpaxyOwwQquYN4BsN+qugzon65n+6u47wacQD1LVrYPCtK6W/rEG1MQxQZTQ2aL904SavljKFmg/m6FOl/6t8sF320EdIRGQ2i4jDIz/rQIsgmh6g1Jx5jCrt92dBrXQuKH99VJRyWrnCgGmFSOWLi0+LZ2I4fRXOgaMNMTHTXJKQsK7kRPGpWaQuQGUUq79JscW2tVsyTnre+5PsZownKC7/sISE5oNKOrZOxFCBGOlpsp8qL64lz4sA/Amdgepa8+CwFsY/J30BTcMnWjpa7KCBgCOjqjjvTGpNkvD4i" + } + ] + } + ] +} diff --git a/ssh_agent_unix.go b/ssh_agent_unix.go deleted file mode 100644 index b7a0436..0000000 --- a/ssh_agent_unix.go +++ /dev/null @@ -1,29 +0,0 @@ -//go:build !windows - -package main - -import ( - "net" - "os" - "strings" - - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" -) - -func sshAgentSigners() ([]ssh.Signer, func(), error) { - sock := strings.TrimSpace(os.Getenv("SSH_AUTH_SOCK")) - if sock == "" { - return nil, nil, nil - } - conn, err := net.Dial("unix", sock) - if err != nil { - return nil, nil, err - } - signers, err := agent.NewClient(conn).Signers() - if err != nil { - _ = conn.Close() - return nil, nil, err - } - return signers, func() { _ = conn.Close() }, nil -} diff --git a/ssh_agent_windows.go b/ssh_agent_windows.go deleted file mode 100644 index 2a9e064..0000000 --- a/ssh_agent_windows.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build windows - -package main - -import ( - "net" - "os" - "strings" - "time" - - "github.com/Microsoft/go-winio" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" -) - -const windowsOpenSSHAgentPipe = `\\.\pipe\openssh-ssh-agent` - -func sshAgentSigners() ([]ssh.Signer, func(), error) { - sock := strings.TrimSpace(os.Getenv("SSH_AUTH_SOCK")) - var conn net.Conn - var err error - timeout := 5 * time.Second - if strings.HasPrefix(sock, `\\.\pipe\`) { - conn, err = winio.DialPipe(sock, &timeout) - } else { - conn, err = winio.DialPipe(windowsOpenSSHAgentPipe, &timeout) - } - if err != nil { - return nil, nil, err - } - signers, err := agent.NewClient(conn).Signers() - if err != nil { - _ = conn.Close() - return nil, nil, err - } - return signers, func() { _ = conn.Close() }, nil -} diff --git a/store/fallback.go b/store/fallback.go new file mode 100644 index 0000000..a90a929 --- /dev/null +++ b/store/fallback.go @@ -0,0 +1,28 @@ +package store + +import ( + "context" + "errors" + "io/fs" +) + +type FallbackReader struct { + Primary Reader + Fallback Reader +} + +func (s FallbackReader) Read(ctx context.Context, path string) ([]byte, error) { + data, err := s.Primary.Read(ctx, path) + if err == nil || errors.Is(err, fs.ErrNotExist) { + return data, err + } + return s.Fallback.Read(ctx, path) +} + +func (s FallbackReader) List(ctx context.Context, prefix string) ([]string, error) { + paths, err := s.Primary.List(ctx, prefix) + if err == nil { + return paths, nil + } + return s.Fallback.List(ctx, prefix) +} diff --git a/store/fs/replace_unix.go b/store/fs/replace_unix.go new file mode 100644 index 0000000..4ff9600 --- /dev/null +++ b/store/fs/replace_unix.go @@ -0,0 +1,9 @@ +//go:build !windows + +package fs + +import "os" + +func replaceFile(source, target string) error { + return os.Rename(source, target) +} diff --git a/store/fs/replace_windows.go b/store/fs/replace_windows.go new file mode 100644 index 0000000..b0f07f0 --- /dev/null +++ b/store/fs/replace_windows.go @@ -0,0 +1,39 @@ +//go:build windows + +package fs + +import ( + "errors" + "time" + + "golang.org/x/sys/windows" +) + +func replaceFile(source, target string) error { + sourcePtr, err := windows.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPtr, err := windows.UTF16PtrFromString(target) + if err != nil { + return err + } + const attempts = 100 + for attempt := 0; attempt < attempts; attempt++ { + err = windows.MoveFileEx(sourcePtr, targetPtr, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) + if err == nil { + return nil + } + if !errors.Is(err, windows.ERROR_ACCESS_DENIED) && + !errors.Is(err, windows.ERROR_SHARING_VIOLATION) && + !errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return err + } + delay := time.Duration(attempt+1) * time.Millisecond + if delay > 10*time.Millisecond { + delay = 10 * time.Millisecond + } + time.Sleep(delay) + } + return err +} diff --git a/store/fs/store.go b/store/fs/store.go new file mode 100644 index 0000000..367969b --- /dev/null +++ b/store/fs/store.go @@ -0,0 +1,291 @@ +// Package fs implements a BucketGit object store rooted in a local directory. +package fs + +import ( + "bytes" + "context" + "errors" + "fmt" + iofs "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/bucketgit/bgit/store" +) + +type Store struct { + root string + resolvedRoot string +} + +func New(root string) (*Store, error) { + root = strings.TrimSpace(root) + if root == "" { + return nil, fmt.Errorf("filesystem store root is required") + } + abs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if err := os.MkdirAll(abs, 0o755); err != nil { + return nil, err + } + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + return nil, err + } + return &Store{root: abs, resolvedRoot: resolved}, nil +} + +func (s *Store) Read(ctx context.Context, objectPath string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + target, err := s.safePath(objectPath, false, true) + if err != nil { + return nil, err + } + data, err := os.ReadFile(target) + if errors.Is(err, iofs.ErrNotExist) { + return nil, iofs.ErrNotExist + } + return data, err +} + +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + target, err := s.safePath(prefix, true, true) + if err != nil { + return nil, err + } + var paths []string + err = filepath.WalkDir(target, func(current string, entry iofs.DirEntry, walkErr error) error { + if walkErr != nil { + if errors.Is(walkErr, iofs.ErrNotExist) { + return nil + } + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + if entry.IsDir() { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("%w: symlink %s", store.ErrInvalidPath, current) + } + rel, err := filepath.Rel(s.root, current) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }) + sort.Strings(paths) + return paths, err +} + +func (s *Store) Write(ctx context.Context, objectPath string, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } + target, err := s.safePath(objectPath, false, false) + if err != nil { + return err + } + dir := filepath.Dir(target) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if err := s.ensureResolvedWithinRoot(dir); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".bgit-write-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if err := tmp.Chmod(0o644); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return replaceFile(tmpName, target) +} + +func (s *Store) Delete(ctx context.Context, objectPath string) error { + if err := ctx.Err(); err != nil { + return err + } + target, err := s.safePath(objectPath, false, true) + if err != nil { + return err + } + err = os.Remove(target) + if errors.Is(err, iofs.ErrNotExist) { + return nil + } + return err +} + +func (s *Store) ListRefs(ctx context.Context) (map[string]string, error) { + return store.ReadRefs(ctx, s) +} + +func (s *Store) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + if err := store.ValidateRefUpdate(ref, oldOID, newOID); err != nil { + return err + } + target, err := s.safePath(ref, false, false) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + release, err := acquireFileLease(ctx, target+".bgit-lock") + if err != nil { + return err + } + defer release() + current, err := s.Read(ctx, ref) + if err != nil && !errors.Is(err, iofs.ErrNotExist) { + return err + } + currentOID := strings.TrimSpace(string(current)) + if errors.Is(err, iofs.ErrNotExist) { + currentOID = "" + } + if !(store.IsZeroOID(oldOID) && currentOID == "") && currentOID != oldOID { + return store.ErrConflict + } + if store.IsZeroOID(newOID) { + return s.Delete(ctx, ref) + } + return s.Write(ctx, ref, []byte(newOID+"\n")) +} + +func (s *Store) CompareAndSwap(ctx context.Context, objectPath string, expected store.ObjectState, replacement []byte) error { + if err := ctx.Err(); err != nil { + return err + } + target, err := s.safePath(objectPath, false, false) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + release, err := acquireFileLease(ctx, target+".bgit-cas-lock") + if err != nil { + return err + } + defer release() + current, readErr := os.ReadFile(target) + exists := readErr == nil + if readErr != nil && !errors.Is(readErr, iofs.ErrNotExist) { + return readErr + } + if exists != expected.Exists || exists && !bytes.Equal(current, expected.Data) { + return store.ErrConflict + } + return s.Write(ctx, objectPath, replacement) +} + +const fileLeaseLifetime = 2 * time.Minute + +func acquireFileLease(ctx context.Context, path string) (func(), error) { + for attempt := 0; attempt < 2; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + lock, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + if _, writeErr := fmt.Fprintf(lock, "%d\n", time.Now().UTC().Unix()); writeErr != nil { + _ = lock.Close() + _ = os.Remove(path) + return nil, writeErr + } + if closeErr := lock.Close(); closeErr != nil { + _ = os.Remove(path) + return nil, closeErr + } + return func() { _ = os.Remove(path) }, nil + } + if !errors.Is(err, os.ErrExist) { + return nil, err + } + info, statErr := os.Stat(path) + if statErr != nil || time.Since(info.ModTime()) <= fileLeaseLifetime { + return nil, store.ErrConflict + } + if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return nil, store.ErrConflict + } + } + return nil, store.ErrConflict +} + +var _ store.RefStore = (*Store)(nil) +var _ store.CompareAndSwapper = (*Store)(nil) + +func (s *Store) safePath(value string, allowEmpty, resolveTarget bool) (string, error) { + rel, err := store.ValidatePath(value, allowEmpty) + if err != nil { + return "", err + } + target := s.root + if rel != "" { + target = filepath.Join(s.root, filepath.FromSlash(rel)) + } + if err := ensureWithinRoot(s.root, target); err != nil { + return "", err + } + if resolveTarget { + resolved, err := filepath.EvalSymlinks(target) + if err == nil { + if err := ensureWithinRoot(s.resolvedRoot, resolved); err != nil { + return "", err + } + } else if !errors.Is(err, iofs.ErrNotExist) { + return "", err + } + } + return target, nil +} + +func (s *Store) ensureResolvedWithinRoot(dir string) error { + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + return err + } + return ensureWithinRoot(s.resolvedRoot, resolved) +} + +func ensureWithinRoot(root, target string) error { + rel, err := filepath.Rel(root, target) + if err != nil { + return err + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return fmt.Errorf("%w: path escapes store root", store.ErrInvalidPath) + } + return nil +} diff --git a/store/fs/store_test.go b/store/fs/store_test.go new file mode 100644 index 0000000..a57dd03 --- /dev/null +++ b/store/fs/store_test.go @@ -0,0 +1,97 @@ +package fs + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/bucketgit/bgit/store" + "github.com/bucketgit/bgit/store/storetest" +) + +func TestStoreContract(t *testing.T) { + storetest.Run(t, func(t *testing.T) store.Writer { + t.Helper() + s, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return s + }) +} + +func TestCompareAndSwapRecoversStaleLease(t *testing.T) { + root := t.TempDir() + backend, err := New(root) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "objects", "state.json") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + lockPath := path + ".bgit-cas-lock" + if err := os.WriteFile(lockPath, []byte("abandoned"), 0o600); err != nil { + t.Fatal(err) + } + stale := time.Now().Add(-fileLeaseLifetime - time.Minute) + if err := os.Chtimes(lockPath, stale, stale); err != nil { + t.Fatal(err) + } + if err := backend.CompareAndSwap(t.Context(), "objects/state.json", store.ObjectState{}, []byte("recovered")); err != nil { + t.Fatal(err) + } +} + +func TestCompareAndSwapContract(t *testing.T) { + storetest.RunCompareAndSwap(t, func(t *testing.T) store.Writer { + backend, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return backend + }) +} + +func TestCompareAndSwapRef(t *testing.T) { + backend, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + one := "0123456789abcdef0123456789abcdef01234567" + two := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + if err := backend.CompareAndSwapRef(t.Context(), "refs/heads/main", "", one); err != nil { + t.Fatal(err) + } + if err := backend.CompareAndSwapRef(t.Context(), "refs/heads/main", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", two); !errors.Is(err, store.ErrConflict) { + t.Fatalf("stale update error = %v", err) + } + if err := backend.CompareAndSwapRef(t.Context(), "refs/heads/main", one, two); err != nil { + t.Fatal(err) + } + refs, err := backend.ListRefs(t.Context()) + if err != nil || refs["refs/heads/main"] != two { + t.Fatalf("refs = %#v, %v", refs, err) + } +} + +func TestStoreRejectsSymlinkEscape(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation commonly requires elevated privileges on Windows") + } + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "objects")); err != nil { + t.Fatal(err) + } + s, err := New(root) + if err != nil { + t.Fatal(err) + } + if err := s.Write(t.Context(), "objects/ab/cdef", []byte("bad")); err == nil { + t.Fatal("expected symlink escape write to fail") + } +} diff --git a/store/gcs/storage_api.go b/store/gcs/storage_api.go new file mode 100644 index 0000000..79627cf --- /dev/null +++ b/store/gcs/storage_api.go @@ -0,0 +1,87 @@ +package gcs + +import ( + "context" + "errors" + "io" + "sort" + + "cloud.google.com/go/storage" + "github.com/bucketgit/bgit/store" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" +) + +type storageAPI struct{ client *storage.Client } + +func (a *storageAPI) Read(ctx context.Context, bucket, name string, generation int64) ([]byte, error) { + object := a.client.Bucket(bucket).Object(name) + if generation > 0 { + object = object.Generation(generation) + } + reader, err := object.NewReader(ctx) + if err != nil { + return nil, translateError(err) + } + defer reader.Close() + return io.ReadAll(reader) +} +func (a *storageAPI) List(ctx context.Context, bucket, prefix string) ([]string, error) { + it := a.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: prefix}) + var names []string + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return nil, translateError(err) + } + names = append(names, attrs.Name) + } + sort.Strings(names) + return names, nil +} +func (a *storageAPI) Write(ctx context.Context, bucket, name string, data []byte, generation *int64) error { + object := a.client.Bucket(bucket).Object(name) + if generation != nil { + conditions := storage.Conditions{GenerationMatch: *generation} + if *generation < 0 { + conditions = storage.Conditions{DoesNotExist: true} + } + object = object.If(conditions) + } + writer := object.NewWriter(ctx) + if _, err := writer.Write(data); err != nil { + _ = writer.Close() + return translateError(err) + } + return translateError(writer.Close()) +} +func (a *storageAPI) Delete(ctx context.Context, bucket, name string, generation *int64) error { + object := a.client.Bucket(bucket).Object(name) + if generation != nil { + object = object.If(storage.Conditions{GenerationMatch: *generation}) + } + return translateError(object.Delete(ctx)) +} +func (a *storageAPI) Attrs(ctx context.Context, bucket, name string) (ObjectAttrs, error) { + attrs, err := a.client.Bucket(bucket).Object(name).Attrs(ctx) + if err != nil { + return ObjectAttrs{}, translateError(err) + } + return ObjectAttrs{Generation: attrs.Generation}, nil +} +func translateError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, storage.ErrObjectNotExist) { + return storage.ErrObjectNotExist + } + var apiErr *googleapi.Error + if errors.As(err, &apiErr) && (apiErr.Code == 409 || apiErr.Code == 412) { + return store.ErrConflict + } + return err +} diff --git a/store/gcs/store.go b/store/gcs/store.go new file mode 100644 index 0000000..9fd5272 --- /dev/null +++ b/store/gcs/store.go @@ -0,0 +1,271 @@ +// Package gcs implements a BucketGit object store backed by Google Cloud Storage. +package gcs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "sort" + "strings" + + "cloud.google.com/go/storage" + "github.com/bucketgit/bgit/store" + "google.golang.org/api/option" +) + +type ObjectAttrs struct{ Generation int64 } + +type API interface { + Read(context.Context, string, string, int64) ([]byte, error) + List(context.Context, string, string) ([]string, error) + Write(context.Context, string, string, []byte, *int64) error + Delete(context.Context, string, string, *int64) error + Attrs(context.Context, string, string) (ObjectAttrs, error) +} + +type Store struct { + client API + bucket string + prefix string + close func() error +} + +type Options struct { + Bucket string + Prefix string + CredentialsFile string +} + +func Load(ctx context.Context, options Options) (*Store, error) { + var clientOptions []option.ClientOption + if path := strings.TrimSpace(options.CredentialsFile); path != "" { + clientOptions = append(clientOptions, option.WithCredentialsFile(path)) + } + client, err := storage.NewClient(ctx, clientOptions...) + if err != nil { + return nil, fmt.Errorf("create GCS client: %w", err) + } + backend, err := New(client, options.Bucket, options.Prefix) + if err != nil { + _ = client.Close() + return nil, err + } + backend.close = client.Close + return backend, nil +} + +func (s *Store) Close() error { + if s == nil || s.close == nil { + return nil + } + return s.close() +} + +func New(client *storage.Client, bucket, prefix string) (*Store, error) { + if client == nil { + return nil, fmt.Errorf("GCS client is required") + } + return NewWithAPI(&storageAPI{client: client}, bucket, prefix) +} + +func NewWithAPI(client API, bucket, prefix string) (*Store, error) { + if client == nil { + return nil, fmt.Errorf("GCS client is required") + } + bucket = strings.TrimSpace(bucket) + if bucket == "" { + return nil, fmt.Errorf("GCS bucket is required") + } + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + if prefix != "" { + if _, err := store.ValidatePath(prefix, false); err != nil { + return nil, fmt.Errorf("GCS prefix: %w", err) + } + } + return &Store{client: client, bucket: bucket, prefix: prefix}, nil +} + +func (s *Store) Read(ctx context.Context, path string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + name, err := s.objectName(path, false) + if err != nil { + return nil, err + } + data, err := s.client.Read(ctx, s.bucket, name, 0) + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, storage.ErrObjectNotExist) { + return nil, fs.ErrNotExist + } + if err != nil { + return nil, s.accessError("read", name, err) + } + return data, nil +} +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + query, err := s.objectName(prefix, true) + if err != nil { + return nil, err + } + if query != "" && !strings.HasSuffix(query, "/") { + query += "/" + } + names, err := s.client.List(ctx, s.bucket, query) + if err != nil { + return nil, s.accessError("list", query, err) + } + var paths []string + for _, name := range names { + rel := strings.TrimPrefix(name, objectPrefix(s.prefix)) + if rel != "" && !strings.HasSuffix(rel, "/") { + paths = append(paths, rel) + } + } + sort.Strings(paths) + return paths, nil +} +func (s *Store) Write(ctx context.Context, path string, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } + name, err := s.objectName(path, false) + if err != nil { + return err + } + if err := s.client.Write(ctx, s.bucket, name, data, nil); err != nil { + return s.accessError("write", name, err) + } + return nil +} +func (s *Store) Delete(ctx context.Context, path string) error { + if err := ctx.Err(); err != nil { + return err + } + name, err := s.objectName(path, false) + if err != nil { + return err + } + err = s.client.Delete(ctx, s.bucket, name, nil) + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, storage.ErrObjectNotExist) { + return nil + } + if err != nil { + return s.accessError("delete", name, err) + } + return nil +} +func (s *Store) ListRefs(ctx context.Context) (map[string]string, error) { + return store.ReadRefs(ctx, s) +} + +func (s *Store) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + if err := store.ValidateRefUpdate(ref, oldOID, newOID); err != nil { + return err + } + name, err := s.objectName(ref, false) + if err != nil { + return err + } + attrs, err := s.client.Attrs(ctx, s.bucket, name) + notFound := errors.Is(err, fs.ErrNotExist) || errors.Is(err, storage.ErrObjectNotExist) + if err != nil && !notFound { + return s.accessError("read ref attributes", name, err) + } + current := "" + if !notFound { + data, err := s.client.Read(ctx, s.bucket, name, attrs.Generation) + if err != nil { + return s.accessError("read ref", name, err) + } + current = strings.TrimSpace(string(data)) + } + if !(store.IsZeroOID(oldOID) && notFound) && current != oldOID { + return store.ErrConflict + } + generation := attrs.Generation + if notFound { + generation = -1 + } + if store.IsZeroOID(newOID) { + if notFound { + return nil + } + err = s.client.Delete(ctx, s.bucket, name, &generation) + } else { + err = s.client.Write(ctx, s.bucket, name, []byte(newOID+"\n"), &generation) + } + if errors.Is(err, store.ErrConflict) { + return store.ErrConflict + } + if err != nil { + return s.accessError("update ref", name, err) + } + return nil +} + +func (s *Store) CompareAndSwap(ctx context.Context, path string, expected store.ObjectState, replacement []byte) error { + if err := ctx.Err(); err != nil { + return err + } + name, err := s.objectName(path, false) + if err != nil { + return err + } + attrs, err := s.client.Attrs(ctx, s.bucket, name) + notFound := errors.Is(err, fs.ErrNotExist) || errors.Is(err, storage.ErrObjectNotExist) + if err != nil && !notFound { + return s.accessError("read conditional object attributes", name, err) + } + var current []byte + if !notFound { + current, err = s.client.Read(ctx, s.bucket, name, attrs.Generation) + if err != nil { + return s.accessError("read conditional object", name, err) + } + } + if (!notFound) != expected.Exists || !notFound && !bytes.Equal(current, expected.Data) { + return store.ErrConflict + } + generation := attrs.Generation + if notFound { + generation = -1 + } + if err := s.client.Write(ctx, s.bucket, name, replacement, &generation); errors.Is(err, store.ErrConflict) { + return store.ErrConflict + } else if err != nil { + return s.accessError("replace conditional object", name, err) + } + return nil +} + +func (s *Store) objectName(value string, allowEmpty bool) (string, error) { + rel, err := store.ValidatePath(value, allowEmpty) + if err != nil { + return "", err + } + if s.prefix == "" { + return rel, nil + } + if rel == "" { + return s.prefix, nil + } + return s.prefix + "/" + rel, nil +} +func (s *Store) accessError(action, object string, err error) error { + return fmt.Errorf("%s gs://%s/%s: %w", action, s.bucket, object, err) +} +func objectPrefix(value string) string { + value = strings.Trim(value, "/") + if value == "" { + return "" + } + return value + "/" +} + +var _ store.RefStore = (*Store)(nil) +var _ store.CompareAndSwapper = (*Store)(nil) diff --git a/store/gcs/store_test.go b/store/gcs/store_test.go new file mode 100644 index 0000000..977995e --- /dev/null +++ b/store/gcs/store_test.go @@ -0,0 +1,139 @@ +package gcs + +import ( + "context" + "errors" + "io/fs" + "sort" + "strings" + "sync" + "testing" + + "cloud.google.com/go/storage" + "github.com/bucketgit/bgit/store" + "github.com/bucketgit/bgit/store/storetest" +) + +type fakeObject struct { + data []byte + generation int64 +} +type fakeGCS struct { + mu sync.Mutex + next int64 + objects map[string]fakeObject +} + +func (f *fakeGCS) Read(_ context.Context, _, name string, generation int64) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + object, ok := f.objects[name] + if !ok { + return nil, fs.ErrNotExist + } + if generation > 0 && object.generation != generation { + return nil, store.ErrConflict + } + return append([]byte(nil), object.data...), nil +} +func (f *fakeGCS) List(_ context.Context, _, prefix string) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var names []string + for name := range f.objects { + if strings.HasPrefix(name, prefix) { + names = append(names, name) + } + } + sort.Strings(names) + return names, nil +} +func (f *fakeGCS) Write(_ context.Context, _, name string, data []byte, generation *int64) error { + f.mu.Lock() + defer f.mu.Unlock() + current, exists := f.objects[name] + if generation != nil { + if *generation < 0 && exists { + return store.ErrConflict + } + if *generation >= 0 && (!exists || current.generation != *generation) { + return store.ErrConflict + } + } + f.next++ + f.objects[name] = fakeObject{data: append([]byte(nil), data...), generation: f.next} + return nil +} +func (f *fakeGCS) Delete(_ context.Context, _, name string, generation *int64) error { + f.mu.Lock() + defer f.mu.Unlock() + current, exists := f.objects[name] + if !exists { + return fs.ErrNotExist + } + if generation != nil && current.generation != *generation { + return store.ErrConflict + } + delete(f.objects, name) + return nil +} +func (f *fakeGCS) Attrs(_ context.Context, _, name string) (ObjectAttrs, error) { + f.mu.Lock() + defer f.mu.Unlock() + object, ok := f.objects[name] + if !ok { + return ObjectAttrs{}, fs.ErrNotExist + } + return ObjectAttrs{Generation: object.generation}, nil +} + +func TestStoreContract(t *testing.T) { + storetest.Run(t, func(t *testing.T) store.Writer { + backend, err := NewWithAPI(&fakeGCS{objects: map[string]fakeObject{}}, "bucket", "repo.git") + if err != nil { + t.Fatal(err) + } + return backend + }) +} + +func TestCompareAndSwapContract(t *testing.T) { + storetest.RunCompareAndSwap(t, func(t *testing.T) store.Writer { + backend, err := NewWithAPI(&fakeGCS{objects: map[string]fakeObject{}}, "bucket", "repo.git") + if err != nil { + t.Fatal(err) + } + return backend + }) +} + +func TestNewValidatesOptions(t *testing.T) { + if _, err := New(nil, "bucket", "repo.git"); err == nil { + t.Fatal("expected nil client error") + } + client := new(storage.Client) + if _, err := New(client, "", "repo.git"); err == nil { + t.Fatal("expected empty bucket error") + } + if _, err := New(client, "bucket", "../repo.git"); !errors.Is(err, store.ErrInvalidPath) { + t.Fatalf("prefix error = %v", err) + } +} + +func TestCompareAndSwapRef(t *testing.T) { + backend, err := NewWithAPI(&fakeGCS{objects: map[string]fakeObject{}}, "bucket", "repo.git") + if err != nil { + t.Fatal(err) + } + one := "0123456789abcdef0123456789abcdef01234567" + two := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + if err := backend.CompareAndSwapRef(t.Context(), "refs/heads/main", "", one); err != nil { + t.Fatal(err) + } + if err := backend.CompareAndSwapRef(t.Context(), "refs/heads/main", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", two); !errors.Is(err, store.ErrConflict) { + t.Fatalf("stale=%v", err) + } + if err := backend.CompareAndSwapRef(t.Context(), "refs/heads/main", one, two); err != nil { + t.Fatal(err) + } +} diff --git a/store/path.go b/store/path.go new file mode 100644 index 0000000..1ecd12f --- /dev/null +++ b/store/path.go @@ -0,0 +1,51 @@ +package store + +import ( + "fmt" + "net/url" + "path" + "strings" +) + +func ValidatePath(value string, allowEmpty bool) (string, error) { + if strings.ContainsRune(value, '\\') { + return "", fmt.Errorf("%w: backslashes are not allowed", ErrInvalidPath) + } + if decoded, err := url.PathUnescape(value); err != nil { + return "", fmt.Errorf("%w: invalid percent encoding", ErrInvalidPath) + } else if decoded != value { + decoded = strings.ReplaceAll(decoded, "\\", "/") + decodedClean := path.Clean(decoded) + if strings.HasPrefix(decoded, "/") || decodedClean != decoded || decodedClean == ".." || strings.HasPrefix(decodedClean, "../") || strings.Contains(decoded, "//") { + return "", fmt.Errorf("%w: encoded path escapes store root", ErrInvalidPath) + } + } + value = strings.TrimSpace(value) + if value == "" { + if allowEmpty { + return "", nil + } + return "", fmt.Errorf("%w: path is empty", ErrInvalidPath) + } + if strings.HasPrefix(value, "/") { + return "", fmt.Errorf("%w: path must be relative", ErrInvalidPath) + } + hasPrefixSeparator := allowEmpty && strings.HasSuffix(value, "/") + clean := path.Clean(value) + if clean == "." { + if allowEmpty { + return "", nil + } + return "", fmt.Errorf("%w: path is empty", ErrInvalidPath) + } + if clean == ".." || strings.HasPrefix(clean, "../") { + return "", fmt.Errorf("%w: path escapes store root", ErrInvalidPath) + } + if clean != value && (!hasPrefixSeparator || clean+"/" != value) { + return "", fmt.Errorf("%w: path is not canonical", ErrInvalidPath) + } + if hasPrefixSeparator { + return clean + "/", nil + } + return clean, nil +} diff --git a/store/path_test.go b/store/path_test.go new file mode 100644 index 0000000..7eb362a --- /dev/null +++ b/store/path_test.go @@ -0,0 +1,25 @@ +package store + +import ( + "errors" + "testing" +) + +func TestValidatePath(t *testing.T) { + for _, valid := range []string{"objects/ab/cdef", "refs/heads/main", "HEAD"} { + if got, err := ValidatePath(valid, false); err != nil || got != valid { + t.Fatalf("ValidatePath(%q) = %q, %v", valid, got, err) + } + } + if got, err := ValidatePath("", true); err != nil || got != "" { + t.Fatalf("empty prefix = %q, %v", got, err) + } + if got, err := ValidatePath("refs/", true); err != nil || got != "refs/" { + t.Fatalf("prefix separator = %q, %v", got, err) + } + for _, invalid := range []string{"", "/etc/passwd", "../secret", "objects/../../secret", "objects\\secret", "objects//secret", "./HEAD"} { + if _, err := ValidatePath(invalid, false); !errors.Is(err, ErrInvalidPath) { + t.Fatalf("ValidatePath(%q) error = %v", invalid, err) + } + } +} diff --git a/store/refs.go b/store/refs.go new file mode 100644 index 0000000..b4f62a9 --- /dev/null +++ b/store/refs.go @@ -0,0 +1,67 @@ +package store + +import ( + "context" + "errors" + "io/fs" + "strings" +) + +func ReadRefs(ctx context.Context, reader Reader) (map[string]string, error) { + refs := map[string]string{} + for _, prefix := range []string{"refs/heads/", "refs/tags/"} { + paths, err := reader.List(ctx, prefix) + if err != nil { + return nil, err + } + for _, path := range paths { + data, err := reader.Read(ctx, path) + if err != nil { + return nil, err + } + if oid := strings.TrimSpace(string(data)); ValidSHA1(oid) { + refs[path] = oid + } + } + } + packed, err := reader.Read(ctx, "packed-refs") + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + if err == nil { + for _, line := range strings.Split(string(packed), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && ValidSHA1(fields[0]) && strings.HasPrefix(fields[1], "refs/") { + refs[fields[1]] = fields[0] + } + } + } + return refs, nil +} + +func ValidSHA1(value string) bool { + value = strings.TrimSpace(value) + if len(value) != 40 { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} + +func ValidateRefUpdate(ref, oldOID, newOID string) error { + if _, err := ValidatePath(ref, false); err != nil || !strings.HasPrefix(ref, "refs/") { + return ErrInvalidPath + } + for _, oid := range []string{oldOID, newOID} { + if oid != "" && oid != strings.Repeat("0", 40) && !ValidSHA1(oid) { + return ErrInvalidPath + } + } + return nil +} + +func IsZeroOID(value string) bool { return value == "" || value == strings.Repeat("0", 40) } diff --git a/store/s3/store.go b/store/s3/store.go new file mode 100644 index 0000000..4f9fe8a --- /dev/null +++ b/store/s3/store.go @@ -0,0 +1,309 @@ +// Package s3 implements a BucketGit object store backed by AWS S3. +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/bucketgit/bgit/store" +) + +type API interface { + GetObject(context.Context, *awss3.GetObjectInput, ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) + ListObjectsV2(context.Context, *awss3.ListObjectsV2Input, ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) + PutObject(context.Context, *awss3.PutObjectInput, ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) + DeleteObject(context.Context, *awss3.DeleteObjectInput, ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) +} + +type Store struct { + client API + bucket string + prefix string +} + +type Options struct { + Bucket string + Prefix string + Profile string + Region string +} + +// Load constructs an S3 store using the standard AWS credential chain. +func Load(ctx context.Context, options Options) (*Store, error) { + var loaders []func(*awsconfig.LoadOptions) error + if region := strings.TrimSpace(options.Region); region != "" { + loaders = append(loaders, awsconfig.WithRegion(region)) + } + if profile := strings.TrimSpace(options.Profile); profile != "" { + loaders = append(loaders, awsconfig.WithSharedConfigProfile(profile)) + } + configuration, err := awsconfig.LoadDefaultConfig(ctx, loaders...) + if err != nil { + return nil, fmt.Errorf("load AWS configuration: %w", err) + } + return New(awss3.NewFromConfig(configuration), options.Bucket, options.Prefix) +} + +func New(client API, bucket, prefix string) (*Store, error) { + if client == nil { + return nil, fmt.Errorf("S3 client is required") + } + bucket = strings.TrimSpace(bucket) + if bucket == "" { + return nil, fmt.Errorf("S3 bucket is required") + } + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + if prefix != "" { + if _, err := store.ValidatePath(prefix, false); err != nil { + return nil, fmt.Errorf("S3 prefix: %w", err) + } + } + return &Store{client: client, bucket: bucket, prefix: prefix}, nil +} + +func (s *Store) Read(ctx context.Context, objectPath string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + key, err := s.objectName(objectPath, false) + if err != nil { + return nil, err + } + out, err := s.client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil { + if isNotFound(err) { + return nil, fs.ErrNotExist + } + return nil, s.accessError("read", key, err) + } + defer out.Body.Close() + return io.ReadAll(out.Body) +} + +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + queryPrefix, err := s.objectName(prefix, true) + if err != nil { + return nil, err + } + if queryPrefix != "" && !strings.HasSuffix(queryPrefix, "/") { + queryPrefix += "/" + } + var paths []string + var token *string + for { + out, err := s.client.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(queryPrefix), + ContinuationToken: token, + }) + if err != nil { + return nil, s.accessError("list", queryPrefix, err) + } + for _, object := range out.Contents { + key := aws.ToString(object.Key) + rel := strings.TrimPrefix(key, objectPrefix(s.prefix)) + if rel != "" && !strings.HasSuffix(rel, "/") { + paths = append(paths, rel) + } + } + if !aws.ToBool(out.IsTruncated) || out.NextContinuationToken == nil { + break + } + token = out.NextContinuationToken + } + sort.Strings(paths) + return paths, nil +} + +func (s *Store) Write(ctx context.Context, objectPath string, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } + key, err := s.objectName(objectPath, false) + if err != nil { + return err + } + _, err = s.client.PutObject(ctx, &awss3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + }) + if err != nil { + return s.accessError("write", key, err) + } + return nil +} + +func (s *Store) Delete(ctx context.Context, objectPath string) error { + if err := ctx.Err(); err != nil { + return err + } + key, err := s.objectName(objectPath, false) + if err != nil { + return err + } + _, err = s.client.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil && !isNotFound(err) { + return s.accessError("delete", key, err) + } + return nil +} + +func (s *Store) ListRefs(ctx context.Context) (map[string]string, error) { + return store.ReadRefs(ctx, s) +} + +func (s *Store) CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error { + if err := store.ValidateRefUpdate(ref, oldOID, newOID); err != nil { + return err + } + key, err := s.objectName(ref, false) + if err != nil { + return err + } + current, err := s.client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + notFound := isNotFound(err) + if err != nil && !notFound { + return s.accessError("read ref", key, err) + } + currentOID, etag := "", "" + if !notFound { + data, readErr := io.ReadAll(current.Body) + closeErr := current.Body.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + currentOID, etag = strings.TrimSpace(string(data)), aws.ToString(current.ETag) + } + if !(store.IsZeroOID(oldOID) && notFound) && currentOID != oldOID { + return store.ErrConflict + } + if store.IsZeroOID(newOID) { + if notFound { + return nil + } + _, err = s.client.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag)}) + } else { + input := &awss3.PutObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), Body: bytes.NewReader([]byte(newOID + "\n"))} + if notFound { + input.IfNoneMatch = aws.String("*") + } else { + input.IfMatch = aws.String(etag) + } + _, err = s.client.PutObject(ctx, input) + } + if isConflict(err) { + return store.ErrConflict + } + if err != nil { + return s.accessError("update ref", key, err) + } + return nil +} + +func (s *Store) CompareAndSwap(ctx context.Context, objectPath string, expected store.ObjectState, replacement []byte) error { + if err := ctx.Err(); err != nil { + return err + } + key, err := s.objectName(objectPath, false) + if err != nil { + return err + } + current, err := s.client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + notFound := isNotFound(err) + if err != nil && !notFound { + return s.accessError("read conditional object", key, err) + } + var data []byte + etag := "" + if !notFound { + data, err = io.ReadAll(current.Body) + closeErr := current.Body.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + etag = aws.ToString(current.ETag) + } + if (!notFound) != expected.Exists || !notFound && !bytes.Equal(data, expected.Data) { + return store.ErrConflict + } + input := &awss3.PutObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), Body: bytes.NewReader(replacement)} + if notFound { + input.IfNoneMatch = aws.String("*") + } else { + input.IfMatch = aws.String(etag) + } + if _, err := s.client.PutObject(ctx, input); isConflict(err) { + return store.ErrConflict + } else if err != nil { + return s.accessError("replace conditional object", key, err) + } + return nil +} + +func (s *Store) objectName(value string, allowEmpty bool) (string, error) { + rel, err := store.ValidatePath(value, allowEmpty) + if err != nil { + return "", err + } + if s.prefix == "" { + return rel, nil + } + if rel == "" { + return s.prefix, nil + } + return s.prefix + "/" + rel, nil +} + +func (s *Store) accessError(action, key string, err error) error { + return fmt.Errorf("%s s3://%s/%s: %w", action, s.bucket, key, err) +} + +func objectPrefix(value string) string { + value = strings.Trim(value, "/") + if value == "" { + return "" + } + return value + "/" +} + +func isNotFound(err error) bool { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchBucket", "NoSuchKey", "NotFound", "404": + return true + } + } + return false +} + +func isConflict(err error) bool { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + return apiErr.ErrorCode() == "PreconditionFailed" || apiErr.ErrorCode() == "ConditionalRequestConflict" + } + return false +} + +var _ store.RefStore = (*Store)(nil) +var _ store.CompareAndSwapper = (*Store)(nil) diff --git a/store/s3/store_test.go b/store/s3/store_test.go new file mode 100644 index 0000000..796df52 --- /dev/null +++ b/store/s3/store_test.go @@ -0,0 +1,112 @@ +package s3 + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/bucketgit/bgit/store" + "github.com/bucketgit/bgit/store/storetest" +) + +type fakeS3 struct { + mu sync.Mutex + objects map[string][]byte +} + +func (f *fakeS3) GetObject(_ context.Context, in *awss3.GetObjectInput, _ ...func(*awss3.Options)) (*awss3.GetObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + data, ok := f.objects[aws.ToString(in.Key)] + if !ok { + return nil, &smithy.GenericAPIError{Code: "NoSuchKey", Message: "missing"} + } + return &awss3.GetObjectOutput{Body: io.NopCloser(strings.NewReader(string(data))), ETag: aws.String(fakeETag(data))}, nil +} + +func (f *fakeS3) ListObjectsV2(_ context.Context, in *awss3.ListObjectsV2Input, _ ...func(*awss3.Options)) (*awss3.ListObjectsV2Output, error) { + f.mu.Lock() + defer f.mu.Unlock() + var keys []string + for key := range f.objects { + if strings.HasPrefix(key, aws.ToString(in.Prefix)) { + keys = append(keys, key) + } + } + sort.Strings(keys) + out := &awss3.ListObjectsV2Output{} + for _, key := range keys { + out.Contents = append(out.Contents, types.Object{Key: aws.String(key)}) + } + return out, nil +} + +func (f *fakeS3) PutObject(_ context.Context, in *awss3.PutObjectInput, _ ...func(*awss3.Options)) (*awss3.PutObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + current, exists := f.objects[aws.ToString(in.Key)] + if aws.ToString(in.IfNoneMatch) == "*" && exists || in.IfMatch != nil && (!exists || aws.ToString(in.IfMatch) != fakeETag(current)) { + return nil, &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "stale"} + } + data, err := io.ReadAll(in.Body) + if err != nil { + return nil, err + } + f.objects[aws.ToString(in.Key)] = data + return &awss3.PutObjectOutput{}, nil +} + +func fakeETag(data []byte) string { + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]) +} + +func (f *fakeS3) DeleteObject(_ context.Context, in *awss3.DeleteObjectInput, _ ...func(*awss3.Options)) (*awss3.DeleteObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.objects, aws.ToString(in.Key)) + return &awss3.DeleteObjectOutput{}, nil +} + +func TestStoreContract(t *testing.T) { + storetest.Run(t, func(t *testing.T) store.Writer { + t.Helper() + s, err := New(&fakeS3{objects: map[string][]byte{}}, "bucket", "repo.git") + if err != nil { + t.Fatal(err) + } + return s + }) +} + +func TestCompareAndSwapContract(t *testing.T) { + storetest.RunCompareAndSwap(t, func(t *testing.T) store.Writer { + backend, err := New(&fakeS3{objects: map[string][]byte{}}, "bucket", "repo.git") + if err != nil { + t.Fatal(err) + } + return backend + }) +} + +func TestNewValidatesOptions(t *testing.T) { + if _, err := New(nil, "bucket", "repo.git"); err == nil { + t.Fatal("expected nil client error") + } + if _, err := New(&fakeS3{}, "", "repo.git"); err == nil { + t.Fatal("expected empty bucket error") + } + if _, err := New(&fakeS3{}, "bucket", "../repo.git"); !errors.Is(err, store.ErrInvalidPath) { + t.Fatalf("prefix error = %v", err) + } +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..7c821c7 --- /dev/null +++ b/store/store.go @@ -0,0 +1,47 @@ +// Package store defines provider-neutral storage contracts used by BucketGit +// repositories and brokers. +package store + +import ( + "context" + "errors" + "io/fs" +) + +var ( + ErrConflict = errors.New("bucketgit store conflict") + ErrInvalidPath = errors.New("bucketgit invalid store path") + ErrReadOnly = errors.New("bucketgit store is read-only") + ErrNotSupported = errors.New("bucketgit store operation is not supported") +) + +var ErrNotFound = fs.ErrNotExist + +type Reader interface { + Read(ctx context.Context, path string) ([]byte, error) + List(ctx context.Context, prefix string) ([]string, error) +} + +type Writer interface { + Reader + Write(ctx context.Context, path string, data []byte) error + Delete(ctx context.Context, path string) error +} + +// ObjectState is the expected value supplied to an object compare-and-swap. +// Exists distinguishes a missing object from an existing zero-byte object. +type ObjectState struct { + Exists bool + Data []byte +} + +// CompareAndSwapper conditionally replaces an object when its current value +// still matches the caller's observed state. +type CompareAndSwapper interface { + CompareAndSwap(ctx context.Context, path string, expected ObjectState, replacement []byte) error +} + +type RefStore interface { + ListRefs(ctx context.Context) (map[string]string, error) + CompareAndSwapRef(ctx context.Context, ref, oldOID, newOID string) error +} diff --git a/store/storetest/contract.go b/store/storetest/contract.go new file mode 100644 index 0000000..57b67fb --- /dev/null +++ b/store/storetest/contract.go @@ -0,0 +1,165 @@ +// Package storetest provides reusable conformance tests for BucketGit stores. +package storetest + +import ( + "bytes" + "context" + "errors" + "io/fs" + "reflect" + "sync" + "testing" + + "github.com/bucketgit/bgit/store" +) + +type Factory func(t *testing.T) store.Writer + +func Run(t *testing.T, factory Factory) { + t.Helper() + t.Run("round-trip", func(t *testing.T) { + s := factory(t) + ctx := context.Background() + if err := s.Write(ctx, "objects/ab/cdef", []byte("payload")); err != nil { + t.Fatal(err) + } + got, err := s.Read(ctx, "objects/ab/cdef") + if err != nil { + t.Fatal(err) + } + if string(got) != "payload" { + t.Fatalf("read = %q", got) + } + if err := s.Write(ctx, "objects/ab/cdef", []byte("replacement")); err != nil { + t.Fatal(err) + } + got, err = s.Read(ctx, "objects/ab/cdef") + if err != nil { + t.Fatal(err) + } + if string(got) != "replacement" { + t.Fatalf("replacement read = %q", got) + } + paths, err := s.List(ctx, "objects") + if err != nil { + t.Fatal(err) + } + if want := []string{"objects/ab/cdef"}; !reflect.DeepEqual(paths, want) { + t.Fatalf("paths = %#v, want %#v", paths, want) + } + if err := s.Delete(ctx, "objects/ab/cdef"); err != nil { + t.Fatal(err) + } + if _, err := s.Read(ctx, "objects/ab/cdef"); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("read deleted error = %v", err) + } + }) + + t.Run("zero-byte", func(t *testing.T) { + s := factory(t) + ctx := context.Background() + if err := s.Write(ctx, "HEAD", nil); err != nil { + t.Fatal(err) + } + got, err := s.Read(ctx, "HEAD") + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("zero-byte object length = %d", len(got)) + } + }) + + t.Run("large-object", func(t *testing.T) { + s := factory(t) + payload := bytes.Repeat([]byte("bucketgit"), 256*1024) + if err := s.Write(context.Background(), "objects/large", payload); err != nil { + t.Fatal(err) + } + got, err := s.Read(context.Background(), "objects/large") + if err != nil || !bytes.Equal(got, payload) { + t.Fatalf("large read length=%d err=%v", len(got), err) + } + }) + + t.Run("concurrent-complete-writes", func(t *testing.T) { + s := factory(t) + payloads := make([][]byte, 6) + var wait sync.WaitGroup + errorsByWriter := make([]error, len(payloads)) + for i := range payloads { + payloads[i] = bytes.Repeat([]byte{byte('a' + i)}, 128*1024) + wait.Add(1) + go func(index int) { + defer wait.Done() + errorsByWriter[index] = s.Write(context.Background(), "objects/concurrent", payloads[index]) + }(i) + } + wait.Wait() + for _, err := range errorsByWriter { + if err != nil { + t.Fatal(err) + } + } + got, err := s.Read(context.Background(), "objects/concurrent") + if err != nil { + t.Fatal(err) + } + complete := false + for _, payload := range payloads { + if bytes.Equal(got, payload) { + complete = true + break + } + } + if !complete { + t.Fatal("concurrent write produced torn content") + } + }) + + t.Run("invalid-paths", func(t *testing.T) { + s := factory(t) + ctx := context.Background() + for _, value := range []string{"../escape", "/absolute", "a/../../escape", "a\\escape", "%2e%2e/escape", "safe%2f..%2fescape"} { + if err := s.Write(ctx, value, []byte("bad")); !errors.Is(err, store.ErrInvalidPath) { + t.Fatalf("Write(%q) error = %v", value, err) + } + } + }) + + t.Run("cancelled-context", func(t *testing.T) { + s := factory(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := s.Write(ctx, "HEAD", nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled write error = %v", err) + } + }) +} + +func RunCompareAndSwap(t *testing.T, factory Factory) { + t.Helper() + backend := factory(t) + cas, ok := backend.(store.CompareAndSwapper) + if !ok { + t.Fatalf("%T does not implement store.CompareAndSwapper", backend) + } + ctx := context.Background() + path := ".bucketgit/broker-state/v1/contract.json" + if err := cas.CompareAndSwap(ctx, path, store.ObjectState{}, []byte("one")); err != nil { + t.Fatal(err) + } + if err := cas.CompareAndSwap(ctx, path, store.ObjectState{}, []byte("stale")); !errors.Is(err, store.ErrConflict) { + t.Fatalf("missing-state stale update error = %v", err) + } + if err := cas.CompareAndSwap(ctx, path, store.ObjectState{Exists: true, Data: []byte("wrong")}, []byte("stale")); !errors.Is(err, store.ErrConflict) { + t.Fatalf("wrong-value stale update error = %v", err) + } + if err := cas.CompareAndSwap(ctx, path, store.ObjectState{Exists: true, Data: []byte("one")}, []byte("two")); err != nil { + t.Fatal(err) + } + data, err := backend.Read(ctx, path) + if err != nil || string(data) != "two" { + t.Fatalf("read after CAS = %q, %v", data, err) + } +} diff --git a/testsuite/run-local-broker.sh b/testsuite/run-local-broker.sh index 8a61507..86a1840 100755 --- a/testsuite/run-local-broker.sh +++ b/testsuite/run-local-broker.sh @@ -14,7 +14,7 @@ esac export GOCACHE="${GOCACHE:-$(go env GOCACHE 2>/dev/null || printf '/tmp/bgit-gocache')}" export GOMODCACHE="${GOMODCACHE:-$(go env GOMODCACHE 2>/dev/null || printf '/tmp/bgit-gomodcache')}" if [[ "${BGIT_TEST_USE_EXISTING_BINARY:-}" != "1" ]]; then - go build -o bgit . + go build -o bgit ./cmd/bgit fi native_path() { diff --git a/testsuite/run.sh b/testsuite/run.sh index e22dc00..c4d508a 100755 --- a/testsuite/run.sh +++ b/testsuite/run.sh @@ -21,7 +21,7 @@ fi export GOCACHE="${GOCACHE:-$(go env GOCACHE 2>/dev/null || printf '/tmp/bgit-gocache')}" if [[ "${BGIT_TEST_USE_EXISTING_BINARY:-}" != "1" ]]; then - go build -o bgit . + go build -o bgit ./cmd/bgit fi export BGIT="${BGIT:-$ROOT/bgit}" diff --git a/testsuite/sdk-consumer/go.mod b/testsuite/sdk-consumer/go.mod new file mode 100644 index 0000000..1068321 --- /dev/null +++ b/testsuite/sdk-consumer/go.mod @@ -0,0 +1,9 @@ +module example.com/bgit-sdk-consumer + +go 1.25.0 + +require github.com/bucketgit/bgit v0.0.0 + +require golang.org/x/sys v0.45.0 // indirect + +replace github.com/bucketgit/bgit => ../.. diff --git a/testsuite/sdk-consumer/go.sum b/testsuite/sdk-consumer/go.sum new file mode 100644 index 0000000..b9f0115 --- /dev/null +++ b/testsuite/sdk-consumer/go.sum @@ -0,0 +1,4 @@ +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/testsuite/sdk-consumer/sdk_test.go b/testsuite/sdk-consumer/sdk_test.go new file mode 100644 index 0000000..7f6209b --- /dev/null +++ b/testsuite/sdk-consumer/sdk_test.go @@ -0,0 +1,31 @@ +package consumer + +import ( + "bytes" + "context" + "testing" + + localbroker "github.com/bucketgit/bgit/broker/local" + "github.com/bucketgit/bgit/protocol" + "github.com/bucketgit/bgit/repository" + fsstore "github.com/bucketgit/bgit/store/fs" + "github.com/bucketgit/bgit/transport" +) + +func TestExternalModuleCanComposeFilesystemLocalBrokerAndTransport(t *testing.T) { + objects, err := fsstore.New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + _ = repository.Open(objects, objects) + + broker, err := localbroker.New(localbroker.Options{Root: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + repoInfo := protocol.Repository{Provider: "file", Bucket: "demo.git", Logical: "demo.git"} + scoped := broker.Repository(repoInfo) + repo := repository.Open(scoped, scoped) + var input, output bytes.Buffer + _ = transport.ServeUploadPack(context.Background(), repo, &input, &output) +} diff --git a/tools/api_inventory/main.go b/tools/api_inventory/main.go new file mode 100644 index 0000000..621282e --- /dev/null +++ b/tools/api_inventory/main.go @@ -0,0 +1,114 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "sort" + "strings" +) + +var packageDirs = []string{ + "protocol", "store", "store/fs", "store/s3", "store/gcs", + "repository", "broker/client", "broker/capability", "broker/local", "transport", +} + +func main() { + check := flag.String("check", "", "verify that a generated API inventory matches this file") + flag.Parse() + var output bytes.Buffer + fmt.Fprintln(&output, "# BucketGit Exported Go API") + fmt.Fprintln(&output) + fmt.Fprintln(&output, "Generated by `go run ./tools/api_inventory`. This is the public BucketGit Go SDK v1 compatibility surface.") + for _, directory := range packageDirs { + symbols, err := exportedSymbols(directory) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintf(&output, "\n## `github.com/bucketgit/bgit/%s`\n\n", directory) + for _, symbol := range symbols { + fmt.Fprintf(&output, "- `%s`\n", symbol) + } + } + if *check != "" { + current, err := os.ReadFile(*check) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if !bytes.Equal(normalizeLineEndings(current), normalizeLineEndings(output.Bytes())) { + fmt.Fprintf(os.Stderr, "%s is stale; regenerate it with `go run ./tools/api_inventory > %s`\n", *check, *check) + os.Exit(1) + } + return + } + _, _ = os.Stdout.Write(output.Bytes()) +} + +func normalizeLineEndings(data []byte) []byte { + return bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) +} + +func exportedSymbols(directory string) ([]string, error) { + set := token.NewFileSet() + packages, err := parser.ParseDir(set, directory, func(info os.FileInfo) bool { + return !strings.HasSuffix(info.Name(), "_test.go") + }, 0) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", directory, err) + } + var symbols []string + for _, parsed := range packages { + for _, file := range parsed.Files { + for _, declaration := range file.Decls { + switch value := declaration.(type) { + case *ast.FuncDecl: + if !ast.IsExported(value.Name.Name) { + continue + } + if value.Recv == nil { + symbols = append(symbols, "func "+value.Name.Name) + } else { + receiver := receiverName(value.Recv.List[0].Type) + if ast.IsExported(receiver) { + symbols = append(symbols, "method "+receiver+"."+value.Name.Name) + } + } + case *ast.GenDecl: + for _, specification := range value.Specs { + switch spec := specification.(type) { + case *ast.TypeSpec: + if ast.IsExported(spec.Name.Name) { + symbols = append(symbols, "type "+spec.Name.Name) + } + case *ast.ValueSpec: + for _, name := range spec.Names { + if ast.IsExported(name.Name) { + symbols = append(symbols, strings.ToLower(value.Tok.String())+" "+name.Name) + } + } + } + } + } + } + } + } + sort.Strings(symbols) + return symbols, nil +} + +func receiverName(expression ast.Expr) string { + switch value := expression.(type) { + case *ast.Ident: + return value.Name + case *ast.StarExpr: + return receiverName(value.X) + default: + return "receiver" + } +} diff --git a/tools/api_inventory/main_test.go b/tools/api_inventory/main_test.go new file mode 100644 index 0000000..906fb07 --- /dev/null +++ b/tools/api_inventory/main_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "bytes" + "testing" +) + +func TestNormalizeLineEndings(t *testing.T) { + got := normalizeLineEndings([]byte("one\r\ntwo\n")) + if !bytes.Equal(got, []byte("one\ntwo\n")) { + t.Fatalf("normalized = %q", got) + } +} diff --git a/transport/fuzz_test.go b/transport/fuzz_test.go new file mode 100644 index 0000000..8605228 --- /dev/null +++ b/transport/fuzz_test.go @@ -0,0 +1,34 @@ +package transport + +import ( + "bufio" + "bytes" + "context" + "testing" + + "github.com/bucketgit/bgit/repository" +) + +func FuzzReadPacket(f *testing.F) { + f.Add([]byte("0008test")) + f.Add([]byte("0000")) + f.Fuzz(func(t *testing.T, data []byte) { _, _ = ReadPacket(bufio.NewReader(bytes.NewReader(data))) }) +} + +func FuzzDecodeReceivedPack(f *testing.F) { + f.Add([]byte("PACK\x00\x00\x00\x02\x00\x00\x00\x00")) + f.Fuzz(func(t *testing.T, data []byte) { + _, _ = decodeReceivedPack(context.Background(), repository.Open(emptyReader{}, nil), data) + }) +} + +type emptyReader struct{} + +func (emptyReader) Read(context.Context, string) ([]byte, error) { return nil, errMissing } +func (emptyReader) List(context.Context, string) ([]string, error) { return nil, nil } + +type missingError struct{} + +func (missingError) Error() string { return "missing" } + +var errMissing missingError diff --git a/transport/pktline.go b/transport/pktline.go new file mode 100644 index 0000000..b5de200 --- /dev/null +++ b/transport/pktline.go @@ -0,0 +1,150 @@ +// Package transport implements Git smart-protocol transport primitives. +package transport + +import ( + "bufio" + "errors" + "fmt" + "io" + "sort" + "strconv" + "strings" +) + +const ( + UploadPackService = "git-upload-pack" + ReceivePackService = "git-receive-pack" + MaxPayload = 65516 +) + +type PacketKind uint8 + +const ( + DataPacket PacketKind = iota + FlushPacket + DelimiterPacket + ResponseEndPacket +) + +type Packet struct { + Kind PacketKind + Data []byte +} + +func ReadPacket(reader *bufio.Reader) (Packet, error) { + var header [4]byte + if _, err := io.ReadFull(reader, header[:]); err != nil { + return Packet{}, err + } + switch string(header[:]) { + case "0000": + return Packet{Kind: FlushPacket}, nil + case "0001": + return Packet{Kind: DelimiterPacket}, nil + case "0002": + return Packet{Kind: ResponseEndPacket}, nil + } + length, err := strconv.ParseUint(string(header[:]), 16, 16) + if err != nil || length < 4 { + return Packet{}, fmt.Errorf("invalid pkt-line length %q", header) + } + payload := make([]byte, int(length)-4) + if _, err := io.ReadFull(reader, payload); err != nil { + return Packet{}, err + } + return Packet{Kind: DataPacket, Data: payload}, nil +} + +func WritePacket(writer io.Writer, data []byte) error { + if len(data) > MaxPayload { + return errors.New("pkt-line payload too large") + } + header := fmt.Sprintf("%04x", len(data)+4) + if _, err := io.WriteString(writer, header); err != nil { + return err + } + _, err := writer.Write(data) + return err +} + +func WriteString(writer io.Writer, data string) error { return WritePacket(writer, []byte(data)) } +func WriteFlush(writer io.Writer) error { _, err := io.WriteString(writer, "0000"); return err } +func WriteDelimiter(writer io.Writer) error { _, err := io.WriteString(writer, "0001"); return err } +func WriteResponseEnd(writer io.Writer) error { _, err := io.WriteString(writer, "0002"); return err } + +func WriteSideband(writer io.Writer, channel byte, data []byte, maxPayload int) error { + if channel < 1 || channel > 3 { + return fmt.Errorf("invalid sideband channel %d", channel) + } + if maxPayload <= 1 || maxPayload > MaxPayload { + maxPayload = MaxPayload + } + for len(data) > 0 { + size := maxPayload - 1 + if len(data) < size { + size = len(data) + } + payload := make([]byte, size+1) + payload[0] = channel + copy(payload[1:], data[:size]) + if err := WritePacket(writer, payload); err != nil { + return err + } + data = data[size:] + } + return nil +} + +type Capabilities []string + +func (c Capabilities) String() string { return strings.Join(c, " ") } + +func UploadPackCapabilities() Capabilities { + return Capabilities{"multi_ack", "multi_ack_detailed", "no-done", "thin-pack", "side-band", "side-band-64k", "ofs-delta", "no-progress", "include-tag", "allow-tip-sha1-in-want", "allow-reachable-sha1-in-want", "symref=HEAD:refs/heads/main", "object-format=sha1", "agent=bgit"} +} + +func ReceivePackCapabilities() Capabilities { + return Capabilities{"report-status", "report-status-v2", "delete-refs", "side-band-64k", "quiet", "atomic", "ofs-delta", "push-options", "object-format=sha1", "agent=bgit"} +} + +func WriteAdvertisedRefs(writer io.Writer, service string, refs map[string]string, capabilities Capabilities) error { + if service != UploadPackService && service != ReceivePackService { + return fmt.Errorf("unsupported service %q", service) + } + names := make([]string, 0, len(refs)) + for name, oid := range refs { + if (name == "HEAD" || strings.HasPrefix(name, "refs/")) && validSHA1(oid) { + names = append(names, name) + } + } + sort.Strings(names) + if index := sort.SearchStrings(names, "HEAD"); index < len(names) && names[index] == "HEAD" { + names = append([]string{"HEAD"}, append(names[:index], names[index+1:]...)...) + } + if len(names) == 0 { + return WriteFlush(writer) + } + for index, name := range names { + line := strings.TrimSpace(refs[name]) + " " + name + if index == 0 { + line += "\x00" + capabilities.String() + } + if err := WriteString(writer, line+"\n"); err != nil { + return err + } + } + return WriteFlush(writer) +} + +func validSHA1(value string) bool { + value = strings.TrimSpace(value) + if len(value) != 40 { + return false + } + for _, char := range value { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + return true +} diff --git a/transport/pktline_test.go b/transport/pktline_test.go new file mode 100644 index 0000000..f8a8edb --- /dev/null +++ b/transport/pktline_test.go @@ -0,0 +1,38 @@ +package transport + +import ( + "bufio" + "bytes" + "testing" +) + +func TestPacketRoundTripAndSpecialPackets(t *testing.T) { + var buffer bytes.Buffer + if err := WriteString(&buffer, "want abc\n"); err != nil { + t.Fatal(err) + } + if err := WriteFlush(&buffer); err != nil { + t.Fatal(err) + } + reader := bufio.NewReader(&buffer) + packet, err := ReadPacket(reader) + if err != nil || packet.Kind != DataPacket || string(packet.Data) != "want abc\n" { + t.Fatalf("packet = %#v, %v", packet, err) + } + packet, err = ReadPacket(reader) + if err != nil || packet.Kind != FlushPacket { + t.Fatalf("flush = %#v, %v", packet, err) + } +} + +func TestAdvertisedRefs(t *testing.T) { + var buffer bytes.Buffer + err := WriteAdvertisedRefs(&buffer, UploadPackService, map[string]string{"refs/heads/main": "0123456789abcdef0123456789abcdef01234567"}, UploadPackCapabilities()) + if err != nil { + t.Fatal(err) + } + packet, err := ReadPacket(bufio.NewReader(&buffer)) + if err != nil || !bytes.Contains(packet.Data, []byte("side-band-64k")) { + t.Fatalf("advertisement = %q, %v", packet.Data, err) + } +} diff --git a/transport/receive_pack.go b/transport/receive_pack.go new file mode 100644 index 0000000..ce6fb47 --- /dev/null +++ b/transport/receive_pack.go @@ -0,0 +1,650 @@ +package transport + +import ( + "bufio" + "bytes" + "compress/zlib" + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "strings" + + "github.com/bucketgit/bgit/repository" + "github.com/bucketgit/bgit/store" +) + +const zeroOID = "0000000000000000000000000000000000000000" + +const DefaultMaxReceivePackSize int64 = 512 << 20 + +type ReceiveCommand struct { + Old, New repository.OID + Ref string + Delete bool + Create bool +} +type ReceiveRequest struct { + Commands []ReceiveCommand + Capabilities map[string]bool + PushOptions []string + Pack io.Reader +} + +type ReceiveStore interface { + store.Writer + store.RefStore +} + +func ServeReceivePack(ctx context.Context, repo *repository.Repository, target ReceiveStore, input io.Reader, output io.Writer) error { + request, err := ReadReceivePackRequest(input) + if err != nil { + return err + } + if len(request.Commands) == 0 { + return nil + } + received := map[repository.OID]repository.Object{} + if needsPack(request.Commands) { + received, err = ingestPack(ctx, repo, target, request.Pack) + } + if err != nil { + if reportsStatus(request.Capabilities) { + _ = writeReceiveReport(output, request, err, nil) + } + return err + } + commandErrors, applyErr := applyCommands(ctx, repo, target, request.Commands, request.Capabilities["atomic"], received) + if reportsStatus(request.Capabilities) { + if reportErr := writeReceiveReport(output, request, nil, commandErrors); reportErr != nil && applyErr == nil { + return reportErr + } + } + return applyErr +} + +func ReadReceivePackRequest(input io.Reader) (ReceiveRequest, error) { + reader := bufio.NewReader(input) + request := ReceiveRequest{Capabilities: map[string]bool{}} + for { + packet, err := ReadPacket(reader) + if err != nil { + return request, err + } + if packet.Kind == FlushPacket { + if request.Capabilities["push-options"] { + request.PushOptions, err = readPushOptions(reader) + if err != nil { + return request, err + } + } + request.Pack = reader + return request, nil + } + if packet.Kind != DataPacket { + continue + } + command, capabilities, err := parseReceiveCommand(string(packet.Data), len(request.Commands) == 0) + if err != nil { + return request, err + } + request.Commands = append(request.Commands, command) + for capability := range capabilities { + request.Capabilities[capability] = true + } + } +} + +func parseReceiveCommand(line string, first bool) (ReceiveCommand, map[string]bool, error) { + line = strings.TrimRight(line, "\n") + capabilities := map[string]bool{} + if first { + if command, text, ok := strings.Cut(line, "\x00"); ok { + line = command + for _, capability := range strings.Fields(text) { + capabilities[capability] = true + if name, _, ok := strings.Cut(capability, "="); ok { + capabilities[name] = true + } + } + } + } + fields := strings.Fields(line) + if len(fields) != 3 || !strings.HasPrefix(fields[2], "refs/") { + return ReceiveCommand{}, nil, fmt.Errorf("invalid receive-pack command %q", line) + } + oldZero, newZero := fields[0] == zeroOID, fields[1] == zeroOID + var oldOID, newOID repository.OID + var err error + if !oldZero { + oldOID, err = repository.ParseOID(fields[0]) + if err != nil { + return ReceiveCommand{}, nil, err + } + } + if !newZero { + newOID, err = repository.ParseOID(fields[1]) + if err != nil { + return ReceiveCommand{}, nil, err + } + } + return ReceiveCommand{Old: oldOID, New: newOID, Ref: fields[2], Create: oldZero && !newZero, Delete: !oldZero && newZero}, capabilities, nil +} + +func readPushOptions(reader *bufio.Reader) ([]string, error) { + var options []string + for { + packet, err := ReadPacket(reader) + if err != nil { + return nil, err + } + if packet.Kind == FlushPacket { + return options, nil + } + if packet.Kind != DataPacket { + return nil, errors.New("invalid push-options packet") + } + options = append(options, strings.TrimRight(string(packet.Data), "\n")) + } +} + +func applyCommands(ctx context.Context, repo *repository.Repository, target ReceiveStore, commands []ReceiveCommand, atomic bool, received map[repository.OID]repository.Object) (map[string]error, error) { + refs, err := target.ListRefs(ctx) + if err != nil { + return nil, err + } + commandErrors := map[string]error{} + var firstErr error + for _, command := range commands { + if err := validateCommand(ctx, repo, refs, command, received); err != nil { + commandErrors[command.Ref] = err + if firstErr == nil { + firstErr = err + } + } + } + if atomic && firstErr != nil { + for _, command := range commands { + if commandErrors[command.Ref] == nil { + commandErrors[command.Ref] = errors.New("atomic push failed") + } + } + return commandErrors, firstErr + } + for _, command := range commands { + if commandErrors[command.Ref] != nil { + continue + } + oldValue, newValue := command.Old.String(), command.New.String() + if command.Create { + oldValue = "" + } + if command.Delete { + newValue = "" + } + if err := target.CompareAndSwapRef(ctx, command.Ref, oldValue, newValue); err != nil { + commandErrors[command.Ref] = err + if firstErr == nil { + firstErr = err + } + continue + } + if command.Delete { + delete(refs, command.Ref) + } else { + refs[command.Ref] = command.New.String() + } + } + return commandErrors, firstErr +} + +func validateCommand(ctx context.Context, repo *repository.Repository, refs map[string]string, command ReceiveCommand, received map[repository.OID]repository.Object) error { + current, exists := refs[command.Ref] + switch { + case command.Create: + if exists { + return errors.New("ref already exists") + } + case command.Delete: + if !exists { + return errors.New("ref does not exist") + } + if current != command.Old.String() { + return errors.New("stale ref") + } + case command.Old != "" && command.New != "": + if !exists { + return errors.New("ref does not exist") + } + if current != command.Old.String() { + return errors.New("stale ref") + } + if strings.HasPrefix(command.Ref, "refs/heads/") { + ancestor, err := isAncestor(ctx, repo, received, command.Old, command.New) + if err != nil { + return err + } + if !ancestor { + return errors.New("non-fast-forward update") + } + } + default: + return nil + } + if command.Delete { + return nil + } + if _, ok := received[command.New]; ok { + return nil + } + if _, err := repo.Object(ctx, command.New); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("missing object %s", command.New) + } + return err + } + return nil +} + +func isAncestor(ctx context.Context, repo *repository.Repository, received map[repository.OID]repository.Object, ancestor, descendant repository.OID) (bool, error) { + stack, seen := []repository.OID{descendant}, map[repository.OID]struct{}{} + for len(stack) > 0 { + oid := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if oid == ancestor { + return true, nil + } + if _, ok := seen[oid]; ok { + continue + } + seen[oid] = struct{}{} + commit, err := commitFromReceived(ctx, repo, received, oid) + if err != nil { + return false, err + } + stack = append(stack, commit.Parents...) + } + return false, nil +} + +func commitFromReceived(ctx context.Context, repo *repository.Repository, received map[repository.OID]repository.Object, oid repository.OID) (repository.Commit, error) { + if object, ok := received[oid]; ok { + if object.Type != repository.CommitObject { + return repository.Commit{}, fmt.Errorf("object %s is %s, not commit", oid, object.Type) + } + return repository.ParseCommitData(oid, object.Data) + } + return repo.Commit(ctx, oid) +} + +type packedRecord struct { + offset int + typ int + data []byte + baseOffset int + baseOID repository.OID + oid repository.OID +} + +func ingestPack(ctx context.Context, repo *repository.Repository, target store.Writer, input io.Reader) (map[repository.OID]repository.Object, error) { + data, err := io.ReadAll(io.LimitReader(input, DefaultMaxReceivePackSize+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > DefaultMaxReceivePackSize { + return nil, fmt.Errorf("receive pack exceeds %d bytes", DefaultMaxReceivePackSize) + } + objects, err := decodeReceivedPack(ctx, repo, data) + if err != nil { + return nil, err + } + result := make(map[repository.OID]repository.Object, len(objects)) + for _, object := range objects { + if err := writeLooseObject(ctx, target, object); err != nil { + return nil, err + } + result[object.OID] = object + } + return result, nil +} + +func decodeReceivedPack(ctx context.Context, repo *repository.Repository, pack []byte) ([]repository.Object, error) { + if len(pack) < 32 || !bytes.Equal(pack[:4], []byte("PACK")) { + return nil, errors.New("invalid pack file") + } + if sum := sha1.Sum(pack[:len(pack)-20]); !bytes.Equal(sum[:], pack[len(pack)-20:]) { + return nil, errors.New("pack checksum mismatch") + } + version := binaryUint32(pack[4:8]) + if version != 2 && version != 3 { + return nil, fmt.Errorf("unsupported pack version %d", version) + } + count := uint64(binaryUint32(pack[8:12])) + if count > uint64((len(pack)-32)/2) { + return nil, errors.New("invalid pack object count") + } + position := 12 + records := make([]packedRecord, 0, int(count)) + byOffset := map[int]int{} + byOID := map[repository.OID]int{} + for range count { + if position >= len(pack)-20 { + return nil, errors.New("truncated pack object") + } + offset := position + typ, header, err := parsePackedHeader(pack[position:]) + if err != nil { + return nil, err + } + position += header + record := packedRecord{offset: offset, typ: typ} + switch typ { + case 1, 2, 3, 4: + record.data, header, err = inflateEntry(pack[position:], repository.DefaultMaxObjectSize) + if err != nil { + return nil, err + } + position += header + record.oid = hashObject(packObjectType(typ), record.data) + byOID[record.oid] = len(records) + case 6: + base, consumed, err := parseOffsetDelta(pack[position:], uint64(offset)) + if err != nil { + return nil, err + } + position += consumed + record.baseOffset = int(base) + record.data, consumed, err = inflateEntry(pack[position:], repository.DefaultMaxObjectSize) + if err != nil { + return nil, err + } + position += consumed + case 7: + if len(pack)-position < 20 { + return nil, errors.New("truncated ref delta") + } + record.baseOID = repository.OID(hex.EncodeToString(pack[position : position+20])) + position += 20 + record.data, header, err = inflateEntry(pack[position:], repository.DefaultMaxObjectSize) + if err != nil { + return nil, err + } + position += header + default: + return nil, fmt.Errorf("unsupported pack object type %d", typ) + } + byOffset[offset] = len(records) + records = append(records, record) + } + if position != len(pack)-20 { + return nil, errors.New("pack object data does not end at checksum") + } + resolved := map[int]repository.Object{} + resolving := map[int]bool{} + var resolve func(int) (repository.Object, error) + resolve = func(index int) (repository.Object, error) { + if object, ok := resolved[index]; ok { + return object, nil + } + if resolving[index] { + return repository.Object{}, errors.New("cyclic pack delta") + } + resolving[index] = true + defer delete(resolving, index) + record := records[index] + if record.typ >= 1 && record.typ <= 4 { + object := repository.Object{OID: record.oid, Type: packObjectType(record.typ), Data: record.data} + resolved[index] = object + return object, nil + } + var base repository.Object + if record.typ == 6 { + baseIndex, ok := byOffset[record.baseOffset] + if !ok { + return repository.Object{}, errors.New("missing ofs-delta base") + } + var err error + base, err = resolve(baseIndex) + if err != nil { + return repository.Object{}, err + } + } else if baseIndex, ok := byOID[record.baseOID]; ok { + var err error + base, err = resolve(baseIndex) + if err != nil { + return repository.Object{}, err + } + } else { + var err error + base, err = repo.Object(ctx, record.baseOID) + if err != nil { + return repository.Object{}, fmt.Errorf("missing ref-delta base %s", record.baseOID) + } + } + data, err := applyPackDelta(base.Data, record.data) + if err != nil { + return repository.Object{}, err + } + object := repository.Object{Type: base.Type, Data: data} + object.OID = hashObject(object.Type, data) + resolved[index] = object + byOID[object.OID] = index + return object, nil + } + objects := make([]repository.Object, len(records)) + for index := range records { + object, err := resolve(index) + if err != nil { + return nil, err + } + objects[index] = object + } + return objects, nil +} + +func writeLooseObject(ctx context.Context, target store.Writer, object repository.Object) error { + raw := append([]byte(fmt.Sprintf("%s %d\x00", object.Type, len(object.Data))), object.Data...) + var compressed bytes.Buffer + writer := zlib.NewWriter(&compressed) + if _, err := writer.Write(raw); err != nil { + _ = writer.Close() + return err + } + if err := writer.Close(); err != nil { + return err + } + oid := object.OID.String() + return target.Write(ctx, "objects/"+oid[:2]+"/"+oid[2:], compressed.Bytes()) +} + +func writeReceiveReport(output io.Writer, request ReceiveRequest, unpackErr error, commandErrors map[string]error) error { + var report strings.Builder + if unpackErr != nil { + _ = WriteString(&report, "unpack "+unpackErr.Error()+"\n") + } else { + _ = WriteString(&report, "unpack ok\n") + } + for _, command := range request.Commands { + if err := commandErrors[command.Ref]; err != nil { + _ = WriteString(&report, "ng "+command.Ref+" "+err.Error()+"\n") + } else { + _ = WriteString(&report, "ok "+command.Ref+"\n") + } + } + _ = WriteFlush(&report) + if !request.Capabilities["side-band-64k"] { + _, err := io.WriteString(output, report.String()) + return err + } + if err := WriteSideband(output, 1, []byte(report.String()), MaxPayload); err != nil { + return err + } + return WriteFlush(output) +} + +func reportsStatus(c map[string]bool) bool { return c["report-status"] || c["report-status-v2"] } +func needsPack(commands []ReceiveCommand) bool { + for _, command := range commands { + if !command.Delete && command.New != "" { + return true + } + } + return false +} +func binaryUint32(data []byte) uint32 { + return uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) +} +func packObjectType(value int) repository.ObjectType { + return map[int]repository.ObjectType{1: repository.CommitObject, 2: repository.TreeObject, 3: repository.BlobObject, 4: repository.TagObject}[value] +} +func hashObject(typ repository.ObjectType, data []byte) repository.OID { + raw := append([]byte(fmt.Sprintf("%s %d\x00", typ, len(data))), data...) + sum := sha1.Sum(raw) + return repository.OID(hex.EncodeToString(sum[:])) +} + +func parsePackedHeader(data []byte) (int, int, error) { + if len(data) == 0 { + return 0, 0, errors.New("truncated pack object header") + } + current := data[0] + typ, pos := int((current>>4)&7), 1 + for current&0x80 != 0 { + if pos >= len(data) { + return 0, 0, errors.New("truncated pack object header") + } + current = data[pos] + pos++ + } + return typ, pos, nil +} +func inflateEntry(data []byte, maxSize int64) ([]byte, int, error) { + source := bytes.NewReader(data) + reader, err := zlib.NewReader(source) + if err != nil { + return nil, 0, err + } + var output bytes.Buffer + if _, err = io.Copy(&output, io.LimitReader(reader, maxSize+1)); err != nil { + _ = reader.Close() + return nil, 0, err + } + if err = reader.Close(); err != nil { + return nil, 0, err + } + if int64(output.Len()) > maxSize { + return nil, 0, fmt.Errorf("packed object exceeds %d bytes", maxSize) + } + return output.Bytes(), len(data) - source.Len(), nil +} +func parseOffsetDelta(data []byte, current uint64) (uint64, int, error) { + if len(data) == 0 { + return 0, 0, errors.New("truncated ofs delta") + } + value := data[0] + offset, pos := uint64(value&0x7f), 1 + for value&0x80 != 0 { + if pos >= len(data) { + return 0, 0, errors.New("truncated ofs delta") + } + value = data[pos] + pos++ + offset = ((offset + 1) << 7) | uint64(value&0x7f) + } + if offset > current { + return 0, 0, errors.New("invalid ofs delta base") + } + return current - offset, pos, nil +} +func applyPackDelta(base, delta []byte) ([]byte, error) { + baseSize, n, err := deltaVarint(delta) + if err != nil || baseSize != len(base) { + return nil, errors.New("delta base size mismatch") + } + position := n + resultSize, n, err := deltaVarint(delta[position:]) + if err != nil { + return nil, err + } + position += n + if resultSize < 0 || int64(resultSize) > repository.DefaultMaxObjectSize { + return nil, fmt.Errorf("delta result exceeds %d bytes", repository.DefaultMaxObjectSize) + } + capacity := resultSize + if capacity > len(base)+len(delta) { + capacity = len(base) + len(delta) + } + if capacity > 1<<20 { + capacity = 1 << 20 + } + result := make([]byte, 0, capacity) + for position < len(delta) { + opcode := delta[position] + position++ + if opcode&0x80 != 0 { + offset, size := 0, 0 + for i := range 4 { + if opcode&(1<= len(delta) { + return nil, errors.New("truncated delta copy") + } + offset |= int(delta[position]) << (8 * i) + position++ + } + } + for i := range 3 { + if opcode&(1<= len(delta) { + return nil, errors.New("truncated delta copy") + } + size |= int(delta[position]) << (8 * i) + position++ + } + } + if size == 0 { + size = 0x10000 + } + if size > len(base) || offset > len(base)-size { + return nil, errors.New("delta copy exceeds base") + } + if size > resultSize-len(result) { + return nil, errors.New("delta copy exceeds declared result size") + } + result = append(result, base[offset:offset+size]...) + } else { + size := int(opcode) + if size == 0 || size > len(delta)-position { + return nil, errors.New("invalid delta insert") + } + if size > resultSize-len(result) { + return nil, errors.New("delta insert exceeds declared result size") + } + result = append(result, delta[position:position+size]...) + position += size + } + } + if len(result) != resultSize { + return nil, errors.New("delta result size mismatch") + } + return result, nil +} +func deltaVarint(data []byte) (int, int, error) { + value, shift := uint64(0), uint(0) + for index, current := range data { + if shift >= 63 { + return 0, 0, errors.New("delta varint overflow") + } + value |= uint64(current&0x7f) << shift + if current&0x80 == 0 { + if value > uint64(^uint(0)>>1) { + return 0, 0, errors.New("delta size overflow") + } + return int(value), index + 1, nil + } + shift += 7 + } + return 0, 0, errors.New("truncated delta varint") +} diff --git a/transport/receive_pack_test.go b/transport/receive_pack_test.go new file mode 100644 index 0000000..d50ede2 --- /dev/null +++ b/transport/receive_pack_test.go @@ -0,0 +1,141 @@ +package transport + +import ( + "bytes" + "context" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/bucketgit/bgit/repository" + "github.com/bucketgit/bgit/store" + fsstore "github.com/bucketgit/bgit/store/fs" +) + +type receiveMemoryStore struct { + objects map[string][]byte + refs map[string]string +} + +func (s *receiveMemoryStore) Read(_ context.Context, path string) ([]byte, error) { + data, ok := s.objects[path] + if !ok { + return nil, fs.ErrNotExist + } + return data, nil +} +func (s *receiveMemoryStore) List(context.Context, string) ([]string, error) { return nil, nil } +func (s *receiveMemoryStore) Write(_ context.Context, path string, data []byte) error { + s.objects[path] = append([]byte(nil), data...) + return nil +} +func (s *receiveMemoryStore) Delete(_ context.Context, path string) error { + delete(s.objects, path) + return nil +} +func (s *receiveMemoryStore) ListRefs(context.Context) (map[string]string, error) { + result := map[string]string{} + for k, v := range s.refs { + result[k] = v + } + return result, nil +} +func (s *receiveMemoryStore) CompareAndSwapRef(_ context.Context, ref, oldOID, newOID string) error { + if s.refs[ref] != oldOID { + return store.ErrConflict + } + if newOID == "" { + delete(s.refs, ref) + } else { + s.refs[ref] = newOID + } + return nil +} + +func TestServeReceivePackDeletesRef(t *testing.T) { + oid := "0123456789abcdef0123456789abcdef01234567" + target := &receiveMemoryStore{objects: map[string][]byte{}, refs: map[string]string{"refs/heads/old": oid}} + var input bytes.Buffer + if err := WriteString(&input, oid+" "+zeroOID+" refs/heads/old\x00report-status\n"); err != nil { + t.Fatal(err) + } + if err := WriteFlush(&input); err != nil { + t.Fatal(err) + } + var output bytes.Buffer + err := ServeReceivePack(context.Background(), repository.Open(target, nil), target, &input, &output) + if err != nil { + t.Fatal(err) + } + if _, ok := target.refs["refs/heads/old"]; ok { + t.Fatal("ref was not deleted") + } + if !strings.Contains(output.String(), "unpack ok") || !strings.Contains(output.String(), "ok refs/heads/old") { + t.Fatalf("report=%q", output.String()) + } +} + +func TestServeReceivePackAcceptsThinPackWithExistingBase(t *testing.T) { + remote := filepath.Join(t.TempDir(), "remote.git") + runGitTest(t, "", "init", "--bare", remote) + worktree := filepath.Join(t.TempDir(), "work") + runGitTest(t, "", "clone", remote, worktree) + runGitTest(t, worktree, "config", "user.name", "Ada") + runGitTest(t, worktree, "config", "user.email", "ada@example.com") + readme := filepath.Join(worktree, "README.md") + if err := os.WriteFile(readme, []byte(strings.Repeat("alpha\n", 200)), 0o644); err != nil { + t.Fatal(err) + } + runGitTest(t, worktree, "add", "README.md") + runGitTest(t, worktree, "commit", "-m", "Initial") + runGitTest(t, worktree, "push", "origin", "HEAD:refs/heads/main") + old := strings.TrimSpace(runGitTest(t, worktree, "rev-parse", "HEAD")) + if err := os.WriteFile(readme, []byte(strings.Repeat("alpha\n", 180)+strings.Repeat("beta\n", 20)), 0o644); err != nil { + t.Fatal(err) + } + runGitTest(t, worktree, "add", "README.md") + runGitTest(t, worktree, "commit", "-m", "Update") + newOID := strings.TrimSpace(runGitTest(t, worktree, "rev-parse", "HEAD")) + command := exec.Command("git", "-C", worktree, "pack-objects", "--stdout", "--thin", "--revs") + command.Stdin = strings.NewReader(newOID + "\n^" + old + "\n") + pack, err := command.Output() + if err != nil { + t.Fatal(err) + } + target, err := fsstore.New(remote) + if err != nil { + t.Fatal(err) + } + var input bytes.Buffer + if err := WriteString(&input, old+" "+newOID+" refs/heads/main\x00report-status\n"); err != nil { + t.Fatal(err) + } + if err := WriteFlush(&input); err != nil { + t.Fatal(err) + } + _, _ = input.Write(pack) + var output bytes.Buffer + if err := ServeReceivePack(t.Context(), repository.Open(target, target), target, &input, &output); err != nil { + t.Fatal(err) + } + refs, err := target.ListRefs(t.Context()) + if err != nil || refs["refs/heads/main"] != newOID { + t.Fatalf("refs=%#v err=%v", refs, err) + } +} + +func runGitTest(t *testing.T, dir string, args ...string) string { + t.Helper() + command := exec.Command("git", args...) + command.Dir = dir + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, output) + } + return string(output) +} + +var _ ReceiveStore = (*receiveMemoryStore)(nil) diff --git a/transport/remote_helper.go b/transport/remote_helper.go new file mode 100644 index 0000000..86fce8c --- /dev/null +++ b/transport/remote_helper.go @@ -0,0 +1,79 @@ +package transport + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "strings" +) + +type Resolver interface { + Serve(ctx context.Context, address, service string, input io.Reader, output io.Writer) error +} + +type ResolverFunc func(context.Context, string, string, io.Reader, io.Writer) error + +func (f ResolverFunc) Serve(ctx context.Context, address, service string, input io.Reader, output io.Writer) error { + return f(ctx, address, service, input, output) +} + +func ServeRemoteHelper(ctx context.Context, resolver Resolver, args []string, input io.Reader, output, errorOutput io.Writer) error { + address := RemoteHelperAddress(args) + if address == "" { + return errors.New("usage: git-remote-bgit []") + } + if resolver == nil { + return errors.New("git remote helper resolver is required") + } + reader, writer := bufio.NewReader(input), bufio.NewWriter(output) + for { + line, err := reader.ReadString('\n') + if err != nil { + if errors.Is(err, io.EOF) && strings.TrimSpace(line) == "" { + return nil + } + return err + } + line = strings.TrimRight(line, "\r\n") + switch { + case line == "": + return nil + case line == "capabilities": + fmt.Fprintln(writer, "connect") + fmt.Fprintln(writer) + if err := writer.Flush(); err != nil { + return err + } + case strings.HasPrefix(line, "connect "): + service := strings.TrimSpace(strings.TrimPrefix(line, "connect ")) + if service != UploadPackService && service != ReceivePackService { + return fmt.Errorf("unsupported git remote helper service %q", service) + } + fmt.Fprintln(writer) + if err := writer.Flush(); err != nil { + return err + } + return resolver.Serve(ctx, address, service, reader, output) + case strings.HasPrefix(line, "option "): + fmt.Fprintln(writer, "unsupported") + if err := writer.Flush(); err != nil { + return err + } + default: + fmt.Fprintf(errorOutput, "unsupported git remote helper command %q\n", line) + return fmt.Errorf("unsupported git remote helper command %q", line) + } + } +} + +func RemoteHelperAddress(args []string) string { + if len(args) >= 2 { + return strings.TrimSpace(args[1]) + } + if len(args) == 1 { + return strings.TrimSpace(args[0]) + } + return "" +} diff --git a/transport/remote_helper_test.go b/transport/remote_helper_test.go new file mode 100644 index 0000000..ed767b2 --- /dev/null +++ b/transport/remote_helper_test.go @@ -0,0 +1,24 @@ +package transport + +import ( + "bytes" + "context" + "io" + "strings" + "testing" +) + +func TestRemoteHelperConnect(t *testing.T) { + var output bytes.Buffer + called := false + err := ServeRemoteHelper(context.Background(), ResolverFunc(func(_ context.Context, address, service string, input io.Reader, output io.Writer) error { + called = address == "bgit://demo.git" && service == UploadPackService + return nil + }), []string{"origin", "bgit://demo.git"}, strings.NewReader("capabilities\nconnect git-upload-pack\n"), &output, io.Discard) + if err != nil || !called { + t.Fatalf("ServeRemoteHelper = %v, called=%v", err, called) + } + if output.String() != "connect\n\n\n" { + t.Fatalf("output = %q", output.String()) + } +} diff --git a/transport/upload_pack.go b/transport/upload_pack.go new file mode 100644 index 0000000..1cf6bf0 --- /dev/null +++ b/transport/upload_pack.go @@ -0,0 +1,185 @@ +package transport + +import ( + "bufio" + "bytes" + "compress/zlib" + "context" + "crypto/sha1" + "encoding/binary" + "errors" + "fmt" + "io" + "strings" + + "github.com/bucketgit/bgit/repository" +) + +type UploadPackRequest struct { + Wants []repository.OID + Haves []repository.OID + Done bool + Sideband bool + Sideband64 bool + responded bool +} + +func ServeUploadPack(ctx context.Context, repo *repository.Repository, input io.Reader, output io.Writer) error { + request, err := ReadUploadPackRequest(input, output) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + if len(request.Wants) == 0 { + return nil + } + if !request.responded { + if err := WriteString(output, "NAK\n"); err != nil { + return err + } + } + oids, err := repo.ReachableObjects(ctx, request.Wants, request.Haves) + if err != nil { + return err + } + pack, err := EncodePack(ctx, repo, oids) + if err != nil { + return err + } + if request.Sideband || request.Sideband64 { + maxPayload := 1000 + if request.Sideband64 { + maxPayload = MaxPayload + } + if err := WriteSideband(output, 1, pack, maxPayload); err != nil { + return err + } + return WriteFlush(output) + } + _, err = output.Write(pack) + return err +} + +func ReadUploadPackRequest(input io.Reader, responses io.Writer) (UploadPackRequest, error) { + reader := bufio.NewReader(input) + var request UploadPackRequest + for { + packet, err := ReadPacket(reader) + if err != nil { + if errors.Is(err, io.EOF) && len(request.Wants) > 0 { + return request, nil + } + return request, err + } + if packet.Kind != DataPacket { + if len(request.Wants) > 0 && request.Done { + return request, nil + } + if len(request.Wants) > 0 && responses != nil { + if err := WriteString(responses, "NAK\n"); err != nil { + return request, err + } + request.responded = true + } + continue + } + fields := strings.Fields(strings.TrimSpace(string(packet.Data))) + if len(fields) == 0 { + continue + } + switch fields[0] { + case "want": + if len(fields) < 2 { + continue + } + oid, err := repository.ParseOID(fields[1]) + if err != nil { + continue + } + request.Wants = append(request.Wants, oid) + for _, capability := range fields[2:] { + if capability == "side-band" { + request.Sideband = true + } + if capability == "side-band-64k" { + request.Sideband64 = true + } + } + case "have": + if len(fields) == 2 { + if oid, err := repository.ParseOID(fields[1]); err == nil { + request.Haves = append(request.Haves, oid) + } + } + case "done": + request.Done = true + return request, nil + } + } +} + +func EncodePack(ctx context.Context, repo *repository.Repository, oids []repository.OID) ([]byte, error) { + var buffer bytes.Buffer + buffer.WriteString("PACK") + _ = binary.Write(&buffer, binary.BigEndian, uint32(2)) + if uint64(len(oids)) > uint64(^uint32(0)) { + return nil, errors.New("too many pack objects") + } + _ = binary.Write(&buffer, binary.BigEndian, uint32(len(oids))) + for _, oid := range oids { + object, err := repo.Object(ctx, oid) + if err != nil { + return nil, err + } + if err := writePackObject(&buffer, object); err != nil { + return nil, fmt.Errorf("write pack object %s: %w", oid, err) + } + } + sum := sha1.Sum(buffer.Bytes()) + buffer.Write(sum[:]) + return buffer.Bytes(), nil +} + +func writePackObject(writer io.Writer, object repository.Object) error { + types := map[repository.ObjectType]int{repository.CommitObject: 1, repository.TreeObject: 2, repository.BlobObject: 3, repository.TagObject: 4} + typ := types[object.Type] + if typ == 0 { + return fmt.Errorf("unsupported object type %q", object.Type) + } + if err := writePackHeader(writer, typ, len(object.Data)); err != nil { + return err + } + compressor := zlib.NewWriter(writer) + if _, err := compressor.Write(object.Data); err != nil { + _ = compressor.Close() + return err + } + return compressor.Close() +} + +func writePackHeader(writer io.Writer, typ, size int) error { + if size < 0 { + return errors.New("negative pack object size") + } + first := byte((typ << 4) | (size & 0x0f)) + size >>= 4 + if size > 0 { + first |= 0x80 + } + if _, err := writer.Write([]byte{first}); err != nil { + return err + } + for size > 0 { + current := byte(size & 0x7f) + size >>= 7 + if size > 0 { + current |= 0x80 + } + if _, err := writer.Write([]byte{current}); err != nil { + return err + } + } + return nil +}